Skip to main content

orbit_link/
inbox.rs

1//! One bounded mailbox per fleet node: many writers, the lane's owner reads.
2//!
3//! Every node has a lane. An edge worker writes an exchange header into the
4//! lane of the service worker it picked; that worker reads its own lane and
5//! nobody else's. Replies travel the same way in the other direction, which is
6//! why one table serves both — a node is either an edge or a service
7//! worker, so its lane only ever receives one kind of frame.
8//!
9//! ## A full lane refuses
10//!
11//! `orbit_core::Ring` is a fan-out log and overwrites: `slot = counter %
12//! capacity`, unconditionally, with no `Result` to refuse through. Correct for
13//! telemetry, fatal here — an overwritten request is a request that vanished
14//! while its caller waited. A writer that finds this lane full gets
15//! [`Error::InboxFull`] and the caller answers 503.
16//!
17//! ## Producer-owned admission and publication
18//!
19//! A full-width turn identifies each physical slot's reuse. Its claim names
20//! one bounded producer intent and moves FREE -> RESERVED -> COMMITTED or
21//! CANCELLED. Helpers may admit a reservation but never read partial bytes.
22//! The FIFO reader acquires COMMITTED before copying, skips CANCELLED, then
23//! publishes next turn, FREE and head in that order. A slow live producer can
24//! still hold the FIFO head; recovery never infers death from elapsed time.
25//!
26//! Writes contain no ownership lock, new syscall or polling wait. Segment
27//! lifecycle code holds producer registrations with kernel locks and performs
28//! exact-generation recovery only after those holds can be acquired.
29
30#[cfg(unix)]
31use std::io;
32use std::sync::atomic::{AtomicU8, AtomicU32, AtomicU64, Ordering};
33
34use orbit_core::process::ProcessOwner;
35
36#[path = "inbox/claims.rs"]
37pub(crate) mod claims;
38use claims::{CANCELLED, COMMITTED};
39
40use super::{Error, Result};
41
42/// Marks a mapped region as this layout, so a segment left by a different
43/// shape is recognised rather than read as garbage.
44pub(crate) const INBOX_MAGIC: u32 = 0x4f54_5849; // "OTXI"
45
46/// The layout's own version, separate from the geometry. Geometry changes are
47/// caught by Orbit, which refuses a segment sized differently from the code's;
48/// this catches a change that keeps the size and moves the meaning.
49///
50/// 13: a receiver's capacity row declares how many of its window's slots run
51/// at once; the rest are the receiver's own queue.
52///
53/// 12: a receiver's capacity row counts the admission slots it holds, so a
54/// full receiver refuses a reservation in one load.
55///
56/// 11: a producer registration carries a waiting word, and a reader that
57/// frees one of its records rings it.
58///
59/// 10: the owner's growth policy and a coalesced pressure word in the table
60/// header, in what was padding.
61///
62/// 9: atomic live-bound policy and retiring admission window.
63///
64/// 8: receiver-owned work reservations and capacity-change interests.
65///
66/// 7: producer registry, owned claims and monotonic consumer handoff.
67///
68/// 6: an interest table follows the watchers, one entry per lane and per
69/// watcher slot, and a watcher carries a role.
70///
71/// 5: a directory-watcher table follows the lanes. Being woken when the
72/// directory moves no longer requires holding a lane, so a process that serves
73/// nothing (a master) can watch it too.
74///
75/// 4: a lane carries a reactor-listener word for descriptor-native async wake.
76///
77/// 3: a claimed lane's owner holds the lane's lock for as long as it lives, so
78/// a lane whose lock can be taken belongs to a process that is gone. A build
79/// before it holds no lock and would read as dead, so the two never share a
80/// table.
81pub(crate) const INBOX_VERSION: u32 = 13;
82
83/// An owner's declared growth: never fewer than `min` workers, never more
84/// than `max`, `step` added per pressure episode. What a worker is, and how
85/// one starts, is the owner's or its supervisor's.
86#[derive(Clone, Copy, Debug, Eq, PartialEq)]
87pub struct GrowthPolicy {
88    pub min: u16,
89    pub max: u16,
90    pub step: u16,
91}
92
93/// A lane taken back from a process that is gone.
94#[derive(Clone, Copy, Debug, PartialEq, Eq)]
95pub struct Reclaimed {
96    pub lane: usize,
97    /// The incarnation its body streams are stamped with.
98    pub incarnation: u64,
99    pub pid: u32,
100}
101
102/// Longest service name a lane can advertise. Long enough for the names
103/// a site route already uses; a longer one is a configuration error, not a
104/// truncation.
105pub const LANE_NAME_MAX: usize = 47;
106
107/// Longest role tag a lane can carry: `asgi`, `php`, `edge`. A word, not a
108/// description.
109pub const LANE_ROLE_MAX: usize = 8;
110
111/// Copy a role out of shared memory into a fixed buffer and ask `accept`.
112fn accept_role(
113    role: &[u8; LANE_ROLE_MAX],
114    role_len: &AtomicU8,
115    accept: impl FnOnce(&str) -> bool,
116) -> bool {
117    let len = role_len.load(Ordering::Acquire) as usize;
118    if len == 0 || len > LANE_ROLE_MAX {
119        return false;
120    }
121    let mut copy = [0u8; LANE_ROLE_MAX];
122    // SAFETY: `role` is a mapped field of `LANE_ROLE_MAX` bytes and `len` is
123    // within it; the bytes are copied before they are interpreted.
124    copy[..len].copy_from_slice(unsafe {
125        std::slice::from_raw_parts(role.as_ptr(), len)
126    });
127    std::str::from_utf8(&copy[..len]).is_ok_and(accept)
128}
129
130/// Per-lane state, in two cache lines on purpose.
131///
132/// The first line is written on every frame — reservation, read position, the
133/// wake word. The second is written once when a process claims the lane and
134/// read only when an edge is choosing a target. Keeping them apart means the
135/// choosing side can read identity without touching the line the hot counters
136/// live on.
137#[repr(C, align(64))]
138pub(crate) struct LaneHeader {
139    /// Next position a writer may claim. Writers CAS this; nobody else writes
140    /// it.
141    pub(crate) reserve: AtomicU64,
142    /// Next position the owner will read. Only the owner writes it.
143    pub(crate) read: AtomicU64,
144    /// How many live consumers serve this lane. A writer that finds zero
145    /// refuses rather than filling a mailbox nobody empties.
146    pub(crate) consumers: AtomicU32,
147    /// Bumped on every commit: the word a parked reader waits on.
148    pub(crate) changes: AtomicU32,
149    /// The incarnation of the process that owns this lane. A consumer count
150    /// left behind by a dead process is stale, and shared memory outlives the
151    /// process that wrote it.
152    pub(crate) incarnation: AtomicU64,
153    pub(crate) waiters: AtomicU32,
154    /// Set while the lane's Tokio receiver is waiting on its named socket.
155    /// A writer clears it before ringing that socket, so concurrent writes
156    /// coalesce into one reactor wake.
157    pub(crate) reactor: AtomicU32,
158    pub(crate) epoch: AtomicU64,
159    initialized: AtomicU32,
160    /// A capable reader stamps its exact inbox epoch here after setup.
161    paired_bodies_epoch: AtomicU32,
162
163    // ── second line: identity, written once per claim ──────────────────
164    /// Which service this lane serves.
165    ///
166    /// A fleet may hold several — an edge belongs to exactly one fleet, so
167    /// every service it reaches is in that fleet with it. The name lives
168    /// here rather than in a table of its own: identity, presence and
169    /// incarnation are one fact about a lane and want one place, and a
170    /// separate table would be a second thing to keep true.
171    ///
172    /// Read as bytes under the lane's own ordering: a claim publishes the
173    /// length last, so a reader that sees a length sees the bytes before it.
174    pub(crate) service: [u8; LANE_NAME_MAX],
175    pub(crate) service_len: AtomicU8,
176    /// The process that holds this lane.
177    ///
178    /// Written for the operator, not for the protocol: nothing here routes by
179    /// it. It is what answers "is this lane a live process or something a
180    /// crashed one left behind", which the rest of the header cannot say — a
181    /// consumer count is a claim, and a claim outlives the process that made
182    /// it.
183    ///
184    /// Zero means the owner did not say. That is what every lane written
185    /// before this field existed reports, and it is why the layout version
186    /// stays where it is: the bytes were padding and padding was zeroed, so no
187    /// reader of either build misreads the other's.
188    ///
189    /// A hint, not proof. A process id can be reused, so "alive" can name a
190    /// different process; turning death into a 502 needs the death itself, not
191    /// this.
192    pub(crate) pid: AtomicU32,
193    /// What serves the name: `asgi`, `php`, `edge`.
194    ///
195    /// The name is the only address, so two roles under one name would
196    /// share an inbox and split its requests between them without a word.
197    /// This is what lets a join see that and refuse. Several processes of one
198    /// role under a name are how a name scales; two roles are two
199    /// services that were given one name.
200    ///
201    /// Written in bytes that were padding, like `pid`: a lane written before
202    /// it existed reads as length zero — a role that did not say — and the
203    /// layout version stays where it is.
204    pub(crate) role: [u8; LANE_ROLE_MAX],
205    pub(crate) role_len: AtomicU8,
206    _identity_padding: [u8; 3],
207}
208
209// Two cache lines exactly: the identity grew into padding, not past it.
210const _: () = assert!(size_of::<LaneHeader>() == 128);
211
212/// One directory watcher's slot: a wake word and a reactor flag, nothing else.
213///
214/// A watcher is not a lane. It serves no name, takes no frames and never
215/// appears in the directory; it is only told when the directory moves, the
216/// way a cell tells whoever waits on it. Every slot is woken on every
217/// directory change and only one that is armed is rung, so the writer never
218/// learns who is listening and nobody is elected to pass the news on. A
219/// watcher that was not armed at that moment sees the directory generation
220/// moved on its next look; the generation is the truth, the wake a hint.
221///
222/// As many slots as lanes: every process that can join the fleet can watch it
223/// instead.
224#[repr(C, align(64))]
225pub(crate) struct WatcherHeader {
226    /// Bumped on every directory change.
227    pub(crate) changes: AtomicU32,
228    /// Set while the watcher's Tokio task waits on its named socket. Cleared
229    /// by the waker before it rings, so changes in a burst ring once.
230    pub(crate) reactor: AtomicU32,
231    /// The process that holds the slot, for the operator. Zero when free.
232    pub(crate) pid: AtomicU32,
233    /// What the watching process is, in the same words a lane uses (`edge`,
234    /// `php`), so a reader can tell which side of a link a watcher is on.
235    pub(crate) role: [u8; LANE_ROLE_MAX],
236    pub(crate) role_len: AtomicU8,
237    _padding: [u8; 43],
238}
239
240const _: () = assert!(size_of::<WatcherHeader>() == 64);
241
242/// Words in one interest set.
243pub(crate) const INTEREST_WORDS: usize = 7;
244/// Bits in one interest set.
245#[cfg_attr(not(feature = "events"), allow(dead_code))]
246pub(crate) const INTEREST_BITS: usize = INTEREST_WORDS * 64;
247
248/// What one lane or watcher wants to receive: a fixed-size set of routing-key
249/// hashes, owned and written only by the slot's holder.
250///
251/// A set rather than a list, so there is no count to run out of: a false
252/// positive forwards something nobody consumes, and the receiver's own
253/// subscriptions drop it. `seq` is odd while the holder rewrites the set; a
254/// reader that sees it odd, or changed across its read, reads again.
255#[repr(C, align(64))]
256pub(crate) struct InterestEntry {
257    pub(crate) seq: AtomicU32,
258    _padding: [u8; 4],
259    pub(crate) bits: [AtomicU64; INTEREST_WORDS],
260}
261
262const _: () = assert!(size_of::<InterestEntry>() == 64);
263
264/// One frame's slot. `seq` is its full turn; `claim` has two state bits and
265/// a producer-record index. Only an acquired COMMITTED claim publishes bytes.
266#[repr(C, align(64))]
267pub(crate) struct SlotHeader {
268    pub(crate) seq: AtomicU64,
269    pub(crate) claim: AtomicU64,
270    pub(crate) work_epoch: AtomicU64,
271    pub(crate) work_claim: AtomicU64,
272    pub(crate) len: AtomicU32,
273    pub(crate) work_slot: AtomicU32,
274    _padding: [u8; 24],
275}
276
277const _: () = assert!(size_of::<SlotHeader>() == 64);
278
279/// What a writer must know to refuse correctly.
280#[derive(Clone, Copy, Debug, Eq, PartialEq)]
281pub enum Admission {
282    /// There is room and someone is reading.
283    Accepted,
284    /// The lane holds `capacity` frames the owner has not taken yet.
285    Full,
286    /// No live consumer. Writing would leave a request nobody answers.
287    NoConsumer,
288}
289
290/// A lane's geometry, shared by both ends because Orbit refuses to open a
291/// segment shaped differently from the code's.
292#[derive(Clone, Copy, Debug, Eq, PartialEq)]
293pub struct InboxGeometry {
294    pub capacity: usize,
295    pub payload_capacity: usize,
296}
297
298impl InboxGeometry {
299    pub const fn new(
300        capacity: usize,
301        payload_capacity: usize,
302    ) -> Self {
303        Self { capacity, payload_capacity }
304    }
305
306    /// Bytes one lane occupies: its header, then a slot header and payload for
307    /// every position.
308    pub const fn lane_bytes(&self) -> usize {
309        size_of::<LaneHeader>() + self.capacity * self.slot_bytes()
310    }
311
312    const fn slot_bytes(&self) -> usize {
313        // Every SlotHeader (and the following lane/registry) requires 64-byte
314        // alignment, even for a caller-selected odd payload capacity.
315        (size_of::<SlotHeader>() + self.payload_capacity + 63) & !63
316    }
317
318    /// What a fleet's whole table occupies: the header, every lane, one
319    /// watcher slot per lane, then one interest entry per lane and per
320    /// watcher slot. Everything new sits after the lanes, so a lane's offset
321    /// and alignment are what they were before.
322    pub const fn segment_bytes(
323        &self,
324        fleet_capacity: usize,
325    ) -> usize {
326        size_of::<TableHeader>()
327            + fleet_capacity
328                * (self.lane_bytes()
329                    + size_of::<WatcherHeader>()
330                    + 2 * size_of::<InterestEntry>())
331            + claims::registry_bytes(fleet_capacity, self.capacity)
332            + crate::admission::bytes(fleet_capacity, self.capacity)
333    }
334
335    pub(crate) fn validate(&self) -> Result<()> {
336        if self.capacity == 0 || !self.capacity.is_power_of_two() {
337            return Err(Error::Malformed(
338                "inbox capacity must be a power of two",
339            ));
340        }
341        if self.payload_capacity == 0 {
342            return Err(Error::Malformed(
343                "inbox payload capacity must not be zero",
344            ));
345        }
346        if self.capacity > u32::MAX as usize
347            || self.payload_capacity > u32::MAX as usize
348        {
349            return Err(Error::Malformed(
350                "inbox geometry exceeds its wire fields",
351            ));
352        }
353        Ok(())
354    }
355}
356
357// The header keeps its size across versions: new words take its padding.
358const _: () = assert!(size_of::<TableHeader>() == 128);
359
360/// The segment's own header, checked before a single lane is touched.
361#[repr(C, align(64))]
362pub(crate) struct TableHeader {
363    pub(crate) magic: AtomicU32,
364    pub(crate) version: AtomicU32,
365    pub(crate) fleet_capacity: AtomicU32,
366    pub(crate) capacity: AtomicU32,
367    pub(crate) payload_capacity: AtomicU32,
368    /// Bumped whenever a lane's identity changes: a claim publishing a name,
369    /// or a release taking one away.
370    ///
371    /// Which lanes serve a name is settled by those two events and by nothing
372    /// else, so a caller that has read it once may keep the answer until this
373    /// number moves. Depth is not in it — depth changes with every frame and
374    /// is read per request, because it is the load signal rather than the
375    /// address.
376    pub(crate) directory: AtomicU64,
377    /// Nonzero while the fleet's owner wants senders to report unmet demand:
378    /// its declared growth policy, written by the owner alone, after the
379    /// bounds below.
380    pub(crate) policy: AtomicU32,
381    /// Nonzero while a report of unmet demand is pending: set by the first
382    /// sender that finds every receiver full, cleared by the owner once it
383    /// has acted. One episode, one wake.
384    pub(crate) pressure: AtomicU32,
385    /// The owner's declared bounds, so a supervisor of any kind reads them
386    /// from the fleet rather than from a configuration of its own.
387    pub(crate) growth_min: AtomicU32,
388    pub(crate) growth_max: AtomicU32,
389    pub(crate) growth_step: AtomicU32,
390    _padding: [u8; 16],
391}
392
393/// A mapped inbox table.
394///
395/// Holds a raw pointer into shared memory rather than a slice, because the
396/// bytes are written by other processes: a `&[u8]` would promise Rust that
397/// nothing else mutates them, and something else always does.
398pub struct Inbox {
399    pub(crate) base: *mut u8,
400    /// How many lanes the table holds.
401    ///
402    /// It is the fleet capacity today because a lane belongs to a node: a
403    /// process claims the index it was given. If lanes were ever leased from a
404    /// pool instead — so one process could hold several, one per service
405    /// it serves — this is the number that would stop being the fleet's. The
406    /// rest of the interface already speaks in lanes rather than nodes, so
407    /// that change would not reach the edge.
408    pub(crate) lanes: usize,
409    pub(crate) geometry: InboxGeometry,
410    /// Installed once, before this mapping is exposed to SHM publishers.
411    producer: Option<claims::Producer>,
412    /// Named descriptor wake used by a Tokio receiver. Memory-backed tables
413    /// keep using their process-local path.
414    #[cfg(unix)]
415    bell: Option<crate::bell::Bell>,
416}
417
418// The pointer addresses shared memory whose only mutable state is atomics.
419unsafe impl Send for Inbox {}
420unsafe impl Sync for Inbox {}
421
422impl Inbox {
423    /// Take ownership of a mapped region as a fresh table.
424    ///
425    /// Only the table header is written. All-zero is already a table of free
426    /// lanes, and a lane is set up by whoever takes it (`claim_lane`, `join`),
427    /// so nothing here touches a lane's pages: the kernel commits a page when
428    /// it is first written, and a lane nobody claims costs no memory. Zeroing
429    /// the whole segment here once made every byte of it resident — 69 MiB
430    /// for 32 lanes of 128 frames of 17 KiB, whether or not a lane was used.
431    ///
432    /// # Safety
433    ///
434    /// `base` must point at `geometry.segment_bytes(fleet_capacity)` writable
435    /// bytes, all zero, that stay mapped for as long as this `Inbox` lives. A
436    /// newly created shared-memory object is: `ftruncate` extends it with
437    /// zeros.
438    /// Raw constructors do not install a producer death hold. Use
439    /// `LinkSegment` for interprocess publishing; a raw mapping alone is not
440    /// a registered SHM producer. Direct use is for process-local storage.
441    pub unsafe fn initialize(
442        base: *mut u8,
443        fleet_capacity: usize,
444        geometry: InboxGeometry,
445    ) -> Result<Self> {
446        geometry.validate()?;
447        if fleet_capacity == 0 || fleet_capacity > u16::MAX as usize {
448            return Err(Error::Malformed(
449                "inbox fleet capacity must fit a nonzero u16",
450            ));
451        }
452        let inbox = Self {
453            base,
454            lanes: fleet_capacity,
455            geometry,
456            producer: None,
457            #[cfg(unix)]
458            bell: None,
459        };
460
461        let header = inbox.table_header();
462        header.fleet_capacity.store(fleet_capacity as u32, Ordering::Relaxed);
463        header.capacity.store(geometry.capacity as u32, Ordering::Relaxed);
464        header
465            .payload_capacity
466            .store(geometry.payload_capacity as u32, Ordering::Relaxed);
467        header.version.store(INBOX_VERSION, Ordering::Relaxed);
468        // Magic last: a reader that sees it sees a finished header.
469        header.magic.store(INBOX_MAGIC, Ordering::Release);
470        Ok(inbox)
471    }
472
473    /// The fleet capacity and inbox geometry an existing table's creator
474    /// recorded in its header, so a caller can adopt the owner's shape
475    /// instead of supplying one.
476    ///
477    /// # Safety
478    ///
479    /// `base` maps at least a [`TableHeader`] of a region whose creator has
480    /// released its initialization lock.
481    pub(crate) unsafe fn persisted_shape(
482        base: *const u8
483    ) -> Result<(usize, InboxGeometry)> {
484        let header = unsafe { &*base.cast::<TableHeader>() };
485        if header.magic.load(Ordering::Acquire) != INBOX_MAGIC {
486            return Err(Error::Malformed(
487                "inbox segment is not an inbox table",
488            ));
489        }
490        if header.version.load(Ordering::Relaxed) != INBOX_VERSION {
491            return Err(Error::Malformed(
492                "inbox segment was written by another layout version",
493            ));
494        }
495        let geometry = InboxGeometry::new(
496            header.capacity.load(Ordering::Relaxed) as usize,
497            header.payload_capacity.load(Ordering::Relaxed) as usize,
498        );
499        geometry.validate()?;
500        Ok((header.fleet_capacity.load(Ordering::Relaxed) as usize, geometry))
501    }
502
503    /// Attach to a table another process created, refusing anything whose
504    /// header does not match this build.
505    ///
506    /// # Safety
507    ///
508    /// As [`Inbox::initialize`].
509    pub unsafe fn attach(
510        base: *mut u8,
511        fleet_capacity: usize,
512        geometry: InboxGeometry,
513    ) -> Result<Self> {
514        geometry.validate()?;
515        if fleet_capacity == 0 || fleet_capacity > u16::MAX as usize {
516            return Err(Error::Malformed(
517                "inbox fleet capacity must fit a nonzero u16",
518            ));
519        }
520        let inbox = Self {
521            base,
522            lanes: fleet_capacity,
523            geometry,
524            producer: None,
525            #[cfg(unix)]
526            bell: None,
527        };
528        let header = inbox.table_header();
529        if header.magic.load(Ordering::Acquire) != INBOX_MAGIC {
530            return Err(Error::Malformed(
531                "inbox segment is not an inbox table",
532            ));
533        }
534        if header.version.load(Ordering::Relaxed) != INBOX_VERSION {
535            return Err(Error::Malformed(
536                "inbox segment was written by another layout version",
537            ));
538        }
539        if header.fleet_capacity.load(Ordering::Relaxed) as usize
540            != fleet_capacity
541            || header.capacity.load(Ordering::Relaxed) as usize
542                != geometry.capacity
543            || header.payload_capacity.load(Ordering::Relaxed) as usize
544                != geometry.payload_capacity
545        {
546            return Err(Error::Malformed(
547                "inbox segment geometry differs from this build",
548            ));
549        }
550        Ok(inbox)
551    }
552
553    pub fn geometry(&self) -> InboxGeometry {
554        self.geometry
555    }
556
557    pub fn lanes(&self) -> usize {
558        self.lanes
559    }
560
561    /// The lane lifetime to fence writes and capacity waits against.
562    pub fn lane_epoch(
563        &self,
564        lane: usize,
565    ) -> Result<u64> {
566        self.check_lane(lane)?;
567        Ok(self.lane_header(lane).epoch.load(Ordering::Acquire))
568    }
569
570    /// Advertise paired body support after the receiver's tables are ready.
571    /// The caller must own the lane. Older receivers never write this field,
572    /// and the lane's epoch changes even when one of them takes over.
573    pub fn advertise_paired_bodies(
574        &self,
575        lane: usize,
576    ) -> Result<()> {
577        let epoch = u32::try_from(self.lane_epoch(lane)?).map_err(|_| {
578            Error::Malformed("paired body advertisement epoch exhausted")
579        })?;
580        let previous = self
581            .lane_header(lane)
582            .paired_bodies_epoch
583            .swap(epoch, Ordering::AcqRel);
584        if previous != epoch {
585            self.table_header().directory.fetch_add(1, Ordering::AcqRel);
586            self.announce_directory();
587        }
588        Ok(())
589    }
590
591    /// The current epoch in which the receiver advertised paired bodies.
592    pub fn paired_bodies_epoch(
593        &self,
594        lane: usize,
595    ) -> Option<u64> {
596        let epoch = self.lane_epoch(lane).ok()?;
597        let stamp =
598            self.lane_header(lane).paired_bodies_epoch.load(Ordering::Acquire);
599        (stamp != 0 && u64::from(stamp) == epoch).then_some(epoch)
600    }
601
602    /// Claim a lane after fencing out its previous reader. Discard completed
603    /// old work without resetting full queue positions. In-flight live writers
604    /// retain ownership; their eventual frames are skipped by destination epoch.
605    /// Named SHM users should use `LinkSegment::join`, which takes the hold.
606    pub fn claim_lane(
607        &self,
608        lane_index: usize,
609        service: &str,
610        role: &str,
611        incarnation: u64,
612    ) -> Result<()> {
613        self.claim_lane_with_capacity(
614            lane_index,
615            service,
616            role,
617            incarnation,
618            None,
619        )
620    }
621
622    /// Publish the work bound before making this lane discoverable.
623    pub fn claim_lane_with_capacity(
624        &self,
625        lane_index: usize,
626        service: &str,
627        role: &str,
628        incarnation: u64,
629        capacity: Option<usize>,
630    ) -> Result<()> {
631        self.check_lane(lane_index)?;
632        check_identity(service, role)?;
633        self.validate_work_capacity(capacity)?;
634        let pid = ProcessOwner::current()?.pid();
635        self.reset_lane(lane_index)?;
636        self.configure_capacity(lane_index, capacity)?;
637        let lane = self.lane_header(lane_index);
638        lane.consumers.store(1, Ordering::Release);
639        self.publish_identity(lane, service, role, incarnation, pid);
640        self.wake(lane_index, lane);
641        Ok(())
642    }
643
644    /// Take a free lane and say which service this process serves.
645    ///
646    /// There is no pool and no allocator. A service process starts on its
647    /// own — its own binary, its own virtual environment, its own moment — and
648    /// introduces itself here. Several processes serving the same name know
649    /// nothing about each other and do not need to: the lane is won with a
650    /// compare-and-swap on the consumer count, so exactly one of them takes
651    /// it.
652    ///
653    /// The name is published after the lane is won and cleared. In between,
654    /// the lane is claimed but anonymous — and [`Inbox::targets`] matches on
655    /// the name, so nothing is addressed to a lane that is still being made
656    /// ready.
657    ///
658    /// The index that comes back is this process's address in the fleet, for
659    /// the inbox and for its body streams alike.
660    ///
661    /// `hold` takes a free lane's hold before the lane is claimed: whatever it
662    /// returns lives as long as the claim, and a lane it cannot hold is some
663    /// live process's. Over shared memory that is the lane's lock
664    /// ([`LinkSegment::join`](super::LinkSegment::join)); a
665    /// table in private memory has nobody else to hold it.
666    ///
667    /// A name already served by another role is refused, naming the lane
668    /// and process that hold it: two roles under one name would share this
669    /// inbox and split its requests between them. The check runs before the
670    /// lane is taken and again once this lane is published, because two
671    /// processes of different roles joining at the same moment each pass
672    /// the first; the second sees the other, and both refuse — a name given to
673    /// two services is a configuration error either way, and it is caught
674    /// at the door rather than half served.
675    pub fn join<H>(
676        &self,
677        service: &str,
678        role: &str,
679        incarnation: u64,
680        hold: impl FnMut(usize) -> Result<Option<H>>,
681    ) -> Result<(usize, H)> {
682        self.join_with_capacity(service, role, incarnation, None, hold)
683    }
684
685    pub fn join_with_capacity<H>(
686        &self,
687        service: &str,
688        role: &str,
689        incarnation: u64,
690        capacity: Option<usize>,
691        mut hold: impl FnMut(usize) -> Result<Option<H>>,
692    ) -> Result<(usize, H)> {
693        check_identity(service, role)?;
694        self.validate_work_capacity(capacity)?;
695        self.refuse_other_role(service, role, None)?;
696        let pid = ProcessOwner::current()?.pid();
697        for lane_index in 0..self.lanes {
698            let lane = self.lane_header(lane_index);
699            if lane.consumers.load(Ordering::Acquire) != 0 {
700                continue;
701            }
702            let Some(held) = hold(lane_index)? else {
703                continue;
704            };
705            if lane
706                .consumers
707                .compare_exchange(0, 1, Ordering::AcqRel, Ordering::Acquire)
708                .is_err()
709            {
710                continue;
711            }
712            self.reset_lane(lane_index)?;
713            self.configure_capacity(lane_index, capacity)?;
714            self.publish_identity(lane, service, role, incarnation, pid);
715            if let Err(conflict) =
716                self.refuse_other_role(service, role, Some(lane_index))
717            {
718                let _ = self.release_lane(lane_index);
719                return Err(conflict);
720            }
721            self.wake(lane_index, lane);
722            return Ok((lane_index, held));
723        }
724        Err(Error::NoLane)
725    }
726
727    /// Take back a claimed lane whose owner is known to be gone, and say who it
728    /// was: the incarnation is what its body streams are stamped with.
729    ///
730    /// The caller vouches for the death — over shared memory, by holding the
731    /// lane's lock. `None` when the lane was not claimed.
732    pub fn reclaim(
733        &self,
734        lane_index: usize,
735    ) -> Result<Option<Reclaimed>> {
736        self.check_lane(lane_index)?;
737        let lane = self.lane_header(lane_index);
738        if lane.consumers.load(Ordering::Acquire) == 0 {
739            return Ok(None);
740        }
741        let reclaimed = Reclaimed {
742            lane: lane_index,
743            incarnation: lane.incarnation.load(Ordering::Acquire),
744            pid: lane.pid.load(Ordering::Acquire),
745        };
746        self.release_lane(lane_index)?;
747        Ok(Some(reclaimed))
748    }
749
750    /// A lane that serves `service` for another role, as an error.
751    ///
752    /// Only claimed lanes count: a lane its owner released holds nothing, and a
753    /// lane a dead process left is taken back before a join gets here.
754    fn refuse_other_role(
755        &self,
756        service: &str,
757        role: &str,
758        own: Option<usize>,
759    ) -> Result<()> {
760        for lane_index in self.serving(service) {
761            if Some(lane_index) == own {
762                continue;
763            }
764            let lane = self.lane_header(lane_index);
765            if lane.consumers.load(Ordering::Acquire) == 0 {
766                continue;
767            }
768            let held_by = self.role(lane_index)?.unwrap_or_default();
769            if held_by != role {
770                return Err(Error::RoleConflict {
771                    name: service.to_owned(),
772                    role: role.to_owned(),
773                    held_by: if held_by.is_empty() {
774                        "an unnamed role".into()
775                    } else {
776                        held_by
777                    },
778                    lane: lane_index,
779                    pid: lane.pid.load(Ordering::Acquire),
780                });
781            }
782        }
783        Ok(())
784    }
785
786    /// Park until something changes in this lane.
787    ///
788    /// `expected` is the value [`Inbox::wait_word`] gave before the caller
789    /// last drained, and the kernel compares it again under its own lock: a
790    /// writer that committed in between means the wait returns instead of
791    /// starting. There is no interval and no deadline here — the wait ends
792    /// because someone wrote, or because [`Inbox::nudge`] woke it on purpose.
793    pub fn park(
794        &self,
795        lane_index: usize,
796        expected: u32,
797    ) -> Result<()> {
798        self.check_lane(lane_index)?;
799        let lane = self.lane_header(lane_index);
800        // Announce before waiting: a writer bumps the word and then reads this
801        // count, so the two orders cross. If it reads zero anyway, it bumped
802        // the word first and the kernel's own comparison ends the wait below.
803        lane.waiters.fetch_add(1, Ordering::AcqRel);
804        let waited = orbit_core::sync::wait_word(&lane.changes, expected);
805        lane.waiters.fetch_sub(1, Ordering::AcqRel);
806        waited.map_err(Error::Io)
807    }
808
809    /// Wake this lane's reader without writing anything.
810    ///
811    /// For the one case that is not a frame: the owner is shutting down and
812    /// the thread parked on the lane has to notice. A flag the reader checks
813    /// plus this, rather than a timeout it would otherwise have to wait out.
814    pub fn nudge(
815        &self,
816        lane_index: usize,
817    ) -> Result<()> {
818        self.check_lane(lane_index)?;
819        self.wake(lane_index, self.lane_header(lane_index));
820        Ok(())
821    }
822
823    /// The service this lane serves, if it is claimed.
824    pub fn service(
825        &self,
826        lane_index: usize,
827    ) -> Result<Option<String>> {
828        self.check_lane(lane_index)?;
829        let lane = self.lane_header(lane_index);
830        let len = lane.service_len.load(Ordering::Acquire) as usize;
831        if len == 0 || len > LANE_NAME_MAX {
832            return Ok(None);
833        }
834        let bytes =
835            unsafe { std::slice::from_raw_parts(lane.service.as_ptr(), len) };
836        Ok(String::from_utf8(bytes.to_vec()).ok())
837    }
838
839    /// The process that claimed this lane, if it said.
840    pub fn owner(
841        &self,
842        lane_index: usize,
843    ) -> Result<Option<u32>> {
844        self.check_lane(lane_index)?;
845        if self.consumers(lane_index)? == 0 {
846            return Ok(None);
847        }
848        Ok(match self.lane_header(lane_index).pid.load(Ordering::Acquire) {
849            0 => None,
850            pid => Some(pid),
851        })
852    }
853
854    /// The last process incarnation on a lane, including one withdrawn while
855    /// its accepted exchanges drain. Not a routing or liveness observation.
856    pub fn last_owner(
857        &self,
858        lane_index: usize,
859    ) -> Result<Option<(u32, u64)>> {
860        self.check_lane(lane_index)?;
861        let lane = self.lane_header(lane_index);
862        let pid = lane.pid.load(Ordering::Acquire);
863        Ok((pid != 0).then(|| (pid, lane.incarnation.load(Ordering::Acquire))))
864    }
865
866    /// What serves this lane's name, if its owner said.
867    /// Whether this lane's role is one `accept` takes. The role is copied
868    /// into a fixed buffer, as [`Inbox::role`] copies it, but never into a
869    /// `String`: callers ask this of every lane for every event they route.
870    pub fn role_accepted(
871        &self,
872        lane_index: usize,
873        accept: impl FnOnce(&str) -> bool,
874    ) -> bool {
875        if self.check_lane(lane_index).is_err() {
876            return false;
877        }
878        let lane = self.lane_header(lane_index);
879        accept_role(&lane.role, &lane.role_len, accept)
880    }
881
882    /// Whether this watcher slot is held and its role is one `accept` takes.
883    /// Allocation-free, as [`Inbox::role_accepted`].
884    pub fn watcher_role_accepted(
885        &self,
886        slot: usize,
887        accept: impl FnOnce(&str) -> bool,
888    ) -> bool {
889        if self.check_lane(slot).is_err() {
890            return false;
891        }
892        let watcher = self.watcher_header(slot);
893        watcher.pid.load(Ordering::Acquire) != 0
894            && accept_role(&watcher.role, &watcher.role_len, accept)
895    }
896
897    pub fn role(
898        &self,
899        lane_index: usize,
900    ) -> Result<Option<String>> {
901        self.check_lane(lane_index)?;
902        let lane = self.lane_header(lane_index);
903        let len = lane.role_len.load(Ordering::Acquire) as usize;
904        if len == 0 || len > LANE_ROLE_MAX {
905            return Ok(None);
906        }
907        let bytes =
908            unsafe { std::slice::from_raw_parts(lane.role.as_ptr(), len) };
909        Ok(String::from_utf8(bytes.to_vec()).ok())
910    }
911
912    /// Whether this lane advertises exactly this service.
913    ///
914    /// Compared as bytes. [`Inbox::service`] allocates a `String`, and
915    /// this is on the dispatch path: every lane in the table is asked, for
916    /// every exchange. An allocation per lane per request made the table cost
917    /// grow with the fleet, which is the one thing a target scan must not do.
918    fn serves(
919        &self,
920        lane_index: usize,
921        service: &[u8],
922    ) -> bool {
923        let lane = self.lane_header(lane_index);
924        let len = lane.service_len.load(Ordering::Acquire) as usize;
925        if len != service.len() || len > LANE_NAME_MAX {
926            return false;
927        }
928        let name =
929            unsafe { std::slice::from_raw_parts(lane.service.as_ptr(), len) };
930        name == service
931    }
932
933    /// Lanes serving `service` that are ready to take a frame, idlest
934    /// first.
935    ///
936    /// This is the whole target-selection story: not a registry, not a health
937    /// report, not a load metric anyone publishes — the queue an edge is about
938    /// to write into, read directly. A worker that joined a moment ago is
939    /// already here; one that left is already gone.
940    pub fn targets(
941        &self,
942        service: &str,
943    ) -> Vec<(usize, u64)> {
944        self.ready(&self.serving(service))
945    }
946
947    /// The generation of the lane directory: which lanes serve which name.
948    ///
949    /// A caller that resolved a name while this number held may use that
950    /// answer until it moves. It changes when a process claims a lane or
951    /// gives one up, and at no other time — a frame written, read or refused
952    /// leaves it alone.
953    pub fn directory(&self) -> u64 {
954        self.table_header().directory.load(Ordering::Acquire)
955    }
956
957    /// The fleet's owner declares its growth policy: `Some` asks senders to
958    /// report unmet demand and publishes the bounds, `None` (static) asks for
959    /// nothing, and `report_pressure` then writes nothing.
960    pub fn set_growth_policy(
961        &self,
962        policy: Option<GrowthPolicy>,
963    ) {
964        let header = self.table_header();
965        match policy {
966            Some(policy) => {
967                header.growth_min.store(policy.min.into(), Ordering::Relaxed);
968                header.growth_max.store(policy.max.into(), Ordering::Relaxed);
969                header.growth_step.store(policy.step.into(), Ordering::Relaxed);
970                header.policy.store(1, Ordering::Release);
971            }
972            None => header.policy.store(0, Ordering::Release),
973        }
974    }
975
976    /// The growth policy the owner declared, if any.
977    pub fn growth_policy(&self) -> Option<GrowthPolicy> {
978        let header = self.table_header();
979        if header.policy.load(Ordering::Acquire) == 0 {
980            return None;
981        }
982        let bound = |word: &AtomicU32| {
983            u16::try_from(word.load(Ordering::Relaxed)).unwrap_or(u16::MAX)
984        };
985        Some(GrowthPolicy {
986            min: bound(&header.growth_min),
987            max: bound(&header.growth_max),
988            step: bound(&header.growth_step),
989        })
990    }
991
992    /// A sender that found every receiver full reports unmet demand. Only
993    /// the first report of an episode writes and rings the directory
994    /// watchers; every later one costs a load and returns `false`, so any
995    /// number of senders produce one wake. Nothing is written unless the
996    /// owner asked for reports.
997    pub fn report_pressure(&self) -> bool {
998        let header = self.table_header();
999        if header.policy.load(Ordering::Acquire) == 0
1000            || header.pressure.load(Ordering::Acquire) != 0
1001            || header
1002                .pressure
1003                .compare_exchange(0, 1, Ordering::AcqRel, Ordering::Acquire)
1004                .is_err()
1005        {
1006            return false;
1007        }
1008        for slot in 0..self.lanes {
1009            self.wake_watcher(slot);
1010        }
1011        true
1012    }
1013
1014    pub fn pressure_pending(&self) -> bool {
1015        self.table_header().pressure.load(Ordering::Acquire) != 0
1016    }
1017
1018    /// The owner ends the episode once it has acted; the next report starts
1019    /// another. Returns whether one was pending.
1020    pub fn take_pressure(&self) -> bool {
1021        self.table_header().pressure.swap(0, Ordering::AcqRel) != 0
1022    }
1023
1024    /// Which lanes advertise this name, live or not.
1025    ///
1026    /// The walk over every lane lives here, so a caller that keeps the result
1027    /// pays it once per directory generation rather than once per exchange.
1028    /// Presence and capacity are deliberately not checked: those change with
1029    /// traffic, and a list that changed with traffic could not be kept.
1030    pub fn serving(
1031        &self,
1032        service: &str,
1033    ) -> Vec<usize> {
1034        let wanted = service.as_bytes();
1035        (0..self.lanes).filter(|lane| self.serves(*lane, wanted)).collect()
1036    }
1037
1038    /// Of these lanes, the ones that will take a frame now, idlest first.
1039    ///
1040    /// This is the per-exchange half: admission and depth are read every time
1041    /// because both answer "now", and depth is what the choice is made on.
1042    pub fn ready(
1043        &self,
1044        lanes: &[usize],
1045    ) -> Vec<(usize, u64)> {
1046        let mut found: Vec<(usize, u64)> = lanes
1047            .iter()
1048            .copied()
1049            .filter(|lane| {
1050                matches!(self.admits(*lane), Ok(Admission::Accepted))
1051            })
1052            .filter_map(|lane| self.depth(lane).ok().map(|depth| (lane, depth)))
1053            .collect();
1054        found.sort_by_key(|(_, depth)| *depth);
1055        found
1056    }
1057
1058    /// The idlest of these lanes that will take a frame now: the first
1059    /// [`Inbox::ready`] would list, read without building the list.
1060    pub fn idlest(
1061        &self,
1062        lanes: &[usize],
1063    ) -> Option<(usize, u64)> {
1064        lanes
1065            .iter()
1066            .copied()
1067            .filter(|lane| {
1068                matches!(self.admits(*lane), Ok(Admission::Accepted))
1069            })
1070            .filter_map(|lane| self.depth(lane).ok().map(|depth| (lane, depth)))
1071            .min_by_key(|(_, depth)| *depth)
1072    }
1073
1074    /// Give up a lane: its owner is leaving and writers should stop choosing
1075    /// it. In-flight frames are left where they are; the next incarnation
1076    /// clears them.
1077    pub fn release_lane(
1078        &self,
1079        lane_index: usize,
1080    ) -> Result<()> {
1081        self.check_lane(lane_index)?;
1082        let lane = self.lane_header(lane_index);
1083        lane.consumers.store(0, Ordering::Release);
1084        lane.service_len.store(0, Ordering::Release);
1085        lane.role_len.store(0, Ordering::Release);
1086        // Keep the ownership stamp until the next claim. Withdrawing from
1087        // routing is not death: accepted streams may still be draining. A
1088        // later confirmed process exit must still be able to end those sides.
1089        self.clear_interest(lane_index);
1090        std::sync::atomic::fence(Ordering::SeqCst);
1091        self.capacity_changed(lane_index);
1092        self.table_header().directory.fetch_add(1, Ordering::AcqRel);
1093        self.wake(lane_index, lane);
1094        self.announce_directory();
1095        Ok(())
1096    }
1097
1098    /// Whether a writer may choose this lane, and why not when it may not.
1099    ///
1100    /// Capacity and presence, never a duration: a writer refused here has been
1101    /// refused now, not after waiting.
1102    pub fn admits(
1103        &self,
1104        lane_index: usize,
1105    ) -> Result<Admission> {
1106        self.check_lane(lane_index)?;
1107        let lane = self.lane_header(lane_index);
1108        if lane.consumers.load(Ordering::Acquire) == 0 {
1109            return Ok(Admission::NoConsumer);
1110        }
1111        let read = lane.read.load(Ordering::Acquire);
1112        // Read the consumer first. A producer may advance `reserve` between
1113        // these loads, but that only makes the snapshot conservatively
1114        // deeper. The opposite order can pair an old reserve with a newer
1115        // read and wrap the subtraction, falsely reporting a nearly empty
1116        // lane as full.
1117        let reserve = lane.reserve.load(Ordering::Acquire);
1118        if reserve.wrapping_sub(read) >= self.geometry.capacity as u64 {
1119            return Ok(Admission::Full);
1120        }
1121        Ok(Admission::Accepted)
1122    }
1123
1124    /// Frames written to this lane that its owner has not taken yet.
1125    ///
1126    /// This is what lets an edge choose a target by reading rather than
1127    /// guessing: with a socket you would need the other side to report its
1128    /// load; here the queue is the shared memory you are about to write into.
1129    pub fn depth(
1130        &self,
1131        lane_index: usize,
1132    ) -> Result<u64> {
1133        self.check_lane(lane_index)?;
1134        let lane = self.lane_header(lane_index);
1135        let read = lane.read.load(Ordering::Acquire);
1136        let reserve = lane.reserve.load(Ordering::Acquire);
1137        Ok(reserve.wrapping_sub(read))
1138    }
1139
1140    pub fn consumers(
1141        &self,
1142        lane_index: usize,
1143    ) -> Result<u32> {
1144        self.check_lane(lane_index)?;
1145        Ok(self.lane_header(lane_index).consumers.load(Ordering::Acquire))
1146    }
1147
1148    pub fn incarnation(
1149        &self,
1150        lane_index: usize,
1151    ) -> Result<u64> {
1152        self.check_lane(lane_index)?;
1153        Ok(self.lane_header(lane_index).incarnation.load(Ordering::Acquire))
1154    }
1155
1156    /// Put one frame in a node's lane.
1157    ///
1158    /// Bounded admission: claim races help the winner and try a later turn,
1159    /// with at most `capacity` attempts. Full capacity or an exhausted
1160    /// contention budget returns `InboxFull`. A producer whose every record
1161    /// names a frame not read yet, on any lane, gets `ProducerBusy`: it can
1162    /// wait for a reader with `LinkSegment::record_released`. Helpers never
1163    /// commit another producer's bytes.
1164    pub fn write(
1165        &self,
1166        lane_index: usize,
1167        payload: &[u8],
1168    ) -> Result<u64> {
1169        self.write_vectored(lane_index, &[payload])
1170    }
1171
1172    /// [`Inbox::write`] for a frame held in pieces — a head encoded here and a
1173    /// body that already has an owner — copied into the slot one after the
1174    /// other, so they need not be joined into one buffer first.
1175    pub fn write_vectored(
1176        &self,
1177        lane_index: usize,
1178        parts: &[&[u8]],
1179    ) -> Result<u64> {
1180        self.write_frame(lane_index, parts, None, None, false)
1181    }
1182
1183    /// Send only to the specified receiver epoch, including after reserving
1184    /// the producer frame. A replacement never receives this publication.
1185    pub fn write_vectored_at_epoch(
1186        &self,
1187        lane: usize,
1188        epoch: u64,
1189        parts: &[&[u8]],
1190    ) -> Result<u64> {
1191        self.write_frame(lane, parts, None, Some(epoch), false)
1192    }
1193
1194    /// A reply write that distinguishes a full inbox from claim contention.
1195    pub fn write_reply_at_epoch(
1196        &self,
1197        lane: usize,
1198        epoch: u64,
1199        parts: &[&[u8]],
1200    ) -> Result<u64> {
1201        self.write_frame(lane, parts, None, Some(epoch), true)
1202    }
1203
1204    pub(crate) fn write_work(
1205        &self,
1206        ticket: crate::WorkTicket,
1207        parts: &[&[u8]],
1208    ) -> Result<u64> {
1209        self.check_work(ticket)?;
1210        self.write_frame(
1211            usize::from(ticket.lane),
1212            parts,
1213            Some(ticket),
1214            Some(ticket.epoch),
1215            false,
1216        )
1217    }
1218
1219    fn write_frame(
1220        &self,
1221        lane_index: usize,
1222        parts: &[&[u8]],
1223        work: Option<crate::WorkTicket>,
1224        epoch: Option<u64>,
1225        reply: bool,
1226    ) -> Result<u64> {
1227        self.check_lane(lane_index)?;
1228        if reply
1229            && (self.lane_epoch(lane_index)? != epoch.unwrap_or_default()
1230                || self.consumers(lane_index)? == 0)
1231        {
1232            return Err(Error::NoConsumer);
1233        }
1234        let len = parts
1235            .iter()
1236            .try_fold(0usize, |len, part| len.checked_add(part.len()))
1237            .ok_or(Error::FrameTooLarge)?;
1238        if len > self.geometry.payload_capacity {
1239            return Err(Error::FrameTooLarge);
1240        }
1241        let mut reservation = match self.reserve_frame(lane_index) {
1242            Err(Error::InboxContended) if !reply => {
1243                return Err(Error::InboxFull);
1244            }
1245            result => result?,
1246        };
1247        if epoch.is_some_and(|epoch| epoch != reservation.epoch)
1248            || (reply && self.consumers(lane_index)? == 0)
1249        {
1250            return Err(Error::NoConsumer);
1251        }
1252        let position = reservation.position;
1253        let slot = self.slot_header(lane_index, position);
1254        let mut at = self.payload_ptr(lane_index, position);
1255        for part in parts {
1256            // SAFETY: the reservation owns this slot's payload until it
1257            // commits, and the parts together fit its capacity.
1258            unsafe {
1259                std::ptr::copy_nonoverlapping(part.as_ptr(), at, part.len());
1260                at = at.add(part.len());
1261            }
1262        }
1263        #[cfg(test)]
1264        claims::checkpoint("payload");
1265        if let Some(ticket) = work {
1266            slot.work_epoch.store(ticket.epoch, Ordering::Relaxed);
1267            slot.work_slot.store(ticket.slot, Ordering::Relaxed);
1268            slot.work_claim.store(ticket.claim, Ordering::Relaxed);
1269        } else {
1270            slot.work_claim.store(0, Ordering::Relaxed);
1271        }
1272        slot.len.store(len as u32, Ordering::Relaxed);
1273        // Admission is not publication: only COMMITTED releases these bytes.
1274        slot.claim.store(reservation.claim(COMMITTED), Ordering::Release);
1275        #[cfg(test)]
1276        claims::checkpoint("committed");
1277        reservation.committed = true;
1278        self.wake(lane_index, self.lane_header(lane_index));
1279        Ok(position)
1280    }
1281
1282    /// Capture the exclusive upper position of currently reserved turns.
1283    /// After recovering a dead producer, its committed frames precede this
1284    /// position even when a still-live producer owns an earlier turn.
1285    pub fn reserved_through(
1286        &self,
1287        lane: usize,
1288    ) -> Result<u64> {
1289        self.check_lane(lane)?;
1290        Ok(self.lane_header(lane).reserve.load(Ordering::Acquire))
1291    }
1292
1293    /// The next turn the single reader will consume.
1294    pub fn read_position(
1295        &self,
1296        lane: usize,
1297    ) -> Result<u64> {
1298        self.check_lane(lane)?;
1299        Ok(self.lane_header(lane).read.load(Ordering::Acquire))
1300    }
1301
1302    /// Take the next frame from this node's own lane, if one is there.
1303    ///
1304    /// Only the lane's owner may call this. It copies out rather than lending
1305    /// a slice, because the slot is released the moment it returns and another
1306    /// process may fill it immediately.
1307    pub fn read(
1308        &self,
1309        lane_index: usize,
1310        out: &mut Vec<u8>,
1311    ) -> Result<bool> {
1312        match self.read_frame(lane_index, out)? {
1313            None => Ok(false),
1314            Some(None) => Ok(true),
1315            Some(Some(ticket)) => {
1316                self.cancel_work(ticket);
1317                Err(Error::Malformed(
1318                    "work frames require an admission-aware receiver",
1319                ))
1320            }
1321        }
1322    }
1323
1324    /// Copy a frame and its optional admission identity. Only the owning
1325    /// receiver may read, then accept work before dispatching its payload.
1326    pub fn read_frame(
1327        &self,
1328        lane_index: usize,
1329        out: &mut Vec<u8>,
1330    ) -> Result<Option<Option<crate::WorkTicket>>> {
1331        self.read_frame_before(lane_index, out, None)
1332    }
1333
1334    pub(crate) fn read_frame_before(
1335        &self,
1336        lane_index: usize,
1337        out: &mut Vec<u8>,
1338        boundary: Option<u64>,
1339    ) -> Result<Option<Option<crate::WorkTicket>>> {
1340        self.check_lane(lane_index)?;
1341        let lane = self.lane_header(lane_index);
1342        for _ in 0..self.geometry.capacity {
1343            let position = lane.read.load(Ordering::Relaxed);
1344            if boundary.is_some_and(|boundary| position >= boundary)
1345                || position >= lane.reserve.load(Ordering::Acquire)
1346            {
1347                return Ok(None);
1348            }
1349            let slot = self.slot_header(lane_index, position);
1350            let claim = slot.claim.load(Ordering::Acquire);
1351            if slot.seq.load(Ordering::Acquire) != position
1352                || !matches!(claim & 3, COMMITTED | CANCELLED)
1353            {
1354                return Ok(None);
1355            }
1356            let intent = self.intent_for_claim(claim)?;
1357            if intent.position != position || intent.lane != lane_index {
1358                return Err(Error::Malformed(
1359                    "committed inbox intent has the wrong turn",
1360                ));
1361            }
1362            let deliver = claim & 3 == COMMITTED
1363                && intent.incarnation == lane.epoch.load(Ordering::Acquire);
1364            let work_claim = slot.work_claim.load(Ordering::Relaxed);
1365            let work = (work_claim != 0).then(|| crate::WorkTicket {
1366                lane: lane_index as u16,
1367                epoch: slot.work_epoch.load(Ordering::Relaxed),
1368                slot: slot.work_slot.load(Ordering::Relaxed),
1369                claim: work_claim,
1370            });
1371            if deliver {
1372                let len = slot.len.load(Ordering::Relaxed) as usize;
1373                if len > self.geometry.payload_capacity {
1374                    return Err(Error::Malformed(
1375                        "committed inbox length is invalid",
1376                    ));
1377                }
1378                out.clear();
1379                out.reserve(len);
1380                unsafe {
1381                    std::ptr::copy_nonoverlapping(
1382                        self.payload_ptr(lane_index, position),
1383                        out.spare_capacity_mut().as_mut_ptr().cast::<u8>(),
1384                        len,
1385                    );
1386                    out.set_len(len);
1387                }
1388            }
1389            self.consume_frame(lane_index, position, claim)?;
1390            if deliver {
1391                return Ok(Some(work));
1392            }
1393        }
1394        Ok(None)
1395    }
1396
1397    /// The word a parked reader waits on, and its current value.
1398    pub fn wait_word(
1399        &self,
1400        lane_index: usize,
1401    ) -> Result<(&AtomicU32, u32)> {
1402        self.check_lane(lane_index)?;
1403        let lane = self.lane_header(lane_index);
1404        Ok((&lane.changes, lane.changes.load(Ordering::Acquire)))
1405    }
1406
1407    /// Retire queued work only after the caller has acquired the consumer's
1408    /// kernel hold and withdrawn the lane. No live reader may still use it.
1409    pub(crate) fn retire_lane(
1410        &self,
1411        lane_index: usize,
1412    ) -> Result<()> {
1413        self.check_lane(lane_index)?;
1414        if self.consumers(lane_index)? != 0 {
1415            return Err(Error::Malformed("retiring an advertised inbox lane"));
1416        }
1417        self.reset_lane(lane_index)
1418    }
1419
1420    fn reset_lane(
1421        &self,
1422        lane_index: usize,
1423    ) -> Result<()> {
1424        let lane = self.lane_header(lane_index);
1425        // The initial all-zero lane has no published incarnation. Thereafter
1426        // full positions never reset, including after a consumer replacement.
1427        let next_epoch = lane
1428            .epoch
1429            .load(Ordering::Acquire)
1430            .checked_add(1)
1431            .ok_or(Error::Malformed("inbox lane generation exhausted"))?;
1432        if lane.initialized.load(Ordering::Acquire) == 0 {
1433            for position in 0..self.geometry.capacity {
1434                let slot = self.slot_header(lane_index, position as u64);
1435                slot.seq.store(position as u64, Ordering::Release);
1436            }
1437            lane.initialized.store(1, Ordering::Release);
1438        } else {
1439            for _ in 0..self.geometry.capacity {
1440                let position = lane.read.load(Ordering::Relaxed);
1441                if position >= lane.reserve.load(Ordering::Acquire) {
1442                    break;
1443                }
1444                if self
1445                    .slot_header(lane_index, position)
1446                    .seq
1447                    .load(Ordering::Acquire)
1448                    > position
1449                {
1450                    // The previous reader released this turn but died before
1451                    // publishing head. Never reset the monotonic cursor.
1452                    let claim = self
1453                        .slot_header(lane_index, position)
1454                        .claim
1455                        .load(Ordering::Acquire);
1456                    if matches!(claim & 3, COMMITTED | CANCELLED)
1457                        && self.intent_for_claim(claim)?.position == position
1458                    {
1459                        self.consume_frame(lane_index, position, claim)?;
1460                    } else {
1461                        lane.read.store(position + 1, Ordering::Release);
1462                    }
1463                    continue;
1464                }
1465                let claim = self
1466                    .slot_header(lane_index, position)
1467                    .claim
1468                    .load(Ordering::Acquire);
1469                if !matches!(claim & 3, COMMITTED | CANCELLED) {
1470                    break;
1471                }
1472                self.consume_frame(lane_index, position, claim)?;
1473            }
1474        }
1475        // Not `consumers`: presence is how a lane is won, and clearing it here
1476        // would hand a just-claimed lane back to the next process looking for
1477        // a free one. `claim_lane` sets it after this; `join` set it before.
1478        lane.service_len.store(0, Ordering::Relaxed);
1479        lane.epoch.store(next_epoch, Ordering::Release);
1480        self.advertise_inbox_space(lane_index, next_epoch);
1481        std::sync::atomic::fence(Ordering::SeqCst);
1482        self.inbox_space_changed(lane_index);
1483        lane.role_len.store(0, Ordering::Relaxed);
1484        lane.waiters.store(0, Ordering::Relaxed);
1485        lane.reactor.store(0, Ordering::Relaxed);
1486        self.clear_interest(lane_index);
1487        self.recover_work(lane_index);
1488        self.configure_capacity(lane_index, None)?;
1489        Ok(())
1490    }
1491
1492    /// Write a lane's identity: the bytes first, the length last, so a reader
1493    /// that sees a length sees the name it belongs to.
1494    fn publish_identity(
1495        &self,
1496        lane: &LaneHeader,
1497        service: &str,
1498        role: &str,
1499        incarnation: u64,
1500        pid: u32,
1501    ) {
1502        unsafe {
1503            std::ptr::copy_nonoverlapping(
1504                service.as_ptr(),
1505                lane.service.as_ptr().cast_mut(),
1506                service.len(),
1507            );
1508            std::ptr::copy_nonoverlapping(
1509                role.as_ptr(),
1510                lane.role.as_ptr().cast_mut(),
1511                role.len(),
1512            );
1513        }
1514        // The role before the name: a reader that sees the name sees who
1515        // serves it.
1516        lane.role_len.store(role.len() as u8, Ordering::Release);
1517        lane.service_len.store(service.len() as u8, Ordering::Release);
1518        lane.incarnation.store(incarnation, Ordering::Release);
1519        lane.pid.store(pid, Ordering::Release);
1520        // Last, and after the name: a caller that sees the new number sees
1521        // every byte that made it new.
1522        self.table_header().directory.fetch_add(1, Ordering::AcqRel);
1523        self.announce_directory();
1524    }
1525
1526    /// Wake every claimed lane and every watcher once: the directory moved.
1527    /// A reader woken with no frame compares [`Inbox::directory`] with the
1528    /// value it last saw, so a process learns who joined or left from the
1529    /// lane it already reads, or from its watcher slot when it holds no lane,
1530    /// with no thread, registry or interval of its own. Claims and releases
1531    /// are rare; traffic never comes through here.
1532    fn announce_directory(&self) {
1533        for lane_index in 0..self.lanes {
1534            let lane = self.lane_header(lane_index);
1535            if lane.consumers.load(Ordering::Acquire) != 0 {
1536                self.wake(lane_index, lane);
1537            }
1538        }
1539        for slot in 0..self.lanes {
1540            self.wake_watcher(slot);
1541        }
1542    }
1543
1544    /// Bump a watcher's word and ring it if it is waiting. A free slot or one
1545    /// whose holder is not waiting costs one atomic add.
1546    fn wake_watcher(
1547        &self,
1548        slot: usize,
1549    ) {
1550        let watcher = self.watcher_header(slot);
1551        watcher.changes.fetch_add(1, Ordering::SeqCst);
1552        #[cfg(unix)]
1553        if let Some(bell) = &self.bell
1554            && let Err(error) = ring_armed(&watcher.reactor, || {
1555                bell.ring(crate::bell::Doorbell::Watcher(slot))
1556            })
1557        {
1558            tracing::warn!(slot, %error, "Link watcher wake failed");
1559        }
1560    }
1561
1562    /// Bump the word a reader watches, and wake it if one is parked.
1563    ///
1564    /// The bump comes first and the waiter count is read after, so a reader
1565    /// that registered itself in between is still seen; one that registers
1566    /// after finds the word already moved and does not start the wait.
1567    pub(crate) fn wake(
1568        &self,
1569        lane_index: usize,
1570        lane: &LaneHeader,
1571    ) {
1572        lane.changes.fetch_add(1, Ordering::SeqCst);
1573        if lane.waiters.load(Ordering::SeqCst) > 0 {
1574            let _ = orbit_core::sync::wake_word(&lane.changes);
1575        }
1576        #[cfg(unix)]
1577        if let Some(bell) = &self.bell
1578            && let Err(error) = ring_armed(&lane.reactor, || {
1579                bell.ring_lane(lane_index, lane.epoch.load(Ordering::Acquire))
1580            })
1581        {
1582            tracing::warn!(lane = lane_index, %error, "Link lane wake failed");
1583        }
1584    }
1585
1586    /// Tell the process waiting to adopt this Link to look again. Only a
1587    /// hint: nobody may be waiting, and the waiter rechecks what exists.
1588    #[cfg(unix)]
1589    pub(crate) fn ring_adopter(&self) {
1590        if let Some(bell) = &self.bell {
1591            let _ = bell.ring(crate::bell::Doorbell::Adopter);
1592        }
1593    }
1594
1595    #[cfg(unix)]
1596    pub(crate) fn install_bell_with_group(
1597        &mut self,
1598        segment: &str,
1599        group: Option<orbit_core::shm::GroupAccess>,
1600    ) -> Result<()> {
1601        self.bell = Some(
1602            crate::bell::Bell::with_group(segment, group, self.lanes)
1603                .map_err(Error::Io)?,
1604        );
1605        Ok(())
1606    }
1607
1608    /// Bind this mapping's producer bell, rung when a reader frees one of
1609    /// its records while it waits.
1610    #[cfg(all(unix, feature = "tokio"))]
1611    pub(crate) fn bind_producer(&self) -> Result<tokio::net::UnixDatagram> {
1612        let index = self
1613            .producer_index()
1614            .ok_or(Error::Malformed("this inbox mapping has no producer"))?;
1615        let bell = self
1616            .bell
1617            .as_ref()
1618            .ok_or(Error::Malformed("the inbox has no reactor bell"))?;
1619        let socket = bell
1620            .bind(crate::bell::Doorbell::Producer(index))
1621            .map_err(Error::Io)?;
1622        tokio::net::UnixDatagram::from_std(socket).map_err(Error::Io)
1623    }
1624
1625    #[cfg(all(unix, feature = "tokio"))]
1626    pub(crate) fn unbind_producer(&self) {
1627        if let (Some(index), Some(bell)) = (self.producer_index(), &self.bell) {
1628            bell.unbind(crate::bell::Doorbell::Producer(index));
1629        }
1630    }
1631
1632    #[cfg(all(unix, feature = "tokio"))]
1633    pub(crate) fn bind_receiver(
1634        &self,
1635        lane_index: usize,
1636    ) -> Result<tokio::net::UnixDatagram> {
1637        self.check_lane(lane_index)?;
1638        let bell = self
1639            .bell
1640            .as_ref()
1641            .ok_or(Error::Malformed("the inbox has no reactor bell"))?;
1642        let socket = bell
1643            .bind(crate::bell::Doorbell::Lane(lane_index))
1644            .map_err(Error::Io)?;
1645        tokio::net::UnixDatagram::from_std(socket).map_err(Error::Io)
1646    }
1647
1648    #[cfg(all(unix, feature = "tokio"))]
1649    pub(crate) fn unbind_receiver(
1650        &self,
1651        lane_index: usize,
1652    ) {
1653        self.disarm_receiver(lane_index);
1654        if let Some(bell) = &self.bell {
1655            bell.unbind(crate::bell::Doorbell::Lane(lane_index));
1656        }
1657    }
1658
1659    #[cfg(all(unix, feature = "tokio"))]
1660    pub(crate) fn receiver_generation(
1661        &self,
1662        lane_index: usize,
1663    ) -> Result<u32> {
1664        self.check_lane(lane_index)?;
1665        Ok(self.lane_header(lane_index).changes.load(Ordering::SeqCst))
1666    }
1667
1668    #[cfg(all(unix, feature = "tokio"))]
1669    pub(crate) fn arm_receiver(
1670        &self,
1671        lane_index: usize,
1672    ) -> Result<()> {
1673        self.check_lane(lane_index)?;
1674        self.lane_header(lane_index).reactor.store(1, Ordering::SeqCst);
1675        Ok(())
1676    }
1677
1678    /// Lower the lane's arm and say whether it was still raised. One that
1679    /// was not has been taken by a writer, which sends exactly one bell for it.
1680    #[cfg(all(unix, feature = "tokio"))]
1681    pub(crate) fn take_receiver_arm(
1682        &self,
1683        lane_index: usize,
1684    ) -> bool {
1685        lane_index < self.lanes
1686            && self.lane_header(lane_index).reactor.swap(0, Ordering::SeqCst)
1687                != 0
1688    }
1689
1690    #[cfg(all(unix, feature = "tokio"))]
1691    pub(crate) fn disarm_receiver(
1692        &self,
1693        lane_index: usize,
1694    ) {
1695        if lane_index < self.lanes {
1696            self.lane_header(lane_index).reactor.store(0, Ordering::SeqCst);
1697        }
1698    }
1699
1700    /// Take a watcher slot whose hold the caller has just won: clear what a
1701    /// previous holder left and bind its socket.
1702    #[cfg(all(unix, feature = "tokio"))]
1703    pub(crate) fn bind_watcher(
1704        &self,
1705        slot: usize,
1706        role: &str,
1707    ) -> Result<tokio::net::UnixDatagram> {
1708        self.check_lane(slot)?;
1709        if role.is_empty() || role.len() > LANE_ROLE_MAX || !role.is_ascii() {
1710            return Err(Error::Malformed("role tag does not fit a watcher"));
1711        }
1712        let bell = self
1713            .bell
1714            .as_ref()
1715            .ok_or(Error::Malformed("the inbox has no reactor bell"))?;
1716        let watcher = self.watcher_header(slot);
1717        let pid = ProcessOwner::current()?.pid();
1718        watcher.reactor.store(0, Ordering::SeqCst);
1719        self.clear_interest(self.lanes + slot);
1720        let socket = bell
1721            .bind(crate::bell::Doorbell::Watcher(slot))
1722            .map_err(Error::Io)?;
1723        unsafe {
1724            std::ptr::copy_nonoverlapping(
1725                role.as_ptr(),
1726                watcher.role.as_ptr().cast_mut(),
1727                role.len(),
1728            );
1729        }
1730        watcher.role_len.store(role.len() as u8, Ordering::Release);
1731        watcher.pid.store(pid, Ordering::Release);
1732        tokio::net::UnixDatagram::from_std(socket).map_err(Error::Io)
1733    }
1734
1735    #[cfg(all(unix, feature = "tokio"))]
1736    pub(crate) fn unbind_watcher(
1737        &self,
1738        slot: usize,
1739    ) {
1740        if slot >= self.lanes {
1741            return;
1742        }
1743        let watcher = self.watcher_header(slot);
1744        watcher.reactor.store(0, Ordering::SeqCst);
1745        watcher.pid.store(0, Ordering::Release);
1746        watcher.role_len.store(0, Ordering::Release);
1747        self.clear_interest(self.lanes + slot);
1748        if let Some(bell) = &self.bell {
1749            bell.unbind(crate::bell::Doorbell::Watcher(slot));
1750        }
1751    }
1752
1753    #[cfg(all(unix, feature = "tokio"))]
1754    pub(crate) fn watcher_generation(
1755        &self,
1756        slot: usize,
1757    ) -> Result<u32> {
1758        self.check_lane(slot)?;
1759        Ok(self.watcher_header(slot).changes.load(Ordering::SeqCst))
1760    }
1761
1762    #[cfg(all(unix, feature = "tokio"))]
1763    pub(crate) fn arm_watcher(
1764        &self,
1765        slot: usize,
1766    ) -> Result<()> {
1767        self.check_lane(slot)?;
1768        self.watcher_header(slot).reactor.store(1, Ordering::SeqCst);
1769        Ok(())
1770    }
1771
1772    #[cfg(all(unix, feature = "tokio"))]
1773    pub(crate) fn disarm_watcher(
1774        &self,
1775        slot: usize,
1776    ) {
1777        if slot < self.lanes {
1778            self.watcher_header(slot).reactor.store(0, Ordering::SeqCst);
1779        }
1780    }
1781
1782    /// The process holding this watcher slot, if one said.
1783    pub fn watcher(
1784        &self,
1785        slot: usize,
1786    ) -> Result<Option<u32>> {
1787        self.check_lane(slot)?;
1788        Ok(match self.watcher_header(slot).pid.load(Ordering::Acquire) {
1789            0 => None,
1790            pid => Some(pid),
1791        })
1792    }
1793
1794    /// What the process holding this watcher slot said it is, if it holds it.
1795    pub fn watcher_role(
1796        &self,
1797        slot: usize,
1798    ) -> Result<Option<String>> {
1799        self.check_lane(slot)?;
1800        let watcher = self.watcher_header(slot);
1801        if watcher.pid.load(Ordering::Acquire) == 0 {
1802            return Ok(None);
1803        }
1804        let len = watcher.role_len.load(Ordering::Acquire) as usize;
1805        if len == 0 || len > LANE_ROLE_MAX {
1806            return Ok(None);
1807        }
1808        let bytes =
1809            unsafe { std::slice::from_raw_parts(watcher.role.as_ptr(), len) };
1810        Ok(String::from_utf8(bytes.to_vec()).ok())
1811    }
1812
1813    /// Replace one interest entry. Only the holder of the lane or watcher
1814    /// slot it belongs to writes it.
1815    pub(crate) fn write_interest(
1816        &self,
1817        entry: usize,
1818        bits: &[u64; INTEREST_WORDS],
1819    ) {
1820        let interest = self.interest_entry(entry);
1821        // One writer per entry, so the parity is set rather than counted: a
1822        // holder that died between the two stores left it odd, and the next
1823        // holder's first write repairs it.
1824        let writing = interest.seq.load(Ordering::Relaxed) | 1;
1825        interest.seq.store(writing, Ordering::Release);
1826        for (word, value) in interest.bits.iter().zip(bits) {
1827            word.store(*value, Ordering::Release);
1828        }
1829        interest.seq.store(writing.wrapping_add(1), Ordering::Release);
1830    }
1831
1832    /// Read one interest entry as it stood between two of its writes.
1833    ///
1834    /// A read that overlaps a write, or meets an entry its writer left half
1835    /// written, does not wait: it answers "wants everything". That is a false
1836    /// positive, which the set already allows — at worst something is
1837    /// forwarded that nobody consumes — and it never parks the reader on a
1838    /// writer that may be gone.
1839    #[cfg_attr(not(feature = "events"), allow(dead_code))]
1840    pub(crate) fn read_interest(
1841        &self,
1842        entry: usize,
1843    ) -> [u64; INTEREST_WORDS] {
1844        let interest = self.interest_entry(entry);
1845        let before = interest.seq.load(Ordering::Acquire);
1846        if before.is_multiple_of(2) {
1847            let mut bits = [0u64; INTEREST_WORDS];
1848            for (value, word) in bits.iter_mut().zip(&interest.bits) {
1849                *value = word.load(Ordering::Acquire);
1850            }
1851            if interest.seq.load(Ordering::Acquire) == before {
1852                return bits;
1853            }
1854        }
1855        [u64::MAX; INTEREST_WORDS]
1856    }
1857
1858    fn clear_interest(
1859        &self,
1860        entry: usize,
1861    ) {
1862        self.write_interest(entry, &[0; INTEREST_WORDS]);
1863    }
1864
1865    fn interest_entry(
1866        &self,
1867        entry: usize,
1868    ) -> &InterestEntry {
1869        unsafe {
1870            &*self
1871                .base
1872                .add(
1873                    size_of::<TableHeader>()
1874                        + self.lanes
1875                            * (self.geometry.lane_bytes()
1876                                + size_of::<WatcherHeader>())
1877                        + entry * size_of::<InterestEntry>(),
1878                )
1879                .cast::<InterestEntry>()
1880        }
1881    }
1882
1883    fn watcher_header(
1884        &self,
1885        slot: usize,
1886    ) -> &WatcherHeader {
1887        unsafe {
1888            &*self
1889                .base
1890                .add(
1891                    size_of::<TableHeader>()
1892                        + self.lanes * self.geometry.lane_bytes()
1893                        + slot * size_of::<WatcherHeader>(),
1894                )
1895                .cast::<WatcherHeader>()
1896        }
1897    }
1898
1899    pub(crate) fn check_lane(
1900        &self,
1901        lane_index: usize,
1902    ) -> Result<()> {
1903        if lane_index >= self.lanes {
1904            return Err(Error::Malformed("lane is outside the table"));
1905        }
1906        Ok(())
1907    }
1908
1909    fn table_header(&self) -> &TableHeader {
1910        unsafe { &*self.base.cast::<TableHeader>() }
1911    }
1912
1913    fn lane_base(
1914        &self,
1915        lane_index: usize,
1916    ) -> *mut u8 {
1917        unsafe {
1918            self.base.add(
1919                size_of::<TableHeader>()
1920                    + lane_index * self.geometry.lane_bytes(),
1921            )
1922        }
1923    }
1924
1925    pub(crate) fn lane_header(
1926        &self,
1927        lane_index: usize,
1928    ) -> &LaneHeader {
1929        unsafe { &*self.lane_base(lane_index).cast::<LaneHeader>() }
1930    }
1931
1932    fn slot_base(
1933        &self,
1934        lane_index: usize,
1935        position: u64,
1936    ) -> *mut u8 {
1937        let index = (position as usize) & (self.geometry.capacity - 1);
1938        let stride = self.geometry.slot_bytes();
1939        unsafe {
1940            self.lane_base(lane_index)
1941                .add(size_of::<LaneHeader>() + index * stride)
1942        }
1943    }
1944
1945    fn slot_header(
1946        &self,
1947        lane_index: usize,
1948        position: u64,
1949    ) -> &SlotHeader {
1950        unsafe { &*self.slot_base(lane_index, position).cast::<SlotHeader>() }
1951    }
1952
1953    fn payload_ptr(
1954        &self,
1955        lane_index: usize,
1956        position: u64,
1957    ) -> *mut u8 {
1958        unsafe {
1959            self.slot_base(lane_index, position).add(size_of::<SlotHeader>())
1960        }
1961    }
1962}
1963
1964/// A lane's identity must fit it: a name and a role tag, neither empty,
1965/// neither truncated.
1966fn check_identity(
1967    service: &str,
1968    role: &str,
1969) -> Result<()> {
1970    if service.is_empty() || service.len() > LANE_NAME_MAX {
1971        return Err(Error::Malformed("service name does not fit a lane"));
1972    }
1973    if role.is_empty() || role.len() > LANE_ROLE_MAX || !role.is_ascii() {
1974        return Err(Error::Malformed("role tag does not fit a lane"));
1975    }
1976    Ok(())
1977}
1978
1979/// Keep the interest visible after a failed send so another writer can retry.
1980/// A parked reader may still require an external wake when delivery fails.
1981#[cfg(unix)]
1982fn ring_armed(
1983    reactor: &AtomicU32,
1984    ring: impl FnOnce() -> io::Result<()>,
1985) -> io::Result<()> {
1986    if reactor.swap(0, Ordering::SeqCst) == 0 {
1987        return Ok(());
1988    }
1989    if let Err(error) = ring() {
1990        reactor.fetch_or(1, Ordering::SeqCst);
1991        return Err(error);
1992    }
1993    Ok(())
1994}
1995
1996#[cfg(all(test, unix, feature = "tokio"))]
1997#[path = "../tests/unit/bell.rs"]
1998mod bell_tests;