orbit_cell/

lib.rs

//! Typed atomic cells in one fleet-shared SHM table, addressed by id.
//!
//! A cell is a place, not a name. [`Cells::allocate`] takes a slot, stamps
//! its type and hands back an [`Orbital`] handle whose [`CellId`] is the
//! only thing that ever needs to travel: any process in the fleet that
//! [`Cells::open`]s the id reads and updates the same 64 bits, atomically,
//! without a lock and without publishing a frame. `orbit-counter` is the
//! keyed sibling; this is memory rather than a dictionary.
//!
//! Every id carries the slot's generation. [`Cells::release`] retires it, so a
//! handle kept past a release answers [`Error::Stale`] instead of touching
//! whoever took the slot next. Cells have no lifetime of their own: what
//! allocates one is expected to release it.

use std::collections::HashMap;
use std::fmt;
use std::marker::PhantomData;
use std::str::FromStr;
use std::sync::atomic::{AtomicU8, AtomicU32, AtomicU64, Ordering};
use std::sync::{Arc, LazyLock, Mutex, MutexGuard, Weak};

use orbit_core::Fleet;
#[cfg(unix)]
use orbit_core::shm::{ShmRegion, ring_segment_name};

mod lifetime;
mod text;
mod wake;

use lifetime::{Lease, Owners};
use wake::Wake;

pub use text::{
    CELL_TEXT_CAPACITY, CELL_TEXT_KIND, CELL_TEXT_MAX, Text, TextId,
};

/// Reserved Orbit SHM kind for the cell table.
pub const CELL_STATE_KIND: u8 = 187;
/// Number of cells one fleet generation can hold at once.
///
/// Compile-time geometry: `ORBIT_CELL_CAPACITY` in the application's
/// `.cargo/config.toml` overrides the default. Must be a power of two. Peers
/// built with a different value are refused when they open the table.
pub const CELL_CAPACITY: usize = orbit_core::compile::usize_from_env(
    option_env!("ORBIT_CELL_CAPACITY"),
    4_096,
);

pub(crate) const SLOT_EMPTY: u8 = 0;
pub(crate) const SLOT_OCCUPIED: u8 = 1;
pub(crate) const SLOT_RETIRED: u8 = 2;

/// Whether this build can park on a shared word. A table refuses to open
/// where it cannot: a sleep loop wearing the shape of a wait is worse
/// than a clear no, and nothing above here should have to ask again.
#[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "macos"))]
pub(crate) fn waits_supported() -> bool {
    orbit_core::sync::supported()
}

#[cfg(not(any(
    target_os = "linux",
    target_os = "freebsd",
    target_os = "macos"
)))]
pub(crate) fn waits_supported() -> bool {
    false
}

/// Park until `word` no longer holds `expected`, through the platform's
/// shared address wait. There is no polling fallback: a table refuses to
/// open where the platform cannot wait, so by here it can.
#[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "macos"))]
pub(crate) fn wait_on(
    word: &AtomicU32,
    expected: u32,
) -> Result<()> {
    orbit_core::sync::wait_word(word, expected).map_err(Error::Io)
}

#[cfg(not(any(
    target_os = "linux",
    target_os = "freebsd",
    target_os = "macos"
)))]
pub(crate) fn wait_on(
    _word: &AtomicU32,
    _expected: u32,
) -> Result<()> {
    Err(Error::Io(std::io::Error::new(
        std::io::ErrorKind::Unsupported,
        "orbit-cell needs a platform that can wait on a shared word",
    )))
}

/// Wake everyone parked on `word`; nothing to do where nobody can park.
pub(crate) fn wake_on(word: &AtomicU32) {
    #[cfg(any(
        target_os = "linux",
        target_os = "freebsd",
        target_os = "macos"
    ))]
    let _ = orbit_core::sync::wake_word(word);
    #[cfg(not(any(
        target_os = "linux",
        target_os = "freebsd",
        target_os = "macos"
    )))]
    let _ = word;
}

#[cfg(unix)]
const STATE_MAGIC: u32 = 0x43_45_4c_4c; // "CELL"
#[cfg(unix)]
const STATE_VERSION: u16 = 4;

pub type Result<T, E = Error> = std::result::Result<T, E>;

/// Additions are expected: a cause discovered later lands here rather
/// than in a new major version, so a caller matches what it handles and
/// leaves the rest to a catch-all.
#[non_exhaustive]
#[derive(Debug)]
pub enum Error {
    /// The id names a slot nothing occupies, or a generation that has been
    /// released since the id was minted.
    Stale(CellId),
    /// The same, for a text cell.
    StaleText(TextId),
    /// The text would not fit the cell; nothing was written.
    TooLong {
        len: usize,
        max: usize,
    },
    /// The caller's read buffer cannot hold the validated text snapshot.
    BufferTooSmall {
        required: usize,
        available: usize,
    },
    /// A text cell held bytes that are not UTF-8: something wrote past the
    /// contract.
    Corrupt(String),
    /// The id names a live cell of another type.
    TypeMismatch {
        id: CellId,
        expected: &'static str,
        found: &'static str,
    },
    /// The update would leave the integer range.
    Overflow,
    /// The native parking generation reached its limit. Value access still
    /// works; release and reallocate to obtain a new parking generation.
    WakeExhausted,
    /// A bounded text read raced bank reuse, or another live writer holds
    /// the cell. The operation does not spin or wait for that writer.
    Contended,
    /// The full-width text publication sequence cannot advance without ABA.
    RevisionExhausted,
    /// A text owner row cannot mint another distinct incarnation.
    OwnerExhausted,
    /// No slot is reusable: slots may be live, pinned after release, or have
    /// exhausted their generation range.
    Full {
        capacity: usize,
    },
    /// The text is not a cell id.
    Malformed(String),
    Io(std::io::Error),
}

impl fmt::Display for Error {
    fn fmt(
        &self,
        f: &mut fmt::Formatter<'_>,
    ) -> fmt::Result {
        match self {
            Self::Stale(id) => write!(f, "cell {id} has been released"),
            Self::StaleText(id) => {
                write!(f, "text cell {id} has been released")
            }
            Self::TooLong { len, max } => {
                write!(f, "text does not fit the cell: len={len} max={max}")
            }
            Self::BufferTooSmall { required, available } => write!(
                f,
                "text read buffer is too small: required={required} available={available}"
            ),
            Self::Corrupt(id) => {
                write!(f, "text cell {id} holds bytes that are not UTF-8")
            }
            Self::TypeMismatch { id, expected, found } => {
                write!(f, "cell {id} holds {found}, not {expected}")
            }
            Self::Overflow => {
                f.write_str("cell value is outside the integer range")
            }
            Self::WakeExhausted => {
                f.write_str("cell parking generation is exhausted")
            }
            Self::Contended => f.write_str("text cell is contended"),
            Self::RevisionExhausted => {
                f.write_str("text publication sequence is exhausted")
            }
            Self::OwnerExhausted => {
                f.write_str("text owner incarnation is exhausted")
            }
            Self::Full { capacity } => {
                write!(
                    f,
                    "cell table has no reusable slot: capacity={capacity}"
                )
            }
            Self::Malformed(text) => write!(f, "not a cell id: {text:?}"),
            Self::Io(error) => write!(f, "Orbit cell io error: {error}"),
        }
    }
}

impl std::error::Error for Error {
    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
        match self {
            Self::Io(error) => Some(error),
            _ => None,
        }
    }
}

impl From<std::io::Error> for Error {
    fn from(value: std::io::Error) -> Self {
        Self::Io(value)
    }
}

/// The address of one cell: its slot and the generation it was allocated in.
///
/// Prints as `cell:<slot>:<generation>` and parses back, which is the form
/// meant to cross a serialization boundary.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct CellId {
    index: u32,
    generation: u32,
}

impl CellId {
    pub const fn index(self) -> u32 {
        self.index
    }

    pub const fn generation(self) -> u32 {
        self.generation
    }

    /// The two halves in one integer, slot high, generation low.
    pub const fn to_bits(self) -> u64 {
        ((self.index as u64) << 32) | self.generation as u64
    }

    pub const fn from_bits(bits: u64) -> Self {
        Self { index: (bits >> 32) as u32, generation: bits as u32 }
    }
}

impl fmt::Display for CellId {
    fn fmt(
        &self,
        f: &mut fmt::Formatter<'_>,
    ) -> fmt::Result {
        write!(f, "cell:{}:{}", self.index, self.generation)
    }
}

impl FromStr for CellId {
    type Err = Error;

    fn from_str(text: &str) -> Result<Self> {
        let malformed = || Error::Malformed(text.to_owned());
        let rest = text.strip_prefix("cell:").ok_or_else(malformed)?;
        let (index, generation) = rest.split_once(':').ok_or_else(malformed)?;
        Ok(Self {
            index: index.parse().map_err(|_| malformed())?,
            generation: generation.parse().map_err(|_| malformed())?,
        })
    }
}

/// A value that fits one cell: 64 bits, and a tag that says which type they
/// spell.
pub trait CellType: Copy + Send + Sync + 'static {
    const TAG: u8;
    const NAME: &'static str;

    fn to_bits(self) -> u64;
    fn from_bits(bits: u64) -> Self;
}

impl CellType for i64 {
    const TAG: u8 = 1;
    const NAME: &'static str = "int";

    fn to_bits(self) -> u64 {
        self as u64
    }

    fn from_bits(bits: u64) -> Self {
        bits as i64
    }
}

impl CellType for u64 {
    const TAG: u8 = 2;
    const NAME: &'static str = "unsigned";

    fn to_bits(self) -> u64 {
        self
    }

    fn from_bits(bits: u64) -> Self {
        bits
    }
}

impl CellType for f64 {
    const TAG: u8 = 3;
    const NAME: &'static str = "float";

    fn to_bits(self) -> u64 {
        self.to_bits()
    }

    fn from_bits(bits: u64) -> Self {
        f64::from_bits(bits)
    }
}

impl CellType for bool {
    const TAG: u8 = 4;
    const NAME: &'static str = "bool";

    fn to_bits(self) -> u64 {
        u64::from(self)
    }

    fn from_bits(bits: u64) -> Self {
        bits != 0
    }
}

fn type_name(tag: u8) -> &'static str {
    match tag {
        1 => i64::NAME,
        2 => u64::NAME,
        3 => f64::NAME,
        4 => bool::NAME,
        _ => "unknown",
    }
}

/// The fleet's cell tables, scalar and text. Cheap to clone.
#[derive(Clone)]
pub struct Cells {
    backend: CellBackend,
    text: text::TextBackend,
}

#[derive(Clone)]
enum CellBackend {
    InMemory(Arc<MemoryCellTable>),
    #[cfg(unix)]
    Shm(Arc<ShmCellTable>),
}

impl Cells {
    pub fn new(fleet: Arc<Fleet>) -> Result<Self> {
        if !crate::waits_supported() {
            return Err(Error::Io(std::io::Error::new(
                std::io::ErrorKind::Unsupported,
                "orbit-cell needs a platform that can wait on a shared word: Linux, FreeBSD, or macOS 14.4 or later",
            )));
        }
        let backend = if fleet.is_shm() {
            #[cfg(unix)]
            {
                CellBackend::Shm(shm_table(
                    &ring_segment_name(fleet.name(), CELL_STATE_KIND),
                    lifetime::owner_capacity(&fleet),
                )?)
            }
            #[cfg(not(unix))]
            unreachable!("non-Unix fleets cannot use POSIX SHM")
        } else {
            CellBackend::InMemory(memory_table(&fleet))
        };
        let text = text::TextBackend::new(&fleet)?;
        Ok(Self { backend, text })
    }

    /// Take a text cell holding `initial`; up to [`CELL_TEXT_MAX`] bytes.
    pub fn allocate_text(
        &self,
        initial: &str,
    ) -> Result<Text> {
        self.text.allocate(initial)
    }

    /// Recover abandoned text writers after an owner-exit notification.
    ///
    /// Takes only kernel holds whose owners have gone, completes their wake
    /// before clearing ownership, and preserves the last committed text.
    /// This cold-path hook does not infer death from a PID or elapsed time.
    /// Call it from the runtime's process-exit handler; no polling is installed.
    /// Returns the number of recovered writer descriptors. In-memory tables
    /// use RAII for unwinding and have no dead process owners to recover.
    pub fn recover_text_writers(&self) -> Result<usize> {
        self.text.recover_writers()
    }

    /// A handle to a live text cell.
    pub fn open_text(
        &self,
        id: TextId,
    ) -> Result<Text> {
        self.text.open(id)
    }

    pub fn release_text(
        &self,
        id: TextId,
    ) -> Result<()> {
        self.text.release(id)
    }

    pub fn is_text_live(
        &self,
        id: TextId,
    ) -> bool {
        self.text.is_live(id)
    }

    /// Take a free slot, stamp it with `T` and `initial`, and return the
    /// handle. The id inside it is what other processes open.
    pub fn allocate<T: CellType>(
        &self,
        initial: T,
    ) -> Result<Orbital<T>> {
        let bits = initial.to_bits();
        let (id, lease) = self.with_structure(|slots, hint, owners| {
            let capacity = slots.len();
            let start = hint.load(Ordering::Relaxed) as usize;
            let allocate = || -> Result<Option<(CellId, Arc<Lease>)>> {
                for offset in 0..capacity {
                    let index = (start + offset) & (capacity - 1);
                    let slot = &slots[index];
                    if matches!(
                        slot.state.load(Ordering::Acquire),
                        SLOT_EMPTY | SLOT_RETIRED
                    ) && slot.generation.load(Ordering::Relaxed) != u32::MAX
                        && !owners.pinned(index)
                    {
                        let lease = owners.lease(index)?;
                        let generation = slot.install(T::TAG, bits);
                        hint.store(
                            ((index + 1) & (capacity - 1)) as u32,
                            Ordering::Relaxed,
                        );
                        return Ok(Some((
                            CellId { index: index as u32, generation },
                            lease,
                        )));
                    }
                }
                Ok(None)
            };
            if let Some(allocated) = allocate()? {
                return Ok(allocated);
            }
            if owners.reclaim_dead()?
                && let Some(allocated) = allocate()?
            {
                return Ok(allocated);
            }
            Err(Error::Full { capacity })
        })?;
        Ok(Orbital {
            cells: self.clone(),
            id,
            _type: PhantomData,
            _lease: lease,
        })
    }

    /// A handle to a live cell of type `T`.
    pub fn open<T: CellType>(
        &self,
        id: CellId,
    ) -> Result<Orbital<T>> {
        let lease = self.with_structure(|slots, _, owners| {
            let slot = slots
                .get(id.index as usize)
                .filter(|slot| slot.is(id))
                .ok_or(Error::Stale(id))?;
            let found = slot.tag.load(Ordering::Relaxed);
            if found != T::TAG {
                return Err(Error::TypeMismatch {
                    id,
                    expected: T::NAME,
                    found: type_name(found),
                });
            }
            owners.lease(id.index as usize)
        })?;
        Ok(Orbital {
            cells: self.clone(),
            id,
            _type: PhantomData,
            _lease: lease,
        })
    }

    /// Retire this generation immediately. Existing handles go stale, but
    /// keep its physical slot pinned until their last clone is dropped.
    pub fn release(
        &self,
        id: CellId,
    ) -> Result<()> {
        self.with_structure(|slots, _, _| {
            let slot = slots
                .get(id.index as usize)
                .filter(|slot| slot.is(id))
                .ok_or(Error::Stale(id))?;
            slot.state.store(SLOT_RETIRED, Ordering::Release);
            // Whoever is parked on it finds the slot gone and answers Stale.
            slot.changed();
            Ok(())
        })
    }

    /// Whether `id` names a live cell right now.
    pub fn is_live(
        &self,
        id: CellId,
    ) -> bool {
        self.slot(id).is_ok()
    }

    /// Clear both tables during quiescent owner boot.
    pub fn reset_all(&self) -> Result<()> {
        self.with_structure(|slots, hint, _| {
            for slot in slots {
                slot.state.store(SLOT_RETIRED, Ordering::Release);
                slot.changed();
            }
            hint.store(0, Ordering::Relaxed);
            Ok(())
        })?;
        self.text.reset_all()
    }

    /// Remove the cell SHM object. Existing mappings remain valid until their
    /// processes release them.
    #[cfg(unix)]
    pub fn unlink(&self) -> Result<()> {
        match &self.backend {
            CellBackend::InMemory(_) => self.reset_all()?,
            CellBackend::Shm(table) => {
                table.shared.region.unlink().map_err(Error::Io)?
            }
        }
        self.text.unlink()
    }

    fn slots(&self) -> &[CellSlot] {
        match &self.backend {
            CellBackend::InMemory(table) => &table.slots,
            #[cfg(unix)]
            CellBackend::Shm(table) => table.slots(),
        }
    }

    fn slot(
        &self,
        id: CellId,
    ) -> Result<&CellSlot> {
        self.slots()
            .get(id.index as usize)
            .filter(|slot| slot.is(id))
            .ok_or(Error::Stale(id))
    }

    fn with_structure<T>(
        &self,
        operation: impl FnOnce(&[CellSlot], &AtomicU32, &Owners) -> Result<T>,
    ) -> Result<T> {
        match &self.backend {
            CellBackend::InMemory(table) => {
                let _local = lock_unpoisoned(&table.structural_lock);
                operation(&table.slots, &table.hint, &table.owners)
            }
            #[cfg(unix)]
            CellBackend::Shm(table) => table.with_structure(operation),
        }
    }
}

/// What [`Orbital::add_until`] did. A bounded add either takes some of what
/// is left or finds nothing left; there is no third answer, and no value
/// past the limit is ever written.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum Claim {
    /// `taken` can be less than asked for: the last caller gets the
    /// remainder rather than being refused, so no unit goes unclaimed.
    Took { taken: i64, after: i64 },
    /// The limit was already reached. Nothing was added, nothing is left.
    Exhausted,
}

/// One cell, typed. Cheap to clone; every clone is the same place.
#[derive(Clone)]
pub struct Orbital<T: CellType> {
    cells: Cells,
    id: CellId,
    _type: PhantomData<T>,
    _lease: Arc<Lease>,
}

impl<T: CellType> Orbital<T> {
    pub fn id(&self) -> CellId {
        self.id
    }

    pub fn load(&self) -> Result<T> {
        Ok(T::from_bits(self.slot()?.value.load(Ordering::Acquire)))
    }

    pub fn store(
        &self,
        value: T,
    ) -> Result<()> {
        let slot = self.slot()?;
        slot.value.store(value.to_bits(), Ordering::Release);
        slot.changed();
        Ok(())
    }

    /// Replace the value and return what it was.
    pub fn swap(
        &self,
        value: T,
    ) -> Result<T> {
        let slot = self.slot()?;
        let previous = slot.value.swap(value.to_bits(), Ordering::AcqRel);
        slot.changed();
        Ok(T::from_bits(previous))
    }

    /// The write count, the thing [`Self::wait_changed`] waits on. Take it
    /// before reading the value, and the wait misses nothing in between.
    pub fn version(&self) -> Result<u32> {
        Ok(self.slot()?.changes.load(Ordering::Acquire))
    }

    /// Park until the cell has been written since `since`, then return the
    /// count now. Coalescing: ten writes while parked wake the caller once,
    /// and [`Self::load`] gives the latest. A released cell wakes every waiter
    /// with [`Error::Stale`]. Blocking; an async runtime wraps it.
    pub fn wait_changed(
        &self,
        since: u32,
    ) -> Result<u32> {
        loop {
            let slot = self.slot()?;
            let now = slot.changes.load(Ordering::SeqCst);
            if now != since {
                return Ok(now);
            }
            slot.wake
                .wait_if(|| slot.changes.load(Ordering::SeqCst) == since)?;
        }
    }

    /// Store `new` only if the cell still holds `current`. `Ok(Ok(previous))`
    /// on success, `Ok(Err(actual))` when it held something else.
    pub fn compare_exchange(
        &self,
        current: T,
        new: T,
    ) -> Result<std::result::Result<T, T>> {
        let slot = self.slot()?;
        let exchanged = slot.value.compare_exchange(
            current.to_bits(),
            new.to_bits(),
            Ordering::AcqRel,
            Ordering::Acquire,
        );
        if exchanged.is_ok() {
            slot.changed();
        }
        Ok(exchanged.map(T::from_bits).map_err(T::from_bits))
    }

    /// [`Self::wait_changed`], but a second word can end it. The waiter
    /// still parks until something wakes it; `interrupt` is what tells it
    /// which kind of wake arrived. `Ok(Some(count))` is a write,
    /// `Ok(None)` is the interrupt.
    ///
    /// This is the shape a caller needs when the wait may outlive its
    /// reason — a process that has to shut down while a script waits on a
    /// cell nothing will write again. It is not a timeout: nothing here
    /// measures time, and a wait nobody interrupts is
    /// [`Self::wait_changed`] exactly.
    ///
    /// The canceller sets `interrupt` and then calls
    /// [`Self::wake_waiters`], in that order. Set-then-wake is what closes
    /// the window: the wake advances a separate parking generation before
    /// waking. A waiter either sees the flag or cannot park on the old
    /// generation. Neither the value nor its version is changed by cancellation.
    pub fn wait_changed_until(
        &self,
        since: u32,
        interrupt: &core::sync::atomic::AtomicBool,
    ) -> Result<Option<u32>> {
        loop {
            if interrupt.load(Ordering::SeqCst) {
                return Ok(None);
            }

            let slot = self.slot()?;
            let now = slot.changes.load(Ordering::SeqCst);
            if now != since {
                return Ok(Some(now));
            }

            slot.wake.wait_if(|| {
                slot.changes.load(Ordering::SeqCst) == since
                    && !interrupt.load(Ordering::SeqCst)
            })?;
        }
    }

    /// Wake everyone parked on this cell without writing to it. On its own
    /// this does not change the value version — a waiter re-checks it, finds
    /// it where it was and parks again — so it is only useful after the word the
    /// waiter also watches has been set. See [`Self::wait_changed_until`].
    pub fn wake_waiters(&self) -> Result<()> {
        self.slot()?.wake.notify();
        Ok(())
    }

    /// Retire the generation. Other handles go stale immediately; the slot
    /// becomes reusable only when the last old handle is dropped.
    pub fn release(self) -> Result<()> {
        self.cells.release(self.id)
    }

    fn slot(&self) -> Result<&CellSlot> {
        self.cells.slot(self.id)
    }
}

impl Orbital<i64> {
    /// Add `by` (negative to subtract) and return the value after.
    pub fn fetch_add(
        &self,
        by: i64,
    ) -> Result<i64> {
        let slot = self.slot()?;
        slot.value
            .try_update(Ordering::AcqRel, Ordering::Acquire, |bits| {
                (bits as i64).checked_add(by).map(|value| value as u64)
            })
            .map(|previous| {
                slot.changed();
                (previous as i64) + by
            })
            .map_err(|_| Error::Overflow)
    }

    pub fn fetch_sub(
        &self,
        by: i64,
    ) -> Result<i64> {
        let slot = self.slot()?;
        slot.value
            .try_update(Ordering::AcqRel, Ordering::Acquire, |bits| {
                (bits as i64).checked_sub(by).map(|value| value as u64)
            })
            .map(|previous| {
                slot.changed();
                (previous as i64) - by
            })
            .map_err(|_| Error::Overflow)
    }

    /// Add up to `by` while the value stays at or under `limit`, in one
    /// compare-and-swap, and say what was taken.
    ///
    /// This is the operation a fleet needs to divide a fixed amount of work
    /// between processes that cannot see each other. `fetch_add` followed by
    /// a test cannot do it: the add lands before the caller learns it went
    /// too far, so N callers overshoot by up to N-1 between them. Here the
    /// bound is decided inside the swap, so the value **never** passes
    /// `limit` however many callers race, and the one that finds less than
    /// `by` left takes the remainder instead of being turned away.
    ///
    /// Taking more than one at a time is how the cost comes down: the
    /// contended word is touched once per claim rather than once per unit.
    /// `limit` is the ceiling for the total, never for a single call.
    ///
    /// The positive distance to the ceiling may exceed `i64::MAX`. Clamping
    /// that distance is safe because `by` itself cannot exceed `i64::MAX`;
    /// the chosen increment and result remain representable.
    ///
    /// **A claim is not a lease.** What this hands back is gone from the
    /// total the moment the swap lands, and nothing gives it back: a caller
    /// that takes a hundred, does forty and then dies leaves sixty that
    /// nobody will ever do, while the count says all hundred were taken. So
    /// `by` is not only a speed dial, it is the exposure — the most one
    /// death can lose. It is the same trade a database sequence makes with
    /// its cache size, and it leaves the same kind of gap. Work that must
    /// not be lost wants a lease whose owner notices a death, which is
    /// `orbit-pool`, not a counter.
    pub fn add_until(
        &self,
        by: i64,
        limit: i64,
    ) -> Result<Claim> {
        if by <= 0 {
            return Err(Error::Overflow);
        }
        let slot = self.slot()?;
        loop {
            let current = slot.value.load(Ordering::Acquire) as i64;
            if current >= limit {
                return Ok(Claim::Exhausted);
            }
            let taken = by.min(limit.saturating_sub(current));
            let after = current + taken;
            if slot
                .value
                .compare_exchange(
                    current as u64,
                    after as u64,
                    Ordering::AcqRel,
                    Ordering::Acquire,
                )
                .is_ok()
            {
                slot.changed();
                return Ok(Claim::Took { taken, after });
            }
            // Lost the race; somebody else moved the value. Read it again
            // rather than retry blind: what is left may now be less, or
            // nothing.
        }
    }
}

impl Orbital<u64> {
    pub fn fetch_add(
        &self,
        by: u64,
    ) -> Result<u64> {
        let slot = self.slot()?;
        slot.value
            .try_update(Ordering::AcqRel, Ordering::Acquire, |bits| {
                bits.checked_add(by)
            })
            .map(|previous| {
                slot.changed();
                previous + by
            })
            .map_err(|_| Error::Overflow)
    }

    pub fn fetch_sub(
        &self,
        by: u64,
    ) -> Result<u64> {
        let slot = self.slot()?;
        slot.value
            .try_update(Ordering::AcqRel, Ordering::Acquire, |bits| {
                bits.checked_sub(by)
            })
            .map(|previous| {
                slot.changed();
                previous - by
            })
            .map_err(|_| Error::Overflow)
    }
}

impl Orbital<f64> {
    /// Add `by` and return the value after. A compare-and-swap loop: floats
    /// have no fetch-add of their own.
    pub fn fetch_add(
        &self,
        by: f64,
    ) -> Result<f64> {
        let slot = self.slot()?;
        let previous = slot
            .value
            .try_update(Ordering::AcqRel, Ordering::Acquire, |bits| {
                Some((f64::from_bits(bits) + by).to_bits())
            })
            .unwrap_or_else(|bits| bits);
        slot.changed();
        Ok(f64::from_bits(previous) + by)
    }
}

impl Orbital<bool> {
    /// Set the flag and return whether it was already set.
    pub fn set(&self) -> Result<bool> {
        self.swap(true)
    }

    /// Clear the flag and return whether it was set.
    pub fn clear(&self) -> Result<bool> {
        self.swap(false)
    }
}

#[repr(C, align(64))]
struct CellStateHeader {
    #[cfg(unix)]
    magic: u32,
    #[cfg(not(unix))]
    _magic: u32,
    #[cfg(unix)]
    version: u16,
    #[cfg(not(unix))]
    _version: u16,
    header_size: u16,
    capacity: u32,
    slot_size: u32,
    /// Where the next allocation starts looking; a hint, never a promise.
    hint: AtomicU32,
    owners: u32,
    _reserved: [u8; 40],
}

impl CellStateHeader {
    fn new(owners: usize) -> Self {
        Self {
            #[cfg(unix)]
            magic: STATE_MAGIC,
            #[cfg(not(unix))]
            _magic: 0,
            #[cfg(unix)]
            version: STATE_VERSION,
            #[cfg(not(unix))]
            _version: 0,
            header_size: size_of::<Self>() as u16,
            capacity: CELL_CAPACITY as u32,
            slot_size: size_of::<CellSlot>() as u32,
            hint: AtomicU32::new(0),
            owners: owners as u32,
            _reserved: [0; 40],
        }
    }
}

/// One cell, laid out for shared memory: a cache line of its own, so two hot
/// cells never contend, and every field atomic, so the mapping is never
/// borrowed `&mut` while another process reads it.
#[repr(C, align(64))]
struct CellSlot {
    state: AtomicU8,
    tag: AtomicU8,
    _reserved: [u8; 2],
    generation: AtomicU32,
    value: AtomicU64,
    /// Public value version, bumped by every write and release. Native
    /// parking uses the separate sequence in `wake`.
    changes: AtomicU32,
    /// Native parking sequence and waiter count, separate from `changes`.
    wake: Wake,
    _padding: [u8; 36],
}

impl CellSlot {
    fn empty() -> Self {
        Self {
            state: AtomicU8::new(SLOT_EMPTY),
            tag: AtomicU8::new(0),
            _reserved: [0; 2],
            generation: AtomicU32::new(0),
            value: AtomicU64::new(0),
            changes: AtomicU32::new(0),
            wake: Wake::new(),
            _padding: [0; 36],
        }
    }

    /// After a write: count it, and wake whoever is waiting for one. `SeqCst`
    /// on both sides of the exchange with [`Orbital::wait_changed`], so a
    /// waiter that announced itself just before this write is either seen
    /// here or sees the new count itself; never neither.
    fn changed(&self) {
        self.changes.fetch_add(1, Ordering::SeqCst);
        self.wake.changed();
    }

    fn is(
        &self,
        id: CellId,
    ) -> bool {
        // `Acquire` on `state` publishes the generation written by `install`.
        self.state.load(Ordering::Acquire) == SLOT_OCCUPIED
            && self.generation.load(Ordering::Relaxed) == id.generation
    }

    /// Under the structural lock. Returns the generation the new cell lives in;
    /// it is bumped on every install, so no released id ever matches again.
    fn install(
        &self,
        tag: u8,
        value: u64,
    ) -> u32 {
        let generation = self
            .generation
            .load(Ordering::Relaxed)
            .checked_add(1)
            .expect("exhausted generations are never reused");
        self.tag.store(tag, Ordering::Relaxed);
        self.value.store(value, Ordering::Relaxed);
        self.changes.store(0, Ordering::Relaxed);
        self.wake.reset();
        self.generation.store(generation, Ordering::Relaxed);
        self.state.store(SLOT_OCCUPIED, Ordering::Release);
        generation
    }
}

pub(crate) fn lock_unpoisoned<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
    mutex.lock().unwrap_or_else(|error| error.into_inner())
}

struct MemoryCellTable {
    /// Held so the fleet's address, which keys the registry, cannot be
    /// reused by another fleet while this table lives.
    _fleet: Arc<Fleet>,
    slots: Vec<CellSlot>,
    hint: AtomicU32,
    structural_lock: Mutex<()>,
    owners: Owners,
}

impl MemoryCellTable {
    fn new(fleet: Arc<Fleet>) -> Self {
        Self {
            _fleet: fleet,
            slots: (0..CELL_CAPACITY).map(|_| CellSlot::empty()).collect(),
            hint: AtomicU32::new(0),
            structural_lock: Mutex::new(()),
            owners: Owners::memory(CELL_CAPACITY),
        }
    }
}

type MemoryRegistry = HashMap<usize, Weak<MemoryCellTable>>;

static MEMORY_TABLES: LazyLock<Mutex<MemoryRegistry>> =
    LazyLock::new(|| Mutex::new(HashMap::new()));

fn memory_table(fleet: &Arc<Fleet>) -> Arc<MemoryCellTable> {
    let fleet_identity = Arc::as_ptr(fleet) as usize;
    let mut tables = lock_unpoisoned(&MEMORY_TABLES);
    tables.retain(|_, table| table.strong_count() > 0);
    if let Some(table) = tables.get(&fleet_identity).and_then(Weak::upgrade) {
        return table;
    }
    let table = Arc::new(MemoryCellTable::new(Arc::clone(fleet)));
    tables.insert(fleet_identity, Arc::downgrade(&table));
    table
}

#[cfg(unix)]
struct ShmCellTable {
    shared: Arc<lifetime::Shared>,
    structural_lock: Mutex<()>,
}

#[cfg(unix)]
impl ShmCellTable {
    fn open_or_create(
        name: &str,
        owners: usize,
    ) -> Result<Self> {
        use std::{io, ptr};

        let (region, _initialization_lock) =
            ShmRegion::open_or_create_locked(name, shm_segment_size(owners)?)?;
        if region.created() {
            unsafe {
                ptr::write(
                    region.as_ptr().cast::<CellStateHeader>(),
                    CellStateHeader::new(owners),
                );
                let slots = region.as_ptr().add(size_of::<CellStateHeader>());
                ptr::write_bytes(
                    slots,
                    0,
                    shm_segment_size(owners)? - size_of::<CellStateHeader>(),
                );
            }
        } else {
            let header = unsafe { &*region.as_ptr().cast::<CellStateHeader>() };
            if header.magic != STATE_MAGIC {
                return Err(Error::Io(io::Error::new(
                    io::ErrorKind::InvalidData,
                    format!(
                        "SHM segment {name} has wrong magic 0x{:08X} (expected 0x{STATE_MAGIC:08X})",
                        header.magic
                    ),
                )));
            }
            if header.version != STATE_VERSION
                || usize::from(header.header_size)
                    != size_of::<CellStateHeader>()
                || header.capacity as usize != CELL_CAPACITY
                || header.slot_size as usize != size_of::<CellSlot>()
                || header.owners as usize != owners
            {
                return Err(Error::Io(io::Error::new(
                    io::ErrorKind::InvalidData,
                    format!(
                        "SHM segment {name} has an incompatible cell-state layout"
                    ),
                )));
            }
        }
        let shared = lifetime::Shared::claim_locked(
            region,
            name,
            owners,
            CELL_CAPACITY,
            owner_offset(),
        )?;
        Ok(Self { shared, structural_lock: Mutex::new(()) })
    }

    fn header(&self) -> &CellStateHeader {
        unsafe { &*self.shared.region.as_ptr().cast::<CellStateHeader>() }
    }

    fn slots(&self) -> &[CellSlot] {
        unsafe {
            std::slice::from_raw_parts(
                self.shared
                    .region
                    .as_ptr()
                    .add(size_of::<CellStateHeader>())
                    .cast::<CellSlot>(),
                CELL_CAPACITY,
            )
        }
    }

    fn with_structure<T>(
        &self,
        operation: impl FnOnce(&[CellSlot], &AtomicU32, &Owners) -> Result<T>,
    ) -> Result<T> {
        let owners = Owners::Shared(self.shared.clone());
        owners.check_process()?;
        let _local = lock_unpoisoned(&self.structural_lock);
        let _process = self.shared.region.lock_exclusive()?;
        operation(self.slots(), &self.header().hint, &owners)
    }
}

#[cfg(unix)]
fn owner_offset() -> usize {
    size_of::<CellStateHeader>() + CELL_CAPACITY * size_of::<CellSlot>()
}

#[cfg(unix)]
fn shm_segment_size(owners: usize) -> Result<usize> {
    lifetime::segment_size(owner_offset(), CELL_CAPACITY, owners)
}

#[cfg(unix)]
fn shm_table(
    name: &str,
    owners: usize,
) -> Result<Arc<ShmCellTable>> {
    static TABLES: LazyLock<Mutex<HashMap<String, Weak<ShmCellTable>>>> =
        LazyLock::new(|| Mutex::new(HashMap::new()));
    let mut tables = lock_unpoisoned(&TABLES);
    tables.retain(|_, table| table.strong_count() != 0);
    if let Some(table) = tables.get(name).and_then(Weak::upgrade)
        && table.shared.is_current(owners)
    {
        return Ok(table);
    }
    let table = Arc::new(ShmCellTable::open_or_create(name, owners)?);
    tables.insert(name.to_owned(), Arc::downgrade(&table));
    Ok(table)
}

const _: () = assert!(CELL_CAPACITY.is_power_of_two());
const _: () = assert!(CELL_CAPACITY <= u32::MAX as usize);
const _: () = assert!(size_of::<CellStateHeader>() == 64);
const _: () = assert!(size_of::<CellSlot>() == 64);

#[cfg(test)]
mod registry_tests {
    use std::sync::Arc;

    use orbit_core::Fleet;

    use super::Cells;

    /// The in-memory registry keys tables by the fleet's address. The
    /// table therefore keeps the fleet alive: otherwise a dropped fleet's
    /// address could be reused by a new one, which would then find and
    /// share the old table.
    #[test]
    fn a_memory_table_keeps_its_fleet_alive() {
        let fleet = Arc::new(Fleet::join("cell-registry-test", 1).unwrap());
        let cells = Cells::new(Arc::clone(&fleet)).unwrap();
        assert!(Arc::strong_count(&fleet) > 1);
        drop(cells);
        assert_eq!(Arc::strong_count(&fleet), 1);
    }
}

#[cfg(test)]
mod lifetime_tests {
    use super::*;

    #[test]
    fn a_retired_cell_keeps_old_waiter_accounting_until_its_handles_drop() {
        let cells =
            Cells::new(Arc::new(Fleet::join("example", 1).unwrap())).unwrap();
        let old = cells.allocate(7_u64).unwrap();
        let id = old.id();
        let held: Vec<_> = (1..CELL_CAPACITY)
            .map(|_| cells.allocate(0_u64).unwrap())
            .collect();
        let slot = old.slot().unwrap();
        slot.wake.waiters.fetch_add(1, Ordering::SeqCst);
        cells.release(id).unwrap();
        assert!(matches!(cells.allocate(42_u64), Err(Error::Full { .. })));
        assert_eq!(slot.wake.waiters.fetch_sub(1, Ordering::SeqCst), 1);
        drop(old);
        let fresh = cells.allocate(42_u64).unwrap();
        assert_eq!(fresh.id().index(), id.index());
        assert_eq!(
            fresh.slot().unwrap().wake.waiters.load(Ordering::Acquire),
            0
        );
        assert_eq!(fresh.load().unwrap(), 42);
        drop(held);
    }

    #[test]
    fn exhausted_cell_generations_are_never_reissued() {
        let cells =
            Cells::new(Arc::new(Fleet::join("example", 1).unwrap())).unwrap();
        cells
            .with_structure(|slots, _, _| {
                for slot in slots {
                    slot.generation.store(u32::MAX, Ordering::Relaxed);
                }
                Ok(())
            })
            .unwrap();
        assert!(matches!(cells.allocate(1_u64), Err(Error::Full { .. })));
    }
}

#[cfg(test)]
mod wait_tests {
    use std::sync::Arc;
    use std::time::Duration;

    use orbit_core::Fleet;

    use super::{Cells, Error};

    fn cells() -> Cells {
        Cells::new(Arc::new(Fleet::join("cell-wait-test", 1).unwrap())).unwrap()
    }

    /// A waiter parks on the change count and a write from another thread
    /// wakes it; it then reads the value the write left. Ten writes while
    /// parked are one wake.
    #[test]
    fn a_write_wakes_a_waiter_once() {
        let cells = cells();
        let cell = cells.allocate(0_i64).unwrap();
        let since = cell.version().unwrap();
        let watcher = cell.clone();
        let waiter = std::thread::spawn(move || watcher.wait_changed(since));

        std::thread::sleep(Duration::from_millis(30));
        for _ in 0..10 {
            cell.fetch_add(1).unwrap();
        }
        let now = waiter.join().unwrap().unwrap();
        assert!(now > since);
        assert_eq!(cell.load().unwrap(), 10);
        // Nothing new since: a wait would park; the count says so.
        assert_eq!(cell.version().unwrap(), cell.version().unwrap());
    }

    /// Releasing the cell is a change too: a parked waiter comes back Stale
    /// instead of sleeping on a slot somebody else may take.
    #[test]
    fn release_wakes_a_waiter_stale() {
        let cells = cells();
        let cell = cells.allocate(7_i64).unwrap();
        let since = cell.version().unwrap();
        let watcher = cell.clone();
        let waiter = std::thread::spawn(move || watcher.wait_changed(since));

        std::thread::sleep(Duration::from_millis(30));
        cell.release().unwrap();
        assert!(matches!(waiter.join().unwrap(), Err(Error::Stale(_))));
    }
}