#[cfg(unix)]
use std::io;
use std::sync::atomic::{AtomicU8, AtomicU32, AtomicU64, Ordering};
use orbit_core::process::ProcessOwner;
#[path = "inbox/claims.rs"]
pub(crate) mod claims;
use claims::{CANCELLED, COMMITTED};
use super::{Error, Result};
pub(crate) const INBOX_MAGIC: u32 = 0x4f54_5849;
pub(crate) const INBOX_VERSION: u32 = 13;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct GrowthPolicy {
pub min: u16,
pub max: u16,
pub step: u16,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Reclaimed {
pub lane: usize,
pub incarnation: u64,
pub pid: u32,
}
pub const LANE_NAME_MAX: usize = 47;
pub const LANE_ROLE_MAX: usize = 8;
fn accept_role(
role: &[u8; LANE_ROLE_MAX],
role_len: &AtomicU8,
accept: impl FnOnce(&str) -> bool,
) -> bool {
let len = role_len.load(Ordering::Acquire) as usize;
if len == 0 || len > LANE_ROLE_MAX {
return false;
}
let mut copy = [0u8; LANE_ROLE_MAX];
copy[..len].copy_from_slice(unsafe {
std::slice::from_raw_parts(role.as_ptr(), len)
});
std::str::from_utf8(©[..len]).is_ok_and(accept)
}
#[repr(C, align(64))]
pub(crate) struct LaneHeader {
pub(crate) reserve: AtomicU64,
pub(crate) read: AtomicU64,
pub(crate) consumers: AtomicU32,
pub(crate) changes: AtomicU32,
pub(crate) incarnation: AtomicU64,
pub(crate) waiters: AtomicU32,
pub(crate) reactor: AtomicU32,
pub(crate) epoch: AtomicU64,
initialized: AtomicU32,
paired_bodies_epoch: AtomicU32,
pub(crate) service: [u8; LANE_NAME_MAX],
pub(crate) service_len: AtomicU8,
pub(crate) pid: AtomicU32,
pub(crate) role: [u8; LANE_ROLE_MAX],
pub(crate) role_len: AtomicU8,
_identity_padding: [u8; 3],
}
const _: () = assert!(size_of::<LaneHeader>() == 128);
#[repr(C, align(64))]
pub(crate) struct WatcherHeader {
pub(crate) changes: AtomicU32,
pub(crate) reactor: AtomicU32,
pub(crate) pid: AtomicU32,
pub(crate) role: [u8; LANE_ROLE_MAX],
pub(crate) role_len: AtomicU8,
_padding: [u8; 43],
}
const _: () = assert!(size_of::<WatcherHeader>() == 64);
pub(crate) const INTEREST_WORDS: usize = 7;
#[cfg_attr(not(feature = "events"), allow(dead_code))]
pub(crate) const INTEREST_BITS: usize = INTEREST_WORDS * 64;
#[repr(C, align(64))]
pub(crate) struct InterestEntry {
pub(crate) seq: AtomicU32,
_padding: [u8; 4],
pub(crate) bits: [AtomicU64; INTEREST_WORDS],
}
const _: () = assert!(size_of::<InterestEntry>() == 64);
#[repr(C, align(64))]
pub(crate) struct SlotHeader {
pub(crate) seq: AtomicU64,
pub(crate) claim: AtomicU64,
pub(crate) work_epoch: AtomicU64,
pub(crate) work_claim: AtomicU64,
pub(crate) len: AtomicU32,
pub(crate) work_slot: AtomicU32,
_padding: [u8; 24],
}
const _: () = assert!(size_of::<SlotHeader>() == 64);
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum Admission {
Accepted,
Full,
NoConsumer,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct InboxGeometry {
pub capacity: usize,
pub payload_capacity: usize,
}
impl InboxGeometry {
pub const fn new(
capacity: usize,
payload_capacity: usize,
) -> Self {
Self { capacity, payload_capacity }
}
pub const fn lane_bytes(&self) -> usize {
size_of::<LaneHeader>() + self.capacity * self.slot_bytes()
}
const fn slot_bytes(&self) -> usize {
(size_of::<SlotHeader>() + self.payload_capacity + 63) & !63
}
pub const fn segment_bytes(
&self,
fleet_capacity: usize,
) -> usize {
size_of::<TableHeader>()
+ fleet_capacity
* (self.lane_bytes()
+ size_of::<WatcherHeader>()
+ 2 * size_of::<InterestEntry>())
+ claims::registry_bytes(fleet_capacity, self.capacity)
+ crate::admission::bytes(fleet_capacity, self.capacity)
}
pub(crate) fn validate(&self) -> Result<()> {
if self.capacity == 0 || !self.capacity.is_power_of_two() {
return Err(Error::Malformed(
"inbox capacity must be a power of two",
));
}
if self.payload_capacity == 0 {
return Err(Error::Malformed(
"inbox payload capacity must not be zero",
));
}
if self.capacity > u32::MAX as usize
|| self.payload_capacity > u32::MAX as usize
{
return Err(Error::Malformed(
"inbox geometry exceeds its wire fields",
));
}
Ok(())
}
}
const _: () = assert!(size_of::<TableHeader>() == 128);
#[repr(C, align(64))]
pub(crate) struct TableHeader {
pub(crate) magic: AtomicU32,
pub(crate) version: AtomicU32,
pub(crate) fleet_capacity: AtomicU32,
pub(crate) capacity: AtomicU32,
pub(crate) payload_capacity: AtomicU32,
pub(crate) directory: AtomicU64,
pub(crate) policy: AtomicU32,
pub(crate) pressure: AtomicU32,
pub(crate) growth_min: AtomicU32,
pub(crate) growth_max: AtomicU32,
pub(crate) growth_step: AtomicU32,
_padding: [u8; 16],
}
pub struct Inbox {
pub(crate) base: *mut u8,
pub(crate) lanes: usize,
pub(crate) geometry: InboxGeometry,
producer: Option<claims::Producer>,
#[cfg(unix)]
bell: Option<crate::bell::Bell>,
}
unsafe impl Send for Inbox {}
unsafe impl Sync for Inbox {}
impl Inbox {
pub unsafe fn initialize(
base: *mut u8,
fleet_capacity: usize,
geometry: InboxGeometry,
) -> Result<Self> {
geometry.validate()?;
if fleet_capacity == 0 || fleet_capacity > u16::MAX as usize {
return Err(Error::Malformed(
"inbox fleet capacity must fit a nonzero u16",
));
}
let inbox = Self {
base,
lanes: fleet_capacity,
geometry,
producer: None,
#[cfg(unix)]
bell: None,
};
let header = inbox.table_header();
header.fleet_capacity.store(fleet_capacity as u32, Ordering::Relaxed);
header.capacity.store(geometry.capacity as u32, Ordering::Relaxed);
header
.payload_capacity
.store(geometry.payload_capacity as u32, Ordering::Relaxed);
header.version.store(INBOX_VERSION, Ordering::Relaxed);
header.magic.store(INBOX_MAGIC, Ordering::Release);
Ok(inbox)
}
pub(crate) unsafe fn persisted_shape(
base: *const u8
) -> Result<(usize, InboxGeometry)> {
let header = unsafe { &*base.cast::<TableHeader>() };
if header.magic.load(Ordering::Acquire) != INBOX_MAGIC {
return Err(Error::Malformed(
"inbox segment is not an inbox table",
));
}
if header.version.load(Ordering::Relaxed) != INBOX_VERSION {
return Err(Error::Malformed(
"inbox segment was written by another layout version",
));
}
let geometry = InboxGeometry::new(
header.capacity.load(Ordering::Relaxed) as usize,
header.payload_capacity.load(Ordering::Relaxed) as usize,
);
geometry.validate()?;
Ok((header.fleet_capacity.load(Ordering::Relaxed) as usize, geometry))
}
pub unsafe fn attach(
base: *mut u8,
fleet_capacity: usize,
geometry: InboxGeometry,
) -> Result<Self> {
geometry.validate()?;
if fleet_capacity == 0 || fleet_capacity > u16::MAX as usize {
return Err(Error::Malformed(
"inbox fleet capacity must fit a nonzero u16",
));
}
let inbox = Self {
base,
lanes: fleet_capacity,
geometry,
producer: None,
#[cfg(unix)]
bell: None,
};
let header = inbox.table_header();
if header.magic.load(Ordering::Acquire) != INBOX_MAGIC {
return Err(Error::Malformed(
"inbox segment is not an inbox table",
));
}
if header.version.load(Ordering::Relaxed) != INBOX_VERSION {
return Err(Error::Malformed(
"inbox segment was written by another layout version",
));
}
if header.fleet_capacity.load(Ordering::Relaxed) as usize
!= fleet_capacity
|| header.capacity.load(Ordering::Relaxed) as usize
!= geometry.capacity
|| header.payload_capacity.load(Ordering::Relaxed) as usize
!= geometry.payload_capacity
{
return Err(Error::Malformed(
"inbox segment geometry differs from this build",
));
}
Ok(inbox)
}
pub fn geometry(&self) -> InboxGeometry {
self.geometry
}
pub fn lanes(&self) -> usize {
self.lanes
}
pub fn lane_epoch(
&self,
lane: usize,
) -> Result<u64> {
self.check_lane(lane)?;
Ok(self.lane_header(lane).epoch.load(Ordering::Acquire))
}
pub fn advertise_paired_bodies(
&self,
lane: usize,
) -> Result<()> {
let epoch = u32::try_from(self.lane_epoch(lane)?).map_err(|_| {
Error::Malformed("paired body advertisement epoch exhausted")
})?;
let previous = self
.lane_header(lane)
.paired_bodies_epoch
.swap(epoch, Ordering::AcqRel);
if previous != epoch {
self.table_header().directory.fetch_add(1, Ordering::AcqRel);
self.announce_directory();
}
Ok(())
}
pub fn paired_bodies_epoch(
&self,
lane: usize,
) -> Option<u64> {
let epoch = self.lane_epoch(lane).ok()?;
let stamp =
self.lane_header(lane).paired_bodies_epoch.load(Ordering::Acquire);
(stamp != 0 && u64::from(stamp) == epoch).then_some(epoch)
}
pub fn claim_lane(
&self,
lane_index: usize,
service: &str,
role: &str,
incarnation: u64,
) -> Result<()> {
self.claim_lane_with_capacity(
lane_index,
service,
role,
incarnation,
None,
)
}
pub fn claim_lane_with_capacity(
&self,
lane_index: usize,
service: &str,
role: &str,
incarnation: u64,
capacity: Option<usize>,
) -> Result<()> {
self.check_lane(lane_index)?;
check_identity(service, role)?;
self.validate_work_capacity(capacity)?;
let pid = ProcessOwner::current()?.pid();
self.reset_lane(lane_index)?;
self.configure_capacity(lane_index, capacity)?;
let lane = self.lane_header(lane_index);
lane.consumers.store(1, Ordering::Release);
self.publish_identity(lane, service, role, incarnation, pid);
self.wake(lane_index, lane);
Ok(())
}
pub fn join<H>(
&self,
service: &str,
role: &str,
incarnation: u64,
hold: impl FnMut(usize) -> Result<Option<H>>,
) -> Result<(usize, H)> {
self.join_with_capacity(service, role, incarnation, None, hold)
}
pub fn join_with_capacity<H>(
&self,
service: &str,
role: &str,
incarnation: u64,
capacity: Option<usize>,
mut hold: impl FnMut(usize) -> Result<Option<H>>,
) -> Result<(usize, H)> {
check_identity(service, role)?;
self.validate_work_capacity(capacity)?;
self.refuse_other_role(service, role, None)?;
let pid = ProcessOwner::current()?.pid();
for lane_index in 0..self.lanes {
let lane = self.lane_header(lane_index);
if lane.consumers.load(Ordering::Acquire) != 0 {
continue;
}
let Some(held) = hold(lane_index)? else {
continue;
};
if lane
.consumers
.compare_exchange(0, 1, Ordering::AcqRel, Ordering::Acquire)
.is_err()
{
continue;
}
self.reset_lane(lane_index)?;
self.configure_capacity(lane_index, capacity)?;
self.publish_identity(lane, service, role, incarnation, pid);
if let Err(conflict) =
self.refuse_other_role(service, role, Some(lane_index))
{
let _ = self.release_lane(lane_index);
return Err(conflict);
}
self.wake(lane_index, lane);
return Ok((lane_index, held));
}
Err(Error::NoLane)
}
pub fn reclaim(
&self,
lane_index: usize,
) -> Result<Option<Reclaimed>> {
self.check_lane(lane_index)?;
let lane = self.lane_header(lane_index);
if lane.consumers.load(Ordering::Acquire) == 0 {
return Ok(None);
}
let reclaimed = Reclaimed {
lane: lane_index,
incarnation: lane.incarnation.load(Ordering::Acquire),
pid: lane.pid.load(Ordering::Acquire),
};
self.release_lane(lane_index)?;
Ok(Some(reclaimed))
}
fn refuse_other_role(
&self,
service: &str,
role: &str,
own: Option<usize>,
) -> Result<()> {
for lane_index in self.serving(service) {
if Some(lane_index) == own {
continue;
}
let lane = self.lane_header(lane_index);
if lane.consumers.load(Ordering::Acquire) == 0 {
continue;
}
let held_by = self.role(lane_index)?.unwrap_or_default();
if held_by != role {
return Err(Error::RoleConflict {
name: service.to_owned(),
role: role.to_owned(),
held_by: if held_by.is_empty() {
"an unnamed role".into()
} else {
held_by
},
lane: lane_index,
pid: lane.pid.load(Ordering::Acquire),
});
}
}
Ok(())
}
pub fn park(
&self,
lane_index: usize,
expected: u32,
) -> Result<()> {
self.check_lane(lane_index)?;
let lane = self.lane_header(lane_index);
lane.waiters.fetch_add(1, Ordering::AcqRel);
let waited = orbit_core::sync::wait_word(&lane.changes, expected);
lane.waiters.fetch_sub(1, Ordering::AcqRel);
waited.map_err(Error::Io)
}
pub fn nudge(
&self,
lane_index: usize,
) -> Result<()> {
self.check_lane(lane_index)?;
self.wake(lane_index, self.lane_header(lane_index));
Ok(())
}
pub fn service(
&self,
lane_index: usize,
) -> Result<Option<String>> {
self.check_lane(lane_index)?;
let lane = self.lane_header(lane_index);
let len = lane.service_len.load(Ordering::Acquire) as usize;
if len == 0 || len > LANE_NAME_MAX {
return Ok(None);
}
let bytes =
unsafe { std::slice::from_raw_parts(lane.service.as_ptr(), len) };
Ok(String::from_utf8(bytes.to_vec()).ok())
}
pub fn owner(
&self,
lane_index: usize,
) -> Result<Option<u32>> {
self.check_lane(lane_index)?;
if self.consumers(lane_index)? == 0 {
return Ok(None);
}
Ok(match self.lane_header(lane_index).pid.load(Ordering::Acquire) {
0 => None,
pid => Some(pid),
})
}
pub fn last_owner(
&self,
lane_index: usize,
) -> Result<Option<(u32, u64)>> {
self.check_lane(lane_index)?;
let lane = self.lane_header(lane_index);
let pid = lane.pid.load(Ordering::Acquire);
Ok((pid != 0).then(|| (pid, lane.incarnation.load(Ordering::Acquire))))
}
pub fn role_accepted(
&self,
lane_index: usize,
accept: impl FnOnce(&str) -> bool,
) -> bool {
if self.check_lane(lane_index).is_err() {
return false;
}
let lane = self.lane_header(lane_index);
accept_role(&lane.role, &lane.role_len, accept)
}
pub fn watcher_role_accepted(
&self,
slot: usize,
accept: impl FnOnce(&str) -> bool,
) -> bool {
if self.check_lane(slot).is_err() {
return false;
}
let watcher = self.watcher_header(slot);
watcher.pid.load(Ordering::Acquire) != 0
&& accept_role(&watcher.role, &watcher.role_len, accept)
}
pub fn role(
&self,
lane_index: usize,
) -> Result<Option<String>> {
self.check_lane(lane_index)?;
let lane = self.lane_header(lane_index);
let len = lane.role_len.load(Ordering::Acquire) as usize;
if len == 0 || len > LANE_ROLE_MAX {
return Ok(None);
}
let bytes =
unsafe { std::slice::from_raw_parts(lane.role.as_ptr(), len) };
Ok(String::from_utf8(bytes.to_vec()).ok())
}
fn serves(
&self,
lane_index: usize,
service: &[u8],
) -> bool {
let lane = self.lane_header(lane_index);
let len = lane.service_len.load(Ordering::Acquire) as usize;
if len != service.len() || len > LANE_NAME_MAX {
return false;
}
let name =
unsafe { std::slice::from_raw_parts(lane.service.as_ptr(), len) };
name == service
}
pub fn targets(
&self,
service: &str,
) -> Vec<(usize, u64)> {
self.ready(&self.serving(service))
}
pub fn directory(&self) -> u64 {
self.table_header().directory.load(Ordering::Acquire)
}
pub fn set_growth_policy(
&self,
policy: Option<GrowthPolicy>,
) {
let header = self.table_header();
match policy {
Some(policy) => {
header.growth_min.store(policy.min.into(), Ordering::Relaxed);
header.growth_max.store(policy.max.into(), Ordering::Relaxed);
header.growth_step.store(policy.step.into(), Ordering::Relaxed);
header.policy.store(1, Ordering::Release);
}
None => header.policy.store(0, Ordering::Release),
}
}
pub fn growth_policy(&self) -> Option<GrowthPolicy> {
let header = self.table_header();
if header.policy.load(Ordering::Acquire) == 0 {
return None;
}
let bound = |word: &AtomicU32| {
u16::try_from(word.load(Ordering::Relaxed)).unwrap_or(u16::MAX)
};
Some(GrowthPolicy {
min: bound(&header.growth_min),
max: bound(&header.growth_max),
step: bound(&header.growth_step),
})
}
pub fn report_pressure(&self) -> bool {
let header = self.table_header();
if header.policy.load(Ordering::Acquire) == 0
|| header.pressure.load(Ordering::Acquire) != 0
|| header
.pressure
.compare_exchange(0, 1, Ordering::AcqRel, Ordering::Acquire)
.is_err()
{
return false;
}
for slot in 0..self.lanes {
self.wake_watcher(slot);
}
true
}
pub fn pressure_pending(&self) -> bool {
self.table_header().pressure.load(Ordering::Acquire) != 0
}
pub fn take_pressure(&self) -> bool {
self.table_header().pressure.swap(0, Ordering::AcqRel) != 0
}
pub fn serving(
&self,
service: &str,
) -> Vec<usize> {
let wanted = service.as_bytes();
(0..self.lanes).filter(|lane| self.serves(*lane, wanted)).collect()
}
pub fn ready(
&self,
lanes: &[usize],
) -> Vec<(usize, u64)> {
let mut found: Vec<(usize, u64)> = lanes
.iter()
.copied()
.filter(|lane| {
matches!(self.admits(*lane), Ok(Admission::Accepted))
})
.filter_map(|lane| self.depth(lane).ok().map(|depth| (lane, depth)))
.collect();
found.sort_by_key(|(_, depth)| *depth);
found
}
pub fn idlest(
&self,
lanes: &[usize],
) -> Option<(usize, u64)> {
lanes
.iter()
.copied()
.filter(|lane| {
matches!(self.admits(*lane), Ok(Admission::Accepted))
})
.filter_map(|lane| self.depth(lane).ok().map(|depth| (lane, depth)))
.min_by_key(|(_, depth)| *depth)
}
pub fn release_lane(
&self,
lane_index: usize,
) -> Result<()> {
self.check_lane(lane_index)?;
let lane = self.lane_header(lane_index);
lane.consumers.store(0, Ordering::Release);
lane.service_len.store(0, Ordering::Release);
lane.role_len.store(0, Ordering::Release);
self.clear_interest(lane_index);
std::sync::atomic::fence(Ordering::SeqCst);
self.capacity_changed(lane_index);
self.table_header().directory.fetch_add(1, Ordering::AcqRel);
self.wake(lane_index, lane);
self.announce_directory();
Ok(())
}
pub fn admits(
&self,
lane_index: usize,
) -> Result<Admission> {
self.check_lane(lane_index)?;
let lane = self.lane_header(lane_index);
if lane.consumers.load(Ordering::Acquire) == 0 {
return Ok(Admission::NoConsumer);
}
let read = lane.read.load(Ordering::Acquire);
let reserve = lane.reserve.load(Ordering::Acquire);
if reserve.wrapping_sub(read) >= self.geometry.capacity as u64 {
return Ok(Admission::Full);
}
Ok(Admission::Accepted)
}
pub fn depth(
&self,
lane_index: usize,
) -> Result<u64> {
self.check_lane(lane_index)?;
let lane = self.lane_header(lane_index);
let read = lane.read.load(Ordering::Acquire);
let reserve = lane.reserve.load(Ordering::Acquire);
Ok(reserve.wrapping_sub(read))
}
pub fn consumers(
&self,
lane_index: usize,
) -> Result<u32> {
self.check_lane(lane_index)?;
Ok(self.lane_header(lane_index).consumers.load(Ordering::Acquire))
}
pub fn incarnation(
&self,
lane_index: usize,
) -> Result<u64> {
self.check_lane(lane_index)?;
Ok(self.lane_header(lane_index).incarnation.load(Ordering::Acquire))
}
pub fn write(
&self,
lane_index: usize,
payload: &[u8],
) -> Result<u64> {
self.write_vectored(lane_index, &[payload])
}
pub fn write_vectored(
&self,
lane_index: usize,
parts: &[&[u8]],
) -> Result<u64> {
self.write_frame(lane_index, parts, None, None, false)
}
pub fn write_vectored_at_epoch(
&self,
lane: usize,
epoch: u64,
parts: &[&[u8]],
) -> Result<u64> {
self.write_frame(lane, parts, None, Some(epoch), false)
}
pub fn write_reply_at_epoch(
&self,
lane: usize,
epoch: u64,
parts: &[&[u8]],
) -> Result<u64> {
self.write_frame(lane, parts, None, Some(epoch), true)
}
pub(crate) fn write_work(
&self,
ticket: crate::WorkTicket,
parts: &[&[u8]],
) -> Result<u64> {
self.check_work(ticket)?;
self.write_frame(
usize::from(ticket.lane),
parts,
Some(ticket),
Some(ticket.epoch),
false,
)
}
fn write_frame(
&self,
lane_index: usize,
parts: &[&[u8]],
work: Option<crate::WorkTicket>,
epoch: Option<u64>,
reply: bool,
) -> Result<u64> {
self.check_lane(lane_index)?;
if reply
&& (self.lane_epoch(lane_index)? != epoch.unwrap_or_default()
|| self.consumers(lane_index)? == 0)
{
return Err(Error::NoConsumer);
}
let len = parts
.iter()
.try_fold(0usize, |len, part| len.checked_add(part.len()))
.ok_or(Error::FrameTooLarge)?;
if len > self.geometry.payload_capacity {
return Err(Error::FrameTooLarge);
}
let mut reservation = match self.reserve_frame(lane_index) {
Err(Error::InboxContended) if !reply => {
return Err(Error::InboxFull);
}
result => result?,
};
if epoch.is_some_and(|epoch| epoch != reservation.epoch)
|| (reply && self.consumers(lane_index)? == 0)
{
return Err(Error::NoConsumer);
}
let position = reservation.position;
let slot = self.slot_header(lane_index, position);
let mut at = self.payload_ptr(lane_index, position);
for part in parts {
unsafe {
std::ptr::copy_nonoverlapping(part.as_ptr(), at, part.len());
at = at.add(part.len());
}
}
#[cfg(test)]
claims::checkpoint("payload");
if let Some(ticket) = work {
slot.work_epoch.store(ticket.epoch, Ordering::Relaxed);
slot.work_slot.store(ticket.slot, Ordering::Relaxed);
slot.work_claim.store(ticket.claim, Ordering::Relaxed);
} else {
slot.work_claim.store(0, Ordering::Relaxed);
}
slot.len.store(len as u32, Ordering::Relaxed);
slot.claim.store(reservation.claim(COMMITTED), Ordering::Release);
#[cfg(test)]
claims::checkpoint("committed");
reservation.committed = true;
self.wake(lane_index, self.lane_header(lane_index));
Ok(position)
}
pub fn reserved_through(
&self,
lane: usize,
) -> Result<u64> {
self.check_lane(lane)?;
Ok(self.lane_header(lane).reserve.load(Ordering::Acquire))
}
pub fn read_position(
&self,
lane: usize,
) -> Result<u64> {
self.check_lane(lane)?;
Ok(self.lane_header(lane).read.load(Ordering::Acquire))
}
pub fn read(
&self,
lane_index: usize,
out: &mut Vec<u8>,
) -> Result<bool> {
match self.read_frame(lane_index, out)? {
None => Ok(false),
Some(None) => Ok(true),
Some(Some(ticket)) => {
self.cancel_work(ticket);
Err(Error::Malformed(
"work frames require an admission-aware receiver",
))
}
}
}
pub fn read_frame(
&self,
lane_index: usize,
out: &mut Vec<u8>,
) -> Result<Option<Option<crate::WorkTicket>>> {
self.read_frame_before(lane_index, out, None)
}
pub(crate) fn read_frame_before(
&self,
lane_index: usize,
out: &mut Vec<u8>,
boundary: Option<u64>,
) -> Result<Option<Option<crate::WorkTicket>>> {
self.check_lane(lane_index)?;
let lane = self.lane_header(lane_index);
for _ in 0..self.geometry.capacity {
let position = lane.read.load(Ordering::Relaxed);
if boundary.is_some_and(|boundary| position >= boundary)
|| position >= lane.reserve.load(Ordering::Acquire)
{
return Ok(None);
}
let slot = self.slot_header(lane_index, position);
let claim = slot.claim.load(Ordering::Acquire);
if slot.seq.load(Ordering::Acquire) != position
|| !matches!(claim & 3, COMMITTED | CANCELLED)
{
return Ok(None);
}
let intent = self.intent_for_claim(claim)?;
if intent.position != position || intent.lane != lane_index {
return Err(Error::Malformed(
"committed inbox intent has the wrong turn",
));
}
let deliver = claim & 3 == COMMITTED
&& intent.incarnation == lane.epoch.load(Ordering::Acquire);
let work_claim = slot.work_claim.load(Ordering::Relaxed);
let work = (work_claim != 0).then(|| crate::WorkTicket {
lane: lane_index as u16,
epoch: slot.work_epoch.load(Ordering::Relaxed),
slot: slot.work_slot.load(Ordering::Relaxed),
claim: work_claim,
});
if deliver {
let len = slot.len.load(Ordering::Relaxed) as usize;
if len > self.geometry.payload_capacity {
return Err(Error::Malformed(
"committed inbox length is invalid",
));
}
out.clear();
out.reserve(len);
unsafe {
std::ptr::copy_nonoverlapping(
self.payload_ptr(lane_index, position),
out.spare_capacity_mut().as_mut_ptr().cast::<u8>(),
len,
);
out.set_len(len);
}
}
self.consume_frame(lane_index, position, claim)?;
if deliver {
return Ok(Some(work));
}
}
Ok(None)
}
pub fn wait_word(
&self,
lane_index: usize,
) -> Result<(&AtomicU32, u32)> {
self.check_lane(lane_index)?;
let lane = self.lane_header(lane_index);
Ok((&lane.changes, lane.changes.load(Ordering::Acquire)))
}
pub(crate) fn retire_lane(
&self,
lane_index: usize,
) -> Result<()> {
self.check_lane(lane_index)?;
if self.consumers(lane_index)? != 0 {
return Err(Error::Malformed("retiring an advertised inbox lane"));
}
self.reset_lane(lane_index)
}
fn reset_lane(
&self,
lane_index: usize,
) -> Result<()> {
let lane = self.lane_header(lane_index);
let next_epoch = lane
.epoch
.load(Ordering::Acquire)
.checked_add(1)
.ok_or(Error::Malformed("inbox lane generation exhausted"))?;
if lane.initialized.load(Ordering::Acquire) == 0 {
for position in 0..self.geometry.capacity {
let slot = self.slot_header(lane_index, position as u64);
slot.seq.store(position as u64, Ordering::Release);
}
lane.initialized.store(1, Ordering::Release);
} else {
for _ in 0..self.geometry.capacity {
let position = lane.read.load(Ordering::Relaxed);
if position >= lane.reserve.load(Ordering::Acquire) {
break;
}
if self
.slot_header(lane_index, position)
.seq
.load(Ordering::Acquire)
> position
{
let claim = self
.slot_header(lane_index, position)
.claim
.load(Ordering::Acquire);
if matches!(claim & 3, COMMITTED | CANCELLED)
&& self.intent_for_claim(claim)?.position == position
{
self.consume_frame(lane_index, position, claim)?;
} else {
lane.read.store(position + 1, Ordering::Release);
}
continue;
}
let claim = self
.slot_header(lane_index, position)
.claim
.load(Ordering::Acquire);
if !matches!(claim & 3, COMMITTED | CANCELLED) {
break;
}
self.consume_frame(lane_index, position, claim)?;
}
}
lane.service_len.store(0, Ordering::Relaxed);
lane.epoch.store(next_epoch, Ordering::Release);
self.advertise_inbox_space(lane_index, next_epoch);
std::sync::atomic::fence(Ordering::SeqCst);
self.inbox_space_changed(lane_index);
lane.role_len.store(0, Ordering::Relaxed);
lane.waiters.store(0, Ordering::Relaxed);
lane.reactor.store(0, Ordering::Relaxed);
self.clear_interest(lane_index);
self.recover_work(lane_index);
self.configure_capacity(lane_index, None)?;
Ok(())
}
fn publish_identity(
&self,
lane: &LaneHeader,
service: &str,
role: &str,
incarnation: u64,
pid: u32,
) {
unsafe {
std::ptr::copy_nonoverlapping(
service.as_ptr(),
lane.service.as_ptr().cast_mut(),
service.len(),
);
std::ptr::copy_nonoverlapping(
role.as_ptr(),
lane.role.as_ptr().cast_mut(),
role.len(),
);
}
lane.role_len.store(role.len() as u8, Ordering::Release);
lane.service_len.store(service.len() as u8, Ordering::Release);
lane.incarnation.store(incarnation, Ordering::Release);
lane.pid.store(pid, Ordering::Release);
self.table_header().directory.fetch_add(1, Ordering::AcqRel);
self.announce_directory();
}
fn announce_directory(&self) {
for lane_index in 0..self.lanes {
let lane = self.lane_header(lane_index);
if lane.consumers.load(Ordering::Acquire) != 0 {
self.wake(lane_index, lane);
}
}
for slot in 0..self.lanes {
self.wake_watcher(slot);
}
}
fn wake_watcher(
&self,
slot: usize,
) {
let watcher = self.watcher_header(slot);
watcher.changes.fetch_add(1, Ordering::SeqCst);
#[cfg(unix)]
if let Some(bell) = &self.bell
&& let Err(error) = ring_armed(&watcher.reactor, || {
bell.ring(crate::bell::Doorbell::Watcher(slot))
})
{
tracing::warn!(slot, %error, "Link watcher wake failed");
}
}
pub(crate) fn wake(
&self,
lane_index: usize,
lane: &LaneHeader,
) {
lane.changes.fetch_add(1, Ordering::SeqCst);
if lane.waiters.load(Ordering::SeqCst) > 0 {
let _ = orbit_core::sync::wake_word(&lane.changes);
}
#[cfg(unix)]
if let Some(bell) = &self.bell
&& let Err(error) = ring_armed(&lane.reactor, || {
bell.ring_lane(lane_index, lane.epoch.load(Ordering::Acquire))
})
{
tracing::warn!(lane = lane_index, %error, "Link lane wake failed");
}
}
#[cfg(unix)]
pub(crate) fn ring_adopter(&self) {
if let Some(bell) = &self.bell {
let _ = bell.ring(crate::bell::Doorbell::Adopter);
}
}
#[cfg(unix)]
pub(crate) fn install_bell_with_group(
&mut self,
segment: &str,
group: Option<orbit_core::shm::GroupAccess>,
) -> Result<()> {
self.bell = Some(
crate::bell::Bell::with_group(segment, group, self.lanes)
.map_err(Error::Io)?,
);
Ok(())
}
#[cfg(all(unix, feature = "tokio"))]
pub(crate) fn bind_producer(&self) -> Result<tokio::net::UnixDatagram> {
let index = self
.producer_index()
.ok_or(Error::Malformed("this inbox mapping has no producer"))?;
let bell = self
.bell
.as_ref()
.ok_or(Error::Malformed("the inbox has no reactor bell"))?;
let socket = bell
.bind(crate::bell::Doorbell::Producer(index))
.map_err(Error::Io)?;
tokio::net::UnixDatagram::from_std(socket).map_err(Error::Io)
}
#[cfg(all(unix, feature = "tokio"))]
pub(crate) fn unbind_producer(&self) {
if let (Some(index), Some(bell)) = (self.producer_index(), &self.bell) {
bell.unbind(crate::bell::Doorbell::Producer(index));
}
}
#[cfg(all(unix, feature = "tokio"))]
pub(crate) fn bind_receiver(
&self,
lane_index: usize,
) -> Result<tokio::net::UnixDatagram> {
self.check_lane(lane_index)?;
let bell = self
.bell
.as_ref()
.ok_or(Error::Malformed("the inbox has no reactor bell"))?;
let socket = bell
.bind(crate::bell::Doorbell::Lane(lane_index))
.map_err(Error::Io)?;
tokio::net::UnixDatagram::from_std(socket).map_err(Error::Io)
}
#[cfg(all(unix, feature = "tokio"))]
pub(crate) fn unbind_receiver(
&self,
lane_index: usize,
) {
self.disarm_receiver(lane_index);
if let Some(bell) = &self.bell {
bell.unbind(crate::bell::Doorbell::Lane(lane_index));
}
}
#[cfg(all(unix, feature = "tokio"))]
pub(crate) fn receiver_generation(
&self,
lane_index: usize,
) -> Result<u32> {
self.check_lane(lane_index)?;
Ok(self.lane_header(lane_index).changes.load(Ordering::SeqCst))
}
#[cfg(all(unix, feature = "tokio"))]
pub(crate) fn arm_receiver(
&self,
lane_index: usize,
) -> Result<()> {
self.check_lane(lane_index)?;
self.lane_header(lane_index).reactor.store(1, Ordering::SeqCst);
Ok(())
}
#[cfg(all(unix, feature = "tokio"))]
pub(crate) fn take_receiver_arm(
&self,
lane_index: usize,
) -> bool {
lane_index < self.lanes
&& self.lane_header(lane_index).reactor.swap(0, Ordering::SeqCst)
!= 0
}
#[cfg(all(unix, feature = "tokio"))]
pub(crate) fn disarm_receiver(
&self,
lane_index: usize,
) {
if lane_index < self.lanes {
self.lane_header(lane_index).reactor.store(0, Ordering::SeqCst);
}
}
#[cfg(all(unix, feature = "tokio"))]
pub(crate) fn bind_watcher(
&self,
slot: usize,
role: &str,
) -> Result<tokio::net::UnixDatagram> {
self.check_lane(slot)?;
if role.is_empty() || role.len() > LANE_ROLE_MAX || !role.is_ascii() {
return Err(Error::Malformed("role tag does not fit a watcher"));
}
let bell = self
.bell
.as_ref()
.ok_or(Error::Malformed("the inbox has no reactor bell"))?;
let watcher = self.watcher_header(slot);
let pid = ProcessOwner::current()?.pid();
watcher.reactor.store(0, Ordering::SeqCst);
self.clear_interest(self.lanes + slot);
let socket = bell
.bind(crate::bell::Doorbell::Watcher(slot))
.map_err(Error::Io)?;
unsafe {
std::ptr::copy_nonoverlapping(
role.as_ptr(),
watcher.role.as_ptr().cast_mut(),
role.len(),
);
}
watcher.role_len.store(role.len() as u8, Ordering::Release);
watcher.pid.store(pid, Ordering::Release);
tokio::net::UnixDatagram::from_std(socket).map_err(Error::Io)
}
#[cfg(all(unix, feature = "tokio"))]
pub(crate) fn unbind_watcher(
&self,
slot: usize,
) {
if slot >= self.lanes {
return;
}
let watcher = self.watcher_header(slot);
watcher.reactor.store(0, Ordering::SeqCst);
watcher.pid.store(0, Ordering::Release);
watcher.role_len.store(0, Ordering::Release);
self.clear_interest(self.lanes + slot);
if let Some(bell) = &self.bell {
bell.unbind(crate::bell::Doorbell::Watcher(slot));
}
}
#[cfg(all(unix, feature = "tokio"))]
pub(crate) fn watcher_generation(
&self,
slot: usize,
) -> Result<u32> {
self.check_lane(slot)?;
Ok(self.watcher_header(slot).changes.load(Ordering::SeqCst))
}
#[cfg(all(unix, feature = "tokio"))]
pub(crate) fn arm_watcher(
&self,
slot: usize,
) -> Result<()> {
self.check_lane(slot)?;
self.watcher_header(slot).reactor.store(1, Ordering::SeqCst);
Ok(())
}
#[cfg(all(unix, feature = "tokio"))]
pub(crate) fn disarm_watcher(
&self,
slot: usize,
) {
if slot < self.lanes {
self.watcher_header(slot).reactor.store(0, Ordering::SeqCst);
}
}
pub fn watcher(
&self,
slot: usize,
) -> Result<Option<u32>> {
self.check_lane(slot)?;
Ok(match self.watcher_header(slot).pid.load(Ordering::Acquire) {
0 => None,
pid => Some(pid),
})
}
pub fn watcher_role(
&self,
slot: usize,
) -> Result<Option<String>> {
self.check_lane(slot)?;
let watcher = self.watcher_header(slot);
if watcher.pid.load(Ordering::Acquire) == 0 {
return Ok(None);
}
let len = watcher.role_len.load(Ordering::Acquire) as usize;
if len == 0 || len > LANE_ROLE_MAX {
return Ok(None);
}
let bytes =
unsafe { std::slice::from_raw_parts(watcher.role.as_ptr(), len) };
Ok(String::from_utf8(bytes.to_vec()).ok())
}
pub(crate) fn write_interest(
&self,
entry: usize,
bits: &[u64; INTEREST_WORDS],
) {
let interest = self.interest_entry(entry);
let writing = interest.seq.load(Ordering::Relaxed) | 1;
interest.seq.store(writing, Ordering::Release);
for (word, value) in interest.bits.iter().zip(bits) {
word.store(*value, Ordering::Release);
}
interest.seq.store(writing.wrapping_add(1), Ordering::Release);
}
#[cfg_attr(not(feature = "events"), allow(dead_code))]
pub(crate) fn read_interest(
&self,
entry: usize,
) -> [u64; INTEREST_WORDS] {
let interest = self.interest_entry(entry);
let before = interest.seq.load(Ordering::Acquire);
if before.is_multiple_of(2) {
let mut bits = [0u64; INTEREST_WORDS];
for (value, word) in bits.iter_mut().zip(&interest.bits) {
*value = word.load(Ordering::Acquire);
}
if interest.seq.load(Ordering::Acquire) == before {
return bits;
}
}
[u64::MAX; INTEREST_WORDS]
}
fn clear_interest(
&self,
entry: usize,
) {
self.write_interest(entry, &[0; INTEREST_WORDS]);
}
fn interest_entry(
&self,
entry: usize,
) -> &InterestEntry {
unsafe {
&*self
.base
.add(
size_of::<TableHeader>()
+ self.lanes
* (self.geometry.lane_bytes()
+ size_of::<WatcherHeader>())
+ entry * size_of::<InterestEntry>(),
)
.cast::<InterestEntry>()
}
}
fn watcher_header(
&self,
slot: usize,
) -> &WatcherHeader {
unsafe {
&*self
.base
.add(
size_of::<TableHeader>()
+ self.lanes * self.geometry.lane_bytes()
+ slot * size_of::<WatcherHeader>(),
)
.cast::<WatcherHeader>()
}
}
pub(crate) fn check_lane(
&self,
lane_index: usize,
) -> Result<()> {
if lane_index >= self.lanes {
return Err(Error::Malformed("lane is outside the table"));
}
Ok(())
}
fn table_header(&self) -> &TableHeader {
unsafe { &*self.base.cast::<TableHeader>() }
}
fn lane_base(
&self,
lane_index: usize,
) -> *mut u8 {
unsafe {
self.base.add(
size_of::<TableHeader>()
+ lane_index * self.geometry.lane_bytes(),
)
}
}
pub(crate) fn lane_header(
&self,
lane_index: usize,
) -> &LaneHeader {
unsafe { &*self.lane_base(lane_index).cast::<LaneHeader>() }
}
fn slot_base(
&self,
lane_index: usize,
position: u64,
) -> *mut u8 {
let index = (position as usize) & (self.geometry.capacity - 1);
let stride = self.geometry.slot_bytes();
unsafe {
self.lane_base(lane_index)
.add(size_of::<LaneHeader>() + index * stride)
}
}
fn slot_header(
&self,
lane_index: usize,
position: u64,
) -> &SlotHeader {
unsafe { &*self.slot_base(lane_index, position).cast::<SlotHeader>() }
}
fn payload_ptr(
&self,
lane_index: usize,
position: u64,
) -> *mut u8 {
unsafe {
self.slot_base(lane_index, position).add(size_of::<SlotHeader>())
}
}
}
fn check_identity(
service: &str,
role: &str,
) -> Result<()> {
if service.is_empty() || service.len() > LANE_NAME_MAX {
return Err(Error::Malformed("service name does not fit a lane"));
}
if role.is_empty() || role.len() > LANE_ROLE_MAX || !role.is_ascii() {
return Err(Error::Malformed("role tag does not fit a lane"));
}
Ok(())
}
#[cfg(unix)]
fn ring_armed(
reactor: &AtomicU32,
ring: impl FnOnce() -> io::Result<()>,
) -> io::Result<()> {
if reactor.swap(0, Ordering::SeqCst) == 0 {
return Ok(());
}
if let Err(error) = ring() {
reactor.fetch_or(1, Ordering::SeqCst);
return Err(error);
}
Ok(())
}
#[cfg(all(test, unix, feature = "tokio"))]
#[path = "../tests/unit/bell.rs"]
mod bell_tests;