use std::sync::atomic::{AtomicU8, AtomicU32, AtomicU64, Ordering};
use super::{Error, Result};
pub(crate) const INBOX_MAGIC: u32 = 0x4F54_5849;
pub(crate) const INBOX_VERSION: u32 = 4;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Reclaimed {
pub lane: usize,
pub incarnation: u64,
pub pid: u32
}
pub const LANE_NAME_MAX: usize = 47;
pub const LANE_ROLE_MAX: usize = 8;
#[repr(C, align(64))]
pub(crate) struct LaneHeader {
pub(crate) reserve: AtomicU64,
pub(crate) read: AtomicU64,
pub(crate) consumers: AtomicU32,
pub(crate) changes: AtomicU32,
pub(crate) incarnation: AtomicU64,
pub(crate) waiters: AtomicU32,
pub(crate) reactor: AtomicU32,
_padding: [u8; 16],
pub(crate) service: [u8; LANE_NAME_MAX],
pub(crate) service_len: AtomicU8,
pub(crate) pid: AtomicU32,
pub(crate) role: [u8; LANE_ROLE_MAX],
pub(crate) role_len: AtomicU8,
_identity_padding: [u8; 3]
}
const _: () = assert!(size_of::<LaneHeader>() == 128);
#[repr(C, align(64))]
pub(crate) struct SlotHeader {
pub(crate) seq: AtomicU64,
pub(crate) len: AtomicU32,
_padding: [u8; 52]
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum Admission {
Accepted,
Full,
NoConsumer
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct InboxGeometry {
pub capacity: usize,
pub payload_capacity: usize
}
impl InboxGeometry {
pub const fn new(
capacity: usize,
payload_capacity: usize
) -> Self {
Self { capacity, payload_capacity }
}
pub const fn lane_bytes(&self) -> usize {
size_of::<LaneHeader>() + self.capacity * (size_of::<SlotHeader>() + self.payload_capacity)
}
pub const fn segment_bytes(
&self,
fleet_capacity: usize
) -> usize {
size_of::<TableHeader>() + fleet_capacity * self.lane_bytes()
}
pub(crate) fn validate(&self) -> Result<()> {
if self.capacity == 0 || !self.capacity.is_power_of_two() {
return Err(Error::Malformed("inbox capacity must be a power of two"));
}
if self.payload_capacity == 0 {
return Err(Error::Malformed("inbox payload capacity must not be zero"));
}
Ok(())
}
}
#[repr(C, align(64))]
pub(crate) struct TableHeader {
pub(crate) magic: AtomicU32,
pub(crate) version: AtomicU32,
pub(crate) fleet_capacity: AtomicU32,
pub(crate) capacity: AtomicU32,
pub(crate) payload_capacity: AtomicU32,
pub(crate) directory: AtomicU64,
_padding: [u8; 36]
}
pub struct Inbox {
base: *mut u8,
lanes: usize,
geometry: InboxGeometry,
#[cfg(unix)]
bell: Option<crate::bell::Bell>
}
unsafe impl Send for Inbox {}
unsafe impl Sync for Inbox {}
impl Inbox {
pub unsafe fn initialize(
base: *mut u8,
fleet_capacity: usize,
geometry: InboxGeometry
) -> Result<Self> {
geometry.validate()?;
let inbox = Self {
base,
lanes: fleet_capacity,
geometry,
#[cfg(unix)]
bell: None
};
let header = inbox.table_header();
header.fleet_capacity.store(fleet_capacity as u32, Ordering::Relaxed);
header.capacity.store(geometry.capacity as u32, Ordering::Relaxed);
header.payload_capacity.store(geometry.payload_capacity as u32, Ordering::Relaxed);
header.version.store(INBOX_VERSION, Ordering::Relaxed);
header.magic.store(INBOX_MAGIC, Ordering::Release);
Ok(inbox)
}
pub unsafe fn attach(
base: *mut u8,
fleet_capacity: usize,
geometry: InboxGeometry
) -> Result<Self> {
geometry.validate()?;
let inbox = Self {
base,
lanes: fleet_capacity,
geometry,
#[cfg(unix)]
bell: None
};
let header = inbox.table_header();
if header.magic.load(Ordering::Acquire) != INBOX_MAGIC {
return Err(Error::Malformed("inbox segment is not an inbox table"));
}
if header.version.load(Ordering::Relaxed) != INBOX_VERSION {
return Err(Error::Malformed("inbox segment was written by another layout version"));
}
if header.fleet_capacity.load(Ordering::Relaxed) as usize != fleet_capacity
|| header.capacity.load(Ordering::Relaxed) as usize != geometry.capacity
|| header.payload_capacity.load(Ordering::Relaxed) as usize != geometry.payload_capacity
{
return Err(Error::Malformed("inbox segment geometry differs from this build"));
}
Ok(inbox)
}
pub fn geometry(&self) -> InboxGeometry {
self.geometry
}
pub fn lanes(&self) -> usize {
self.lanes
}
pub fn claim_lane(
&self,
lane_index: usize,
service: &str,
role: &str,
incarnation: u64
) -> Result<()> {
self.check_lane(lane_index)?;
check_identity(service, role)?;
self.reset_lane(lane_index);
let lane = self.lane_header(lane_index);
lane.consumers.store(1, Ordering::Release);
self.publish_identity(lane, service, role, incarnation);
self.wake(lane_index, lane);
Ok(())
}
pub fn join<H>(
&self,
service: &str,
role: &str,
incarnation: u64,
mut hold: impl FnMut(usize) -> Result<Option<H>>
) -> Result<(usize, H)> {
check_identity(service, role)?;
self.refuse_other_role(service, role, None)?;
for lane_index in 0..self.lanes {
let lane = self.lane_header(lane_index);
if lane.consumers.load(Ordering::Acquire) != 0 {
continue;
}
let Some(held) = hold(lane_index)? else {
continue;
};
if lane.consumers.compare_exchange(0, 1, Ordering::AcqRel, Ordering::Acquire).is_err() {
continue;
}
self.reset_lane(lane_index);
self.publish_identity(lane, service, role, incarnation);
if let Err(conflict) = self.refuse_other_role(service, role, Some(lane_index)) {
let _ = self.release_lane(lane_index);
return Err(conflict);
}
self.wake(lane_index, lane);
return Ok((lane_index, held));
}
Err(Error::NoLane)
}
pub fn reclaim(
&self,
lane_index: usize
) -> Result<Option<Reclaimed>> {
self.check_lane(lane_index)?;
let lane = self.lane_header(lane_index);
if lane.consumers.load(Ordering::Acquire) == 0 {
return Ok(None);
}
let reclaimed = Reclaimed {
lane: lane_index,
incarnation: lane.incarnation.load(Ordering::Acquire),
pid: lane.pid.load(Ordering::Acquire)
};
self.release_lane(lane_index)?;
Ok(Some(reclaimed))
}
fn refuse_other_role(
&self,
service: &str,
role: &str,
own: Option<usize>
) -> Result<()> {
for lane_index in self.serving(service) {
if Some(lane_index) == own {
continue;
}
let lane = self.lane_header(lane_index);
if lane.consumers.load(Ordering::Acquire) == 0 {
continue;
}
let held_by = self.role(lane_index)?.unwrap_or_default();
if held_by != role {
return Err(Error::RoleConflict {
name: service.to_owned(),
role: role.to_owned(),
held_by: if held_by.is_empty() { "an unnamed role".into() } else { held_by },
lane: lane_index,
pid: lane.pid.load(Ordering::Acquire)
});
}
}
Ok(())
}
pub fn park(
&self,
lane_index: usize,
expected: u32
) -> Result<()> {
self.check_lane(lane_index)?;
let lane = self.lane_header(lane_index);
lane.waiters.fetch_add(1, Ordering::AcqRel);
let waited = orbit_core::sync::wait_word(&lane.changes, expected);
lane.waiters.fetch_sub(1, Ordering::AcqRel);
waited.map_err(Error::Io)
}
pub fn nudge(
&self,
lane_index: usize
) -> Result<()> {
self.check_lane(lane_index)?;
self.wake(lane_index, self.lane_header(lane_index));
Ok(())
}
pub fn service(
&self,
lane_index: usize
) -> Result<Option<String>> {
self.check_lane(lane_index)?;
let lane = self.lane_header(lane_index);
let len = lane.service_len.load(Ordering::Acquire) as usize;
if len == 0 || len > LANE_NAME_MAX {
return Ok(None);
}
let bytes = unsafe { std::slice::from_raw_parts(lane.service.as_ptr(), len) };
Ok(String::from_utf8(bytes.to_vec()).ok())
}
pub fn owner(
&self,
lane_index: usize
) -> Result<Option<u32>> {
self.check_lane(lane_index)?;
Ok(match self.lane_header(lane_index).pid.load(Ordering::Acquire) {
0 => None,
pid => Some(pid)
})
}
pub fn role(
&self,
lane_index: usize
) -> Result<Option<String>> {
self.check_lane(lane_index)?;
let lane = self.lane_header(lane_index);
let len = lane.role_len.load(Ordering::Acquire) as usize;
if len == 0 || len > LANE_ROLE_MAX {
return Ok(None);
}
let bytes = unsafe { std::slice::from_raw_parts(lane.role.as_ptr(), len) };
Ok(String::from_utf8(bytes.to_vec()).ok())
}
fn serves(
&self,
lane_index: usize,
service: &[u8]
) -> bool {
let lane = self.lane_header(lane_index);
let len = lane.service_len.load(Ordering::Acquire) as usize;
if len != service.len() || len > LANE_NAME_MAX {
return false;
}
let name = unsafe { std::slice::from_raw_parts(lane.service.as_ptr(), len) };
name == service
}
pub fn targets(
&self,
service: &str
) -> Vec<(usize, u64)> {
self.ready(&self.serving(service))
}
pub fn directory(&self) -> u64 {
self.table_header().directory.load(Ordering::Acquire)
}
pub fn serving(
&self,
service: &str
) -> Vec<usize> {
let wanted = service.as_bytes();
(0..self.lanes).filter(|lane| self.serves(*lane, wanted)).collect()
}
pub fn ready(
&self,
lanes: &[usize]
) -> Vec<(usize, u64)> {
let mut found: Vec<(usize, u64)> = lanes
.iter()
.copied()
.filter(|lane| matches!(self.admits(*lane), Ok(Admission::Accepted)))
.filter_map(|lane| self.depth(lane).ok().map(|depth| (lane, depth)))
.collect();
found.sort_by_key(|(_, depth)| *depth);
found
}
pub fn release_lane(
&self,
lane_index: usize
) -> Result<()> {
self.check_lane(lane_index)?;
let lane = self.lane_header(lane_index);
lane.consumers.store(0, Ordering::Release);
lane.service_len.store(0, Ordering::Release);
lane.role_len.store(0, Ordering::Release);
lane.pid.store(0, Ordering::Release);
self.table_header().directory.fetch_add(1, Ordering::AcqRel);
self.wake(lane_index, lane);
Ok(())
}
pub fn admits(
&self,
lane_index: usize
) -> Result<Admission> {
self.check_lane(lane_index)?;
let lane = self.lane_header(lane_index);
if lane.consumers.load(Ordering::Acquire) == 0 {
return Ok(Admission::NoConsumer);
}
let read = lane.read.load(Ordering::Acquire);
let reserve = lane.reserve.load(Ordering::Acquire);
if reserve.wrapping_sub(read) >= self.geometry.capacity as u64 {
return Ok(Admission::Full);
}
Ok(Admission::Accepted)
}
pub fn depth(
&self,
lane_index: usize
) -> Result<u64> {
self.check_lane(lane_index)?;
let lane = self.lane_header(lane_index);
let read = lane.read.load(Ordering::Acquire);
let reserve = lane.reserve.load(Ordering::Acquire);
Ok(reserve.wrapping_sub(read))
}
pub fn consumers(
&self,
lane_index: usize
) -> Result<u32> {
self.check_lane(lane_index)?;
Ok(self.lane_header(lane_index).consumers.load(Ordering::Acquire))
}
pub fn incarnation(
&self,
lane_index: usize
) -> Result<u64> {
self.check_lane(lane_index)?;
Ok(self.lane_header(lane_index).incarnation.load(Ordering::Acquire))
}
pub fn write(
&self,
lane_index: usize,
payload: &[u8]
) -> Result<u64> {
self.check_lane(lane_index)?;
if payload.len() > self.geometry.payload_capacity {
return Err(Error::FrameTooLarge);
}
let lane = self.lane_header(lane_index);
let capacity = self.geometry.capacity as u64;
let position = loop {
if lane.consumers.load(Ordering::Acquire) == 0 {
return Err(Error::InboxFull);
}
let read = lane.read.load(Ordering::Acquire);
let reserve = lane.reserve.load(Ordering::Acquire);
if reserve.wrapping_sub(read) >= capacity {
return Err(Error::InboxFull);
}
let slot = self.slot_header(lane_index, reserve);
if slot.seq.load(Ordering::Acquire) != reserve {
std::hint::spin_loop();
continue;
}
if lane
.reserve
.compare_exchange_weak(reserve, reserve + 1, Ordering::AcqRel, Ordering::Acquire)
.is_ok()
{
break reserve;
}
std::hint::spin_loop();
};
let slot = self.slot_header(lane_index, position);
unsafe {
std::ptr::copy_nonoverlapping(
payload.as_ptr(),
self.payload_ptr(lane_index, position),
payload.len()
);
}
slot.len.store(payload.len() as u32, Ordering::Relaxed);
slot.seq.store(position + 1, Ordering::Release);
self.wake(lane_index, lane);
Ok(position)
}
pub fn read(
&self,
lane_index: usize,
out: &mut Vec<u8>
) -> Result<bool> {
self.check_lane(lane_index)?;
let lane = self.lane_header(lane_index);
let position = lane.read.load(Ordering::Relaxed);
let slot = self.slot_header(lane_index, position);
if slot.seq.load(Ordering::Acquire) != position + 1 {
return Ok(false);
}
let len = slot.len.load(Ordering::Relaxed) as usize;
out.clear();
out.reserve(len);
unsafe {
std::ptr::copy_nonoverlapping(
self.payload_ptr(lane_index, position),
out.spare_capacity_mut().as_mut_ptr().cast::<u8>(),
len
);
out.set_len(len);
}
lane.read.store(position + 1, Ordering::Release);
slot.seq.store(position + self.geometry.capacity as u64, Ordering::Release);
Ok(true)
}
pub fn wait_word(
&self,
lane_index: usize
) -> Result<(&AtomicU32, u32)> {
self.check_lane(lane_index)?;
let lane = self.lane_header(lane_index);
Ok((&lane.changes, lane.changes.load(Ordering::Acquire)))
}
fn reset_lane(
&self,
lane_index: usize
) {
let lane = self.lane_header(lane_index);
lane.reserve.store(0, Ordering::Relaxed);
lane.read.store(0, Ordering::Relaxed);
lane.service_len.store(0, Ordering::Relaxed);
lane.role_len.store(0, Ordering::Relaxed);
lane.waiters.store(0, Ordering::Relaxed);
lane.reactor.store(0, Ordering::Relaxed);
for position in 0..self.geometry.capacity {
let slot = self.slot_header(lane_index, position as u64);
slot.len.store(0, Ordering::Relaxed);
slot.seq.store(position as u64, Ordering::Release);
}
}
fn publish_identity(
&self,
lane: &LaneHeader,
service: &str,
role: &str,
incarnation: u64
) {
unsafe {
std::ptr::copy_nonoverlapping(
service.as_ptr(),
lane.service.as_ptr().cast_mut(),
service.len()
);
std::ptr::copy_nonoverlapping(role.as_ptr(), lane.role.as_ptr().cast_mut(), role.len());
}
lane.role_len.store(role.len() as u8, Ordering::Release);
lane.service_len.store(service.len() as u8, Ordering::Release);
lane.incarnation.store(incarnation, Ordering::Release);
lane.pid.store(std::process::id(), Ordering::Release);
self.table_header().directory.fetch_add(1, Ordering::AcqRel);
}
fn wake(
&self,
lane_index: usize,
lane: &LaneHeader
) {
lane.changes.fetch_add(1, Ordering::SeqCst);
if lane.waiters.load(Ordering::SeqCst) > 0 {
let _ = orbit_core::sync::wake_word(&lane.changes);
}
#[cfg(unix)]
if lane.reactor.swap(0, Ordering::SeqCst) != 0
&& let Some(bell) = &self.bell
{
bell.ring(lane_index);
}
}
#[cfg(unix)]
pub(crate) fn install_bell(
&mut self,
segment: &str
) -> Result<()> {
self.bell = Some(crate::bell::Bell::new(segment).map_err(Error::Io)?);
Ok(())
}
#[cfg(all(unix, feature = "tokio"))]
pub(crate) fn bind_receiver(
&self,
lane_index: usize
) -> Result<tokio::net::UnixDatagram> {
self.check_lane(lane_index)?;
let bell = self.bell.as_ref().ok_or(Error::Malformed("the inbox has no reactor bell"))?;
let socket = bell.bind(lane_index).map_err(Error::Io)?;
tokio::net::UnixDatagram::from_std(socket).map_err(Error::Io)
}
#[cfg(all(unix, feature = "tokio"))]
pub(crate) fn unbind_receiver(
&self,
lane_index: usize
) {
self.disarm_receiver(lane_index);
if let Some(bell) = &self.bell {
bell.unbind(lane_index);
}
}
#[cfg(all(unix, feature = "tokio"))]
pub(crate) fn receiver_generation(
&self,
lane_index: usize
) -> Result<u32> {
self.check_lane(lane_index)?;
Ok(self.lane_header(lane_index).changes.load(Ordering::SeqCst))
}
#[cfg(all(unix, feature = "tokio"))]
pub(crate) fn arm_receiver(
&self,
lane_index: usize
) -> Result<()> {
self.check_lane(lane_index)?;
self.lane_header(lane_index).reactor.store(1, Ordering::SeqCst);
Ok(())
}
#[cfg(all(unix, feature = "tokio"))]
pub(crate) fn disarm_receiver(
&self,
lane_index: usize
) {
if lane_index < self.lanes {
self.lane_header(lane_index).reactor.store(0, Ordering::SeqCst);
}
}
fn check_lane(
&self,
lane_index: usize
) -> Result<()> {
if lane_index >= self.lanes {
return Err(Error::Malformed("lane is outside the table"));
}
Ok(())
}
fn table_header(&self) -> &TableHeader {
unsafe { &*self.base.cast::<TableHeader>() }
}
fn lane_base(
&self,
lane_index: usize
) -> *mut u8 {
unsafe { self.base.add(size_of::<TableHeader>() + lane_index * self.geometry.lane_bytes()) }
}
fn lane_header(
&self,
lane_index: usize
) -> &LaneHeader {
unsafe { &*self.lane_base(lane_index).cast::<LaneHeader>() }
}
fn slot_base(
&self,
lane_index: usize,
position: u64
) -> *mut u8 {
let index = (position as usize) & (self.geometry.capacity - 1);
let stride = size_of::<SlotHeader>() + self.geometry.payload_capacity;
unsafe { self.lane_base(lane_index).add(size_of::<LaneHeader>() + index * stride) }
}
fn slot_header(
&self,
lane_index: usize,
position: u64
) -> &SlotHeader {
unsafe { &*self.slot_base(lane_index, position).cast::<SlotHeader>() }
}
fn payload_ptr(
&self,
lane_index: usize,
position: u64
) -> *mut u8 {
unsafe { self.slot_base(lane_index, position).add(size_of::<SlotHeader>()) }
}
}
fn check_identity(
service: &str,
role: &str
) -> Result<()> {
if service.is_empty() || service.len() > LANE_NAME_MAX {
return Err(Error::Malformed("service name does not fit a lane"));
}
if role.is_empty() || role.len() > LANE_ROLE_MAX || !role.is_ascii() {
return Err(Error::Malformed("role tag does not fit a lane"));
}
Ok(())
}