use std::mem::{align_of, size_of};
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
use orbit_core::process::ProcessOwner;
use orbit_core::shm::ShmRegion;
use crate::{Error, Joined, LinkSegment, Result};
const MAGIC: u64 = u64::from_le_bytes(*b"OLCTRL01");
const MASK: u64 = 3;
const FREE: u64 = 0;
const PENDING: u64 = 1;
const CANCELLED: u64 = 2;
const FAILED: u64 = 3;
#[repr(C)]
struct Header {
magic: AtomicU64,
lanes: AtomicU64,
slots_per_lane: AtomicU64,
}
#[repr(C)]
struct Slot {
state: AtomicU64,
origin_epoch: AtomicU64,
runner: AtomicU64,
runner_epoch: AtomicU64,
}
const _: () = assert!(size_of::<Slot>() == 32);
const _: () =
assert!(size_of::<Header>().is_multiple_of(align_of::<AtomicU64>()));
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct ExchangeControlTicket {
pub origin: u16,
pub origin_epoch: u64,
pub runner: u16,
pub runner_epoch: u64,
pub slot: u32,
pub generation: u64,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum ExchangeControlState {
Pending,
Cancelled,
Failed,
}
pub struct ExchangeControls {
segment: Arc<LinkSegment>,
region: ShmRegion,
kind: u8,
slots_per_lane: usize,
next: AtomicUsize,
process: ProcessOwner,
}
impl ExchangeControls {
pub fn open(
segment: Arc<LinkSegment>,
kind: u8,
slots_per_lane: u32,
) -> Result<Self> {
let spec = segment.spec();
if kind == spec.inbox_kind
|| kind == spec.streams.kind
|| slots_per_lane == 0
{
return Err(Error::Malformed(
"invalid Link exchange control geometry",
));
}
let lanes = segment.inbox().lanes();
let slots_per_lane = slots_per_lane as usize;
let size = lanes
.checked_mul(slots_per_lane)
.and_then(|slots| slots.checked_mul(size_of::<Slot>()))
.and_then(|bytes| {
bytes.checked_add(
size_of::<Header>() + lanes * size_of::<AtomicU64>(),
)
})
.ok_or(Error::Malformed(
"Link exchange control geometry overflow",
))?;
let process = ProcessOwner::current()?;
let (region, lock) = segment.open_control_region(kind, size)?;
let table = Self {
segment,
region,
kind,
slots_per_lane,
next: AtomicUsize::new(0),
process,
};
let header = unsafe { &*table.region.as_ptr().cast::<Header>() };
if table.region.created() {
header.lanes.store(lanes as u64, Ordering::Relaxed);
header
.slots_per_lane
.store(slots_per_lane as u64, Ordering::Relaxed);
header.magic.store(MAGIC, Ordering::Release);
}
if header.magic.load(Ordering::Acquire) != MAGIC
|| header.lanes.load(Ordering::Relaxed) != lanes as u64
|| header.slots_per_lane.load(Ordering::Relaxed)
!= slots_per_lane as u64
{
return Err(Error::Malformed(
"Link exchange control layout mismatch",
));
}
drop(lock);
Ok(table)
}
pub fn reserve(
self: &Arc<Self>,
caller: &Joined,
runner: usize,
runner_epoch: u64,
) -> Result<Option<ExchangeControl>> {
self.check_owner(caller)?;
let inbox = self.segment.inbox();
let origin_epoch = caller.epoch();
if inbox.lane_epoch(runner)? != runner_epoch
|| inbox.consumers(runner)? == 0
|| inbox.consumers(caller.lane)? == 0
{
return Err(Error::NoConsumer);
}
self.prepare_lane(caller.lane, origin_epoch)?;
let first =
self.next.fetch_add(1, Ordering::Relaxed) % self.slots_per_lane;
for offset in 0..self.slots_per_lane {
let index = (first + offset) % self.slots_per_lane;
let slot = self.slot(caller.lane, index);
let old = slot.state.load(Ordering::Acquire);
if old & MASK != FREE {
continue;
}
let Some(generation) = (old & !MASK).checked_add(MASK + 1) else {
continue;
};
if slot
.state
.compare_exchange(
old,
generation | PENDING,
Ordering::AcqRel,
Ordering::Acquire,
)
.is_err()
{
continue;
}
slot.origin_epoch.store(origin_epoch, Ordering::Relaxed);
slot.runner.store(runner as u64, Ordering::Relaxed);
slot.runner_epoch.store(runner_epoch, Ordering::Release);
return Ok(Some(ExchangeControl {
table: Arc::clone(self),
ticket: ExchangeControlTicket {
origin: caller.lane as u16,
origin_epoch,
runner: runner as u16,
runner_epoch,
slot: index as u32,
generation,
},
}));
}
Ok(None)
}
pub fn state(
&self,
ticket: ExchangeControlTicket,
) -> Result<ExchangeControlState> {
self.decode_state(ticket, true)
}
fn decode_state(
&self,
ticket: ExchangeControlTicket,
require_runner: bool,
) -> Result<ExchangeControlState> {
let state = self.checked_state(ticket, require_runner)?;
match state & MASK {
PENDING => Ok(ExchangeControlState::Pending),
CANCELLED => Ok(ExchangeControlState::Cancelled),
FAILED => Ok(ExchangeControlState::Failed),
_ => Err(Error::StaleWork),
}
}
pub fn caller_state(
&self,
ticket: ExchangeControlTicket,
) -> Result<ExchangeControlState> {
self.decode_state(ticket, false)
}
pub fn fail(
&self,
runner: &Joined,
ticket: ExchangeControlTicket,
) -> Result<bool> {
self.check_owner(runner)?;
if runner.lane != usize::from(ticket.runner)
|| runner.epoch() != ticket.runner_epoch
{
return Err(Error::StaleWork);
}
self.transition(ticket, FAILED, usize::from(ticket.origin))
}
pub fn unlink(&self) -> Result<()> {
self.segment.unlink_control_region(&self.region)
}
fn check_owner(
&self,
joined: &Joined,
) -> Result<()> {
if !self.process.is_current()
|| !joined.belongs_to(&self.segment)
|| self.segment.inbox().last_owner(joined.lane)?.map(|(pid, _)| pid)
!= Some(self.process.pid())
{
return Err(Error::StaleWork);
}
Ok(())
}
fn checked_state(
&self,
ticket: ExchangeControlTicket,
require_runner: bool,
) -> Result<u64> {
let inbox = self.segment.inbox();
let origin = usize::from(ticket.origin);
let runner = usize::from(ticket.runner);
let index = ticket.slot as usize;
if ticket.generation == 0
|| ticket.generation & MASK != 0
|| index >= self.slots_per_lane
|| inbox.lane_epoch(origin)? != ticket.origin_epoch
|| (require_runner
&& inbox.lane_epoch(runner)? != ticket.runner_epoch)
{
return Err(Error::StaleWork);
}
let slot = self.slot(origin, index);
let state = slot.state.load(Ordering::Acquire);
if state & !MASK != ticket.generation
|| state & MASK == FREE
|| slot.runner_epoch.load(Ordering::Acquire) != ticket.runner_epoch
|| slot.origin_epoch.load(Ordering::Relaxed) != ticket.origin_epoch
|| slot.runner.load(Ordering::Relaxed) != runner as u64
{
return Err(Error::StaleWork);
}
let state = slot.state.load(Ordering::Acquire);
if state & !MASK != ticket.generation || state & MASK == FREE {
return Err(Error::StaleWork);
}
Ok(state)
}
fn transition(
&self,
ticket: ExchangeControlTicket,
terminal: u64,
notify: usize,
) -> Result<bool> {
if self.checked_state(ticket, true)? & MASK != PENDING {
return Ok(false);
}
let slot = self.slot(usize::from(ticket.origin), ticket.slot as usize);
let changed = slot
.state
.compare_exchange(
ticket.generation | PENDING,
ticket.generation | terminal,
Ordering::AcqRel,
Ordering::Acquire,
)
.is_ok();
if changed {
self.segment.inbox().notify_capacity_observer(notify);
}
Ok(changed)
}
fn prepare_lane(
&self,
lane: usize,
epoch: u64,
) -> Result<()> {
let prepared = unsafe {
&*self
.region
.as_ptr()
.add(size_of::<Header>())
.cast::<AtomicU64>()
.add(lane)
};
if prepared.load(Ordering::Acquire) != epoch {
let _lock =
self.segment.lock_control_region(self.kind, &self.region)?;
if prepared.load(Ordering::Relaxed) != epoch {
for index in 0..self.slots_per_lane {
self.slot(lane, index)
.state
.fetch_and(!MASK, Ordering::AcqRel);
}
prepared.store(epoch, Ordering::Release);
}
}
Ok(())
}
fn slot(
&self,
lane: usize,
index: usize,
) -> &Slot {
unsafe {
&*self
.region
.as_ptr()
.add(
size_of::<Header>()
+ self.segment.inbox().lanes() * size_of::<AtomicU64>(),
)
.cast::<Slot>()
.add(lane * self.slots_per_lane + index)
}
}
}
pub struct ExchangeControl {
table: Arc<ExchangeControls>,
ticket: ExchangeControlTicket,
}
impl ExchangeControl {
pub fn ticket(&self) -> ExchangeControlTicket {
self.ticket
}
pub fn caller_state(&self) -> Result<ExchangeControlState> {
self.table.caller_state(self.ticket)
}
pub fn cancel(&self) -> Result<bool> {
if !self.table.process.is_current() {
return Err(Error::StaleWork);
}
self.table.transition(
self.ticket,
CANCELLED,
usize::from(self.ticket.runner),
)
}
pub fn complete(self) {
self.release(false);
}
fn release(
&self,
cancel: bool,
) {
if !self.table.process.is_current() {
return;
}
let ticket = self.ticket;
let slot =
self.table.slot(usize::from(ticket.origin), ticket.slot as usize);
let previous = slot.state.try_update(
Ordering::AcqRel,
Ordering::Acquire,
|state| {
(state & !MASK == ticket.generation && state & MASK != FREE)
.then_some(ticket.generation)
},
);
if cancel && previous == Ok(ticket.generation | PENDING) {
self.table
.segment
.inbox()
.notify_capacity_observer(usize::from(ticket.runner));
}
}
}
impl Drop for ExchangeControl {
fn drop(&mut self) {
self.release(true);
}
}
#[cfg(all(test, feature = "tokio"))]
#[path = "../tests/unit/control.rs"]
mod tests;