use crate::{
TaskFailure,
signal::{Signal, lock},
task::SharedTaskRecord,
wait::WaitHub,
};
use std::{
collections::{BTreeMap, VecDeque},
sync::{
Arc, Mutex,
atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering},
},
};
#[cfg(feature = "runtime-evidence")]
type EvidenceEmitter = crate::diagnostics::evidence::Emitter;
#[cfg(not(feature = "runtime-evidence"))]
type EvidenceEmitter = ();
#[repr(align(64))]
struct RetiredTasks(AtomicU64);
pub(crate) struct SpawnPacket {
pub(crate) record: SharedTaskRecord,
pub(crate) entry: Option<crate::task_body::TaskStart>,
#[cfg(test)]
pub(crate) test_id: crate::TaskId,
}
#[derive(Default)]
struct InboxState {
starts: VecDeque<SpawnPacket>,
stopped: bool,
abort: BTreeMap<u64, TaskFailure>,
}
pub(crate) struct Inbox {
pub(crate) started: AtomicBool,
pub(crate) scheduler_stopped: AtomicBool,
pub(crate) reclaimed: AtomicBool,
capacity: usize,
state: Mutex<InboxState>,
pending_starts: AtomicUsize,
stopped: AtomicBool,
abort_pending: AtomicBool,
retired_tasks: RetiredTasks,
pub(crate) signal: Arc<Signal>,
pub(crate) hub: Arc<WaitHub>,
#[cfg(test)]
pub(crate) before_notify_hook: Mutex<Option<Box<dyn FnOnce() + Send>>>,
#[cfg(feature = "runtime-evidence")]
evidence: Option<crate::diagnostics::evidence::Emitter>,
}
impl Inbox {
pub(crate) fn new(capacity: usize, wait_capacity: usize, track_wakes: bool) -> Self {
Self::construct(capacity, wait_capacity, track_wakes, None)
}
#[cfg(feature = "runtime-evidence")]
pub(crate) fn with_evidence(
capacity: usize,
wait_capacity: usize,
track_wakes: bool,
evidence: crate::diagnostics::evidence::Emitter,
) -> Self {
Self::construct(capacity, wait_capacity, track_wakes, Some(evidence))
}
fn construct(
capacity: usize,
wait_capacity: usize,
track_wakes: bool,
evidence: Option<EvidenceEmitter>,
) -> Self {
#[cfg(not(feature = "runtime-evidence"))]
let _ = evidence;
let signal = Arc::new(Signal::default());
#[cfg(feature = "runtime-evidence")]
let hub = evidence.as_ref().map_or_else(
|| {
if track_wakes {
WaitHub::new_tracked(wait_capacity, Arc::clone(&signal))
} else {
WaitHub::new(wait_capacity, Arc::clone(&signal))
}
},
|evidence| {
WaitHub::with_evidence(
wait_capacity,
Arc::clone(&signal),
evidence.clone(),
track_wakes,
)
},
);
#[cfg(not(feature = "runtime-evidence"))]
let hub = if track_wakes {
WaitHub::new_tracked(wait_capacity, Arc::clone(&signal))
} else {
WaitHub::new(wait_capacity, Arc::clone(&signal))
};
Self {
started: AtomicBool::new(false),
scheduler_stopped: AtomicBool::new(false),
reclaimed: AtomicBool::new(false),
capacity,
state: Mutex::default(),
pending_starts: AtomicUsize::new(0),
stopped: AtomicBool::new(false),
abort_pending: AtomicBool::new(false),
retired_tasks: RetiredTasks(AtomicU64::new(0)),
hub: Arc::new(hub),
signal,
#[cfg(test)]
before_notify_hook: Mutex::new(None),
#[cfg(feature = "runtime-evidence")]
evidence,
}
}
pub(crate) fn can_accept(&self) -> bool {
!self.stopped.load(Ordering::Acquire)
&& self.pending_starts.load(Ordering::Acquire) < self.capacity
}
pub(crate) fn push(&self, packet: SpawnPacket) -> std::result::Result<(), SpawnPacket> {
#[cfg(feature = "runtime-evidence")]
let record = Arc::clone(&packet.record);
let mut state = lock(&self.state);
if state.stopped || state.starts.len() >= self.capacity {
return Err(packet);
}
let was_empty = state.starts.is_empty();
state.starts.push_back(packet);
let depth = state.starts.len();
self.pending_starts.store(depth, Ordering::Release);
#[cfg(feature = "runtime-evidence")]
{
self.record_task_accepted(&record);
self.record_depth(depth);
}
drop(state);
#[cfg(test)]
let hook = lock(&self.before_notify_hook).take();
#[cfg(test)]
if let Some(hook) = hook {
hook();
}
if was_empty {
self.signal.notify();
} else {
self.signal.notify_if_waiting();
}
Ok(())
}
pub(crate) fn drain_into(&self, packets: &mut VecDeque<SpawnPacket>, limit: usize) -> usize {
if limit == 0 || self.pending_starts.load(Ordering::Acquire) == 0 {
return 0;
}
let mut state = lock(&self.state);
let initial_depth = state.starts.len();
let count = initial_depth.min(limit);
packets.extend(state.starts.drain(..count));
self.pending_starts
.store(state.starts.len(), Ordering::Release);
#[cfg(feature = "runtime-evidence")]
{
let remaining = state.starts.len();
for depth in (remaining..initial_depth).rev() {
self.record_depth(depth);
}
}
drop(state);
count
}
pub(crate) fn pop_scope(&self, scope: Option<u64>) -> Option<SpawnPacket> {
if self.pending_starts.load(Ordering::Acquire) == 0 {
return None;
}
let mut state = lock(&self.state);
let index = state
.starts
.iter()
.position(|packet| scope.is_none_or(|scope| packet.record.lock().scope == scope))?;
let packet = state.starts.remove(index);
let depth = state.starts.len();
self.pending_starts.store(depth, Ordering::Release);
#[cfg(feature = "runtime-evidence")]
self.record_depth(depth);
drop(state);
packet
}
pub(crate) fn cleanup_complete(&self) -> bool {
(!self.started.load(Ordering::Acquire)
|| (self.scheduler_stopped.load(Ordering::Acquire)
&& self.reclaimed.load(Ordering::Acquire)))
&& self.pending() == 0
}
pub(crate) fn pending(&self) -> usize {
self.pending_starts.load(Ordering::Acquire)
}
pub(crate) fn has_queued_starts_at_wait_boundary(&self) -> bool {
!lock(&self.state).starts.is_empty()
}
pub(crate) fn retire_tasks(&self, count: usize) {
self.retired_tasks.0.fetch_add(
u64::try_from(count).expect("completion batch fits u64"),
Ordering::Release,
);
}
pub(crate) fn retired_tasks(&self) -> u64 {
self.retired_tasks.0.load(Ordering::Acquire)
}
pub(crate) fn stop(&self) {
lock(&self.state).stopped = true;
self.stopped.store(true, Ordering::Release);
self.signal.notify();
}
pub(crate) fn stopped(&self) -> bool {
self.stopped.load(Ordering::Acquire)
}
pub(crate) fn abort(&self, scope: u64, reason: TaskFailure) {
let mut state = lock(&self.state);
state.abort.insert(scope, reason);
self.abort_pending.store(true, Ordering::Release);
drop(state);
self.signal.notify();
}
pub(crate) fn clear_abort(&self, scope: u64) {
let mut state = lock(&self.state);
state.abort.remove(&scope);
self.abort_pending
.store(!state.abort.is_empty(), Ordering::Release);
}
pub(crate) fn take_abort(&self) -> Option<(u64, TaskFailure)> {
if !self.abort_pending.load(Ordering::Acquire) {
return None;
}
let mut state = lock(&self.state);
let abort = state.abort.pop_first();
self.abort_pending
.store(!state.abort.is_empty(), Ordering::Release);
abort
}
#[cfg(feature = "runtime-evidence")]
fn record_depth(&self, depth: usize) {
if let Some(evidence) = &self.evidence {
evidence.record(crate::diagnostics::evidence::RuntimeEventKind::QueueDepth {
carrier: evidence.carrier(),
queue: crate::diagnostics::evidence::QueueKind::Start,
depth,
capacity: self.capacity,
});
}
}
#[cfg(feature = "runtime-evidence")]
fn record_task_accepted(&self, record: &SharedTaskRecord) {
let Some(evidence) = &self.evidence else {
return;
};
let record = record.lock();
evidence.record(
crate::diagnostics::evidence::RuntimeEventKind::TaskAccepted {
task: record.id,
scope: crate::diagnostics::ScopeId::new(record.scope),
parent: record.parent,
carrier: record.carrier,
},
);
}
}
#[cfg(test)]
#[path = "inbox_test.rs"]
mod inbox_test;