use std::collections::HashMap;
use std::path::Path;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Arc, Mutex, MutexGuard, Weak};
use std::thread::JoinHandle;
use std::time::Duration;
use astrs_shm::{
AttachOptions, Consumer, RecvError, Sample, Segment, SegmentClient, SegmentKey, StartPosition,
};
use astrs_wire::{DataId, DataflowId, Metadata, PortRef, ShmSegmentSpec, WireDecode};
use crate::error::{NodeError, Result};
use crate::events::{Event, QueuedEvent};
use crate::payload::Payload;
use crate::session::SessionShared;
use crate::session::routes::RoutePlane;
pub const RECV_SLICE: Duration = Duration::from_millis(25);
pub const MAX_CONSECUTIVE_LAGS: u32 = 8;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
#[non_exhaustive]
pub struct InputPlaneStats {
pub attaches: u64,
pub detaches: u64,
pub refusals: u64,
pub samples: u64,
pub lagged: u64,
}
#[derive(Debug)]
struct InputRoute {
plane: RoutePlane,
segment: Option<ShmSegmentSpec>,
reader: Option<ReaderHandle>,
retiring: Vec<ReaderHandle>,
epoch: u64,
}
impl InputRoute {
const fn daemon() -> Self {
Self {
plane: RoutePlane::Daemon,
segment: None,
reader: None,
retiring: Vec::new(),
epoch: 0,
}
}
}
#[derive(Debug)]
enum ClosureOutcome {
Taken,
AlreadyPending,
Finished(QueuedEvent),
}
#[derive(Debug, Default)]
struct ClosureSlot {
pending: Option<QueuedEvent>,
finished: bool,
}
#[derive(Debug)]
struct ReaderHandle {
stop: Arc<AtomicBool>,
drain: Arc<AtomicBool>,
closing: Arc<Mutex<ClosureSlot>>,
join: JoinHandle<()>,
}
impl ReaderHandle {
fn abort(&self) {
self.stop.store(true, Ordering::Release);
}
fn signal_drain(&self) {
self.drain.store(true, Ordering::Release);
}
fn take_closure(&self, event: QueuedEvent) -> ClosureOutcome {
let mut slot = lock(&self.closing);
if slot.finished {
return ClosureOutcome::Finished(event);
}
if slot.pending.is_some() {
return ClosureOutcome::AlreadyPending;
}
slot.pending = Some(event);
self.signal_drain();
ClosureOutcome::Taken
}
fn join(self) {
self.abort();
let _ = self.join.join();
}
}
#[derive(Debug, Default)]
pub struct InputRouteTable {
routes: Mutex<HashMap<DataId, InputRoute>>,
attaches: AtomicU64,
detaches: AtomicU64,
refusals: AtomicU64,
samples: AtomicU64,
lagged: AtomicU64,
}
impl InputRouteTable {
#[must_use]
pub fn new() -> Self {
Self {
routes: Mutex::new(HashMap::new()),
attaches: AtomicU64::new(0),
detaches: AtomicU64::new(0),
refusals: AtomicU64::new(0),
samples: AtomicU64::new(0),
lagged: AtomicU64::new(0),
}
}
#[must_use]
pub fn plane(&self, input: &DataId) -> RoutePlane {
lock(&self.routes)
.get(input)
.map_or(RoutePlane::Daemon, |route| route.plane)
}
#[must_use]
pub fn planes(&self) -> Vec<(DataId, RoutePlane)> {
let mut planes: Vec<(DataId, RoutePlane)> = lock(&self.routes)
.iter()
.map(|(id, route)| (id.clone(), route.plane))
.collect();
planes.sort_by(|left, right| left.0.cmp(&right.0));
planes
}
#[must_use]
pub fn segment(&self, input: &DataId) -> Option<ShmSegmentSpec> {
lock(&self.routes)
.get(input)
.and_then(|route| route.segment.clone())
}
#[must_use]
pub fn attached_count(&self) -> usize {
lock(&self.routes)
.values()
.filter(|route| route.plane.is_zero_copy())
.count()
}
#[must_use]
pub fn stats(&self) -> InputPlaneStats {
InputPlaneStats {
attaches: self.attaches.load(Ordering::Relaxed),
detaches: self.detaches.load(Ordering::Relaxed),
refusals: self.refusals.load(Ordering::Relaxed),
samples: self.samples.load(Ordering::Relaxed),
lagged: self.lagged.load(Ordering::Relaxed),
}
}
pub fn count_refusal(&self) {
self.refusals.fetch_add(1, Ordering::Relaxed);
}
pub fn attach(
&self,
shared: &Arc<SessionShared>,
input: &DataId,
source: &PortRef,
segment: &ShmSegmentSpec,
) -> Result<()> {
let retiring = {
let mut routes = lock(&self.routes);
let route = routes
.entry(input.clone())
.or_insert_with(InputRoute::daemon);
if route.plane.is_zero_copy()
&& route.segment.as_ref().is_some_and(|open| {
open.generation == segment.generation && open.name == segment.name
})
{
return Ok(());
}
if let Some(reader) = route.reader.take() {
reader.abort();
route.retiring.push(reader);
}
route.plane = RoutePlane::Daemon;
route.segment = None;
core::mem::take(&mut route.retiring)
};
for reader in retiring {
reader.join();
}
let (mapped, consumer) = open_consumer(
shared.dataflow,
source,
segment,
shared.shm_broker.as_deref(),
)?;
let stop = Arc::new(AtomicBool::new(false));
let drain = Arc::new(AtomicBool::new(false));
let closing: Arc<Mutex<ClosureSlot>> = Arc::new(Mutex::new(ClosureSlot::default()));
let epoch = {
let mut routes = lock(&self.routes);
let route = routes
.entry(input.clone())
.or_insert_with(InputRoute::daemon);
route.epoch = route.epoch.wrapping_add(1);
route.epoch
};
let reader = ReaderContext {
input: input.clone(),
source: source.clone(),
session: Arc::downgrade(shared),
stop: Arc::clone(&stop),
drain: Arc::clone(&drain),
closing: Arc::clone(&closing),
epoch,
};
let join = std::thread::Builder::new()
.name(format!("astrs-shm-{input}"))
.spawn(move || reader.run(consumer))
.map_err(|error| {
NodeError::Pattern(format!(
"could not start the ring reader for `{input}`: {error}"
))
})?;
let mut routes = lock(&self.routes);
let route = routes
.entry(input.clone())
.or_insert_with(InputRoute::daemon);
route.plane = RoutePlane::Shm;
route.segment = Some(segment.clone());
route.reader = Some(ReaderHandle {
stop,
drain,
closing,
join,
});
drop(mapped);
self.attaches.fetch_add(1, Ordering::Relaxed);
Ok(())
}
pub fn detach(&self, input: &DataId) -> bool {
let mut routes = lock(&self.routes);
let Some(route) = routes.get_mut(input) else {
return false;
};
let was_attached = route.plane.is_zero_copy();
route.plane = RoutePlane::Daemon;
route.segment = None;
route.epoch = route.epoch.wrapping_add(1);
if let Some(reader) = route.reader.take() {
reader.signal_drain();
route.retiring.push(reader);
}
if was_attached {
self.detaches.fetch_add(1, Ordering::Relaxed);
}
was_attached
}
pub fn close_input(&self, input: &DataId, event: QueuedEvent) -> Option<QueuedEvent> {
let mut routes = lock(&self.routes);
let Some(route) = routes.get_mut(input) else {
return Some(event);
};
if !route.plane.is_zero_copy() {
return Self::hand_to_retiring(route, event);
}
let Some(reader) = route.reader.take() else {
return Some(event);
};
let outcome = reader.take_closure(event);
route.plane = RoutePlane::Daemon;
route.segment = None;
route.epoch = route.epoch.wrapping_add(1);
route.retiring.push(reader);
self.detaches.fetch_add(1, Ordering::Relaxed);
match outcome {
ClosureOutcome::Taken | ClosureOutcome::AlreadyPending => None,
ClosureOutcome::Finished(event) => Some(event),
}
}
fn hand_to_retiring(route: &mut InputRoute, event: QueuedEvent) -> Option<QueuedEvent> {
let mut event = event;
for reader in route.retiring.iter().rev() {
match reader.take_closure(event) {
ClosureOutcome::Taken | ClosureOutcome::AlreadyPending => return None,
ClosureOutcome::Finished(returned) => event = returned,
}
}
Some(event)
}
pub fn drain_all(&self) -> usize {
let readers: Vec<ReaderHandle> = {
let mut routes = lock(&self.routes);
let mut readers = Vec::new();
for route in routes.values_mut() {
if route.plane.is_zero_copy() {
self.detaches.fetch_add(1, Ordering::Relaxed);
}
route.plane = RoutePlane::Daemon;
route.segment = None;
route.epoch = route.epoch.wrapping_add(1);
if let Some(reader) = route.reader.take() {
readers.push(reader);
}
readers.append(&mut route.retiring);
}
for reader in &readers {
reader.signal_drain();
}
readers
};
let waited = readers.len();
for reader in readers {
reader.join();
}
waited
}
pub fn close(&self) {
let readers: Vec<ReaderHandle> = {
let mut routes = lock(&self.routes);
let mut readers = Vec::new();
for route in routes.values_mut() {
if route.plane.is_zero_copy() {
self.detaches.fetch_add(1, Ordering::Relaxed);
}
route.plane = RoutePlane::Daemon;
route.segment = None;
route.epoch = route.epoch.wrapping_add(1);
if let Some(reader) = route.reader.take() {
readers.push(reader);
}
readers.append(&mut route.retiring);
}
for reader in &readers {
reader.abort();
}
readers
};
for reader in readers {
reader.join();
}
}
fn mark_detached(&self, input: &DataId, epoch: u64) {
let mut routes = lock(&self.routes);
let Some(route) = routes.get_mut(input) else {
return;
};
if route.epoch != epoch {
return;
}
if route.plane.is_zero_copy() {
self.detaches.fetch_add(1, Ordering::Relaxed);
}
route.plane = RoutePlane::Daemon;
route.segment = None;
}
fn count_sample(&self) {
self.samples.fetch_add(1, Ordering::Relaxed);
}
fn count_lagged(&self, missed: u64) {
self.lagged.fetch_add(missed, Ordering::Relaxed);
}
}
struct ReaderContext {
input: DataId,
source: PortRef,
session: Weak<SessionShared>,
stop: Arc<AtomicBool>,
drain: Arc<AtomicBool>,
closing: Arc<Mutex<ClosureSlot>>,
epoch: u64,
}
impl ReaderContext {
fn run(self, mut consumer: Consumer) {
let mut consecutive_lags = 0u32;
while !self.stop.load(Ordering::Acquire) {
let Some(session) = self.session.upgrade() else {
break;
};
if session.is_closed() {
break;
}
if self.drain.load(Ordering::Acquire) {
self.finish(&session, &mut consumer);
break;
}
match consumer.next_blocking(RECV_SLICE) {
Ok(sample) => {
consecutive_lags = 0;
self.deliver(&session, sample);
}
Err(RecvError::Empty) => {}
Err(RecvError::Lagged(missed)) => {
session.inputs.count_lagged(missed);
consecutive_lags += 1;
if consecutive_lags >= MAX_CONSECUTIVE_LAGS {
session.source.push_control(Event::Error(format!(
"input `{}` fell behind its ring {MAX_CONSECUTIVE_LAGS} times in a \
row; reading it through the daemon instead",
self.input
)));
break;
}
}
Err(RecvError::Closed) => break,
Err(error) => {
session.source.push_control(Event::Error(format!(
"input `{}` left the shared-memory plane: {error}",
self.input
)));
break;
}
}
}
drop(consumer);
let closure = {
let mut slot = lock(&self.closing);
slot.finished = true;
slot.pending.take()
};
if let Some(session) = self.session.upgrade() {
if let Some(closure) = closure {
session.source.push_input(&self.input, closure);
}
session.inputs.mark_detached(&self.input, self.epoch);
}
}
fn finish(&self, session: &Arc<SessionShared>, consumer: &mut Consumer) {
loop {
match consumer.try_next() {
Ok(sample) => self.deliver(session, sample),
Err(RecvError::Lagged(missed)) => session.inputs.count_lagged(missed),
Err(_) => break,
}
}
}
fn deliver(&self, session: &Arc<SessionShared>, sample: Sample) {
let metadata = slot_metadata(sample.metadata(), session);
let _accepted = session.observe_remote(metadata.timestamp);
let payload = Payload::zero_copy(sample);
session.source.push_input(
&self.input,
QueuedEvent::Input {
source: self.source.clone(),
metadata,
payload,
},
);
session.inputs.count_sample();
}
}
fn slot_metadata(bytes: &[u8], session: &Arc<SessionShared>) -> Metadata {
if bytes.is_empty() {
return Metadata::new(session.hlc_now());
}
Metadata::decode_exact(bytes).unwrap_or_else(|_| Metadata::new(session.hlc_now()))
}
pub fn open_consumer(
dataflow: DataflowId,
source: &PortRef,
segment: &ShmSegmentSpec,
broker: Option<&Path>,
) -> Result<(Arc<Segment>, Consumer)> {
let key = SegmentKey::new(
dataflow,
source.node().clone(),
source.port().clone(),
segment.generation,
);
let canonical = key.canonical();
if canonical != segment.name {
return Err(NodeError::Pattern(format!(
"input route upgrade names segment `{}`, but `{source}` at generation {} is \
`{canonical}`",
segment.name, segment.generation,
)));
}
let mut client = match broker {
Some(path) => Some(SegmentClient::connect(path)?),
None => None,
};
let mapped = match client.as_mut() {
Some(client) => client.attach(&key)?,
None => Segment::open_named(&key.os_name(), Some(&key))?,
}
.shared();
let consumer = Consumer::attach(
Arc::clone(&mapped),
AttachOptions::new()
.with_start(StartPosition::Latest)
.with_doorbell(true)
.expecting(key.clone()),
)?;
if let Some(client) = client.as_mut()
&& let Some(doorbell) = consumer.doorbell()
{
let ringer = doorbell.ringer()?;
if let Err(error) =
client.register_doorbell(&key, consumer.index(), consumer.token(), ringer)
{
tracing::debug!(%error, segment = %canonical, "the broker refused a doorbell");
}
}
Ok((mapped, consumer))
}
fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
mutex
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
use astrs_wire::{DataflowId, NodeId};
fn input() -> DataId {
DataId::new("frames").unwrap()
}
fn source() -> PortRef {
PortRef::from_parts("camera", "image").unwrap()
}
#[test]
fn a_fresh_input_is_on_the_daemon_path() {
let table = InputRouteTable::new();
assert_eq!(table.plane(&input()), RoutePlane::Daemon);
assert_eq!(table.segment(&input()), None);
assert_eq!(table.attached_count(), 0);
assert!(table.planes().is_empty());
assert_eq!(table.stats(), InputPlaneStats::default());
}
#[test]
fn detaching_an_unknown_input_is_a_no_op() {
let table = InputRouteTable::new();
assert!(!table.detach(&input()));
table.close();
assert_eq!(table.stats().detaches, 0);
}
#[test]
fn refusals_are_counted() {
let table = InputRouteTable::new();
table.count_refusal();
assert_eq!(table.stats().refusals, 1);
}
#[test]
fn a_segment_name_that_does_not_match_the_producer_is_refused() {
let spec = ShmSegmentSpec::new("astrs-not-this-segment", 3, 8, 4096);
let error = open_consumer(DataflowId::from_u128(1), &source(), &spec, None).unwrap_err();
assert!(matches!(error, NodeError::Pattern(_)), "{error}");
assert!(error.to_string().contains("camera/image"));
}
#[test]
fn the_canonical_producer_name_is_what_the_daemon_sends() {
let dataflow = DataflowId::from_u128(0x9001);
let key = SegmentKey::new(
dataflow,
NodeId::new("camera").unwrap(),
DataId::new("image").unwrap(),
5,
);
assert!(key.canonical().ends_with("/camera/image/5"));
let hashed = ShmSegmentSpec::new(key.os_name().as_str(), 5, 8, 4096);
assert!(open_consumer(dataflow, &source(), &hashed, None).is_err());
}
#[cfg(unix)]
#[test]
fn a_real_segment_is_opened_verified_and_read_from() {
use astrs_shm::{Backing, Producer, SegmentConfig};
let dataflow = DataflowId::from_u128((u128::from(std::process::id()) << 32) | 0x9002);
let key = SegmentKey::new(
dataflow,
NodeId::new("camera").unwrap(),
DataId::new("image").unwrap(),
1,
);
let config = SegmentConfig::new(4, 4096)
.unwrap()
.with_backing(Backing::Named);
let Ok(segment) = Segment::create(key.clone(), config) else {
return;
};
let segment = segment.shared();
let mut producer = Producer::new(Arc::clone(&segment)).unwrap();
let spec = ShmSegmentSpec::new(key.canonical(), 1, 4, 4096);
let (mapped, mut consumer) =
open_consumer(dataflow, &source(), &spec, None).expect("the segment opens");
assert_eq!(mapped.layout().slot_count(), 4);
producer.send(b"after", b"").unwrap();
let sample = consumer.next_blocking(Duration::from_secs(2)).unwrap();
assert_eq!(sample.payload(), b"after");
let stale = ShmSegmentSpec::new(key.canonical(), 2, 4, 4096);
assert!(open_consumer(dataflow, &source(), &stale, None).is_err());
drop(sample);
drop(consumer);
let _ = segment.unlink();
}
#[cfg(unix)]
fn session(
dataflow: DataflowId,
) -> (
Arc<SessionShared>,
tokio::sync::mpsc::Receiver<crate::session::Outgoing>,
) {
use astrs_wire::{FrameLimits, InputSpec, NodeSource, NodeSpawnSpec, SessionId};
let spec = Arc::new(
NodeSpawnSpec::new(
dataflow,
NodeId::new("detect").unwrap(),
0,
NodeSource::Dynamic,
)
.with_input(InputSpec::new(input(), source())),
);
let events = Arc::new(crate::events::EventSource::new());
for declared in &spec.inputs {
events.register_input(declared).unwrap();
}
let (sender, receiver) = tokio::sync::mpsc::channel(crate::session::OUTGOING_CAPACITY);
let shared = Arc::new(SessionShared::new(
spec,
SessionId::from_u128(11),
sender,
events,
crate::runtime::NodeRuntime::owned().unwrap(),
4096,
FrameLimits::uds(),
));
(shared, receiver)
}
#[cfg(unix)]
fn ring(
dataflow: DataflowId,
generation: u64,
) -> Option<(
SegmentKey,
Arc<Segment>,
astrs_shm::Producer,
ShmSegmentSpec,
)> {
use astrs_shm::{Backing, Producer, SegmentConfig};
let key = SegmentKey::new(
dataflow,
NodeId::new("camera").unwrap(),
DataId::new("image").unwrap(),
generation,
);
let config = SegmentConfig::new(4, 4096)
.unwrap()
.with_backing(Backing::Named);
let segment = Segment::create(key.clone(), config).ok()?.shared();
let producer = Producer::new(Arc::clone(&segment)).ok()?;
let spec = ShmSegmentSpec::new(key.canonical(), generation, 4, 4096);
Some((key, segment, producer, spec))
}
#[cfg(unix)]
#[test]
fn a_new_generation_replaces_an_attachment_rather_than_being_ignored() {
let dataflow = DataflowId::from_u128((u128::from(std::process::id()) << 32) | 0x9004);
let Some((_key_one, first, _producer_one, spec_one)) = ring(dataflow, 1) else {
return;
};
let Some((_key_two, second, mut producer_two, spec_two)) = ring(dataflow, 2) else {
let _ = first.unlink();
return;
};
let (shared, _outgoing) = session(dataflow);
shared
.inputs
.attach(&shared, &input(), &source(), &spec_one)
.expect("the first ring attaches");
assert_eq!(shared.inputs.plane(&input()), RoutePlane::Shm);
assert_eq!(
shared.inputs.segment(&input()).map(|s| s.generation),
Some(1)
);
assert_eq!(shared.inputs.stats().attaches, 1);
shared
.inputs
.attach(&shared, &input(), &source(), &spec_one)
.expect("a repeated offer is accepted");
assert_eq!(shared.inputs.stats().attaches, 1, "no second attachment");
shared
.inputs
.attach(&shared, &input(), &source(), &spec_two)
.expect("the second ring attaches");
assert_eq!(shared.inputs.stats().attaches, 2);
assert_eq!(
shared.inputs.segment(&input()).map(|s| s.generation),
Some(2)
);
assert_eq!(
shared.inputs.planes(),
vec![(input(), RoutePlane::Shm)],
"the plane is reported per input"
);
producer_two.send(b"second-incarnation", b"").unwrap();
let deadline = std::time::Instant::now() + Duration::from_secs(5);
let mut delivered = None;
while std::time::Instant::now() < deadline && delivered.is_none() {
if let Some(crate::events::Event::Input { id, data, .. }) = shared.source.try_next() {
assert_eq!(id, input());
delivered = Some(data);
} else {
std::thread::sleep(Duration::from_millis(5));
}
}
let payload = delivered.expect("a sample from the second ring");
assert!(payload.is_zero_copy());
assert_eq!(payload.bytes(), b"second-incarnation");
let slot = payload.slot().expect("a ring slot");
assert!(
slot.is_inside_mapping() && slot.is_at_layout_offset(),
"{slot:?}"
);
drop(payload);
assert!(shared.inputs.detach(&input()));
assert_eq!(shared.inputs.plane(&input()), RoutePlane::Daemon);
assert!(!shared.inputs.detach(&input()), "a detach happens once");
shared.inputs.close();
let _ = first.unlink();
let _ = second.unlink();
}
#[cfg(unix)]
#[test]
fn latest_skips_what_the_daemon_already_delivered() {
use astrs_shm::{Backing, Producer, SegmentConfig};
let dataflow = DataflowId::from_u128((u128::from(std::process::id()) << 32) | 0x9003);
let key = SegmentKey::new(
dataflow,
NodeId::new("camera").unwrap(),
DataId::new("image").unwrap(),
1,
);
let config = SegmentConfig::new(4, 4096)
.unwrap()
.with_backing(Backing::Named);
let Ok(segment) = Segment::create(key.clone(), config) else {
return;
};
let segment = segment.shared();
let mut producer = Producer::new(Arc::clone(&segment)).unwrap();
producer.send(b"before", b"").unwrap();
let spec = ShmSegmentSpec::new(key.canonical(), 1, 4, 4096);
let (_mapped, mut consumer) = open_consumer(dataflow, &source(), &spec, None).unwrap();
assert!(
matches!(consumer.try_next(), Err(RecvError::Empty)),
"a message published before the attach is not replayed"
);
producer.send(b"after", b"").unwrap();
let sample = consumer.next_blocking(Duration::from_secs(2)).unwrap();
assert_eq!(sample.payload(), b"after");
drop(sample);
drop(consumer);
let _ = segment.unlink();
}
}