Skip to main content

astrs_node_api/events/
source.rs

1//! [`EventSource`] — the shared state between the session's reader task and
2//! the node's [`EventStream`](super::EventStream).
3//!
4//! One `Arc<EventSource>` sits between them. The reader pushes; the stream
5//! pops; neither knows about the other. That separation is what lets the
6//! testing harness feed a stream without a socket, and what lets a node drop
7//! its stream (which the session notices and reports to the daemon as
8//! [`astrs_wire::NodeRequest::EventStreamDropped`], §7.3) without racing the
9//! reader.
10//!
11//! # Priority and fairness
12//!
13//! [`EventSource::try_next`] drains the control deque first and only then
14//! asks the [`EventMux`] for data — blueprint §11.3's "control lane
15//! pre-empts data lane", applied at the one point where both lanes meet.
16//! Within the data lane the mux's own round robin keeps a hot input from
17//! starving a quiet one.
18//!
19//! # Counters
20//!
21//! Every drop is counted, and every drop *signal* from the queue is turned
22//! into an [`Event::Error`] on the control lane, because blueprint §11.2 says
23//! a backpressure exhaustion is an ERROR-log-and-metric event rather than a
24//! silent loss. It is also written to the process log as a
25//! `tracing::error!` — the control-lane `Event::Error` only reaches a node
26//! that is reading its own event stream, while the process log reaches
27//! whatever is watching `astrs/logs/*` or stderr — throttled to once per
28//! input per `DROP_LOG_WINDOW` rather than once per dropped message: see
29//! `EventSource::log_drop_signal`, this module's own (private) rate limiter.
30//!
31//! # Deadlines (§11.3)
32//!
33//! An input whose [`InputSpec::deadline`] is set gets an
34//! [`astrs_scheduler::DeadlineMonitor`] entry alongside its queue
35//! registration. Every *data* event [`EventSource::try_next`] hands to the
36//! node opens a measurement for that input, superseding (not queuing behind)
37//! any measurement already open for the same input — the newest arrival is
38//! what a node's next publish is judged against, the same "the newest state
39//! is what matters" reasoning [`QueuePolicy::DropOldest`] applies to the
40//! queue itself, and what keeps the open set bounded by the node's own input
41//! count rather than by how long it has been running. [`EventSource::finish_deadlines`]
42//! is the other half: called once per publish (see
43//! [`crate::session::SessionShared::send_request`]), it closes every open
44//! measurement against `now` and turns a [`astrs_scheduler::DeadlineOutcome::Violated`]
45//! into an `Event::Error` on the control lane — `describe_signal`'s §11.2
46//! pattern, mirrored for §11.3.
47
48use std::collections::{HashMap, VecDeque};
49use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
50use std::sync::{Mutex, MutexGuard};
51use std::time::{Duration, Instant};
52
53use astrs_scheduler::{
54    DeadlineMonitor, DeadlineOutcome, DeadlineToken, EventMux, InputHandle, PushOutcome,
55    QueueSignal, QueueSnapshot,
56};
57use astrs_wire::{DataId, InputSpec, PriorityLane, QueuePolicy};
58
59use crate::error::Result;
60use crate::events::{Event, QueuedEvent};
61use crate::signal::Signal;
62
63/// How often, per input, a queue drop signal is allowed to reach the process
64/// log as a `tracing::error!` (§11.2).
65///
66/// The in-process [`Event::Error`] pushed alongside it is unthrottled — a
67/// node's own event stream is a bounded channel the node already reads at
68/// its own pace, and every entry on it is one already-accounted-for drop,
69/// not a new one. The process log is different: a full stack trace of
70/// `ERROR` lines, one per dropped message, during a sustained overload would
71/// itself become an unbounded-write problem sitting right behind the one it
72/// was reporting.
73const DROP_LOG_WINDOW: Duration = Duration::from_secs(5);
74
75/// Counters a node (and `astrs list`) can read off a live stream.
76#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
77#[non_exhaustive]
78pub struct StreamStats {
79    /// Events delivered to the node.
80    pub delivered: u64,
81    /// Data-lane events dropped by a queue policy.
82    pub dropped: u64,
83    /// Control-lane events currently waiting.
84    pub control_depth: u64,
85    /// Times a queue reported a condition worth logging (§11.2).
86    pub queue_signals: u64,
87    /// Times an input's `InputSpec::deadline` was exceeded (§11.3).
88    pub deadline_violations: u64,
89}
90
91/// The shared inbox behind an [`EventStream`](super::EventStream).
92pub struct EventSource {
93    /// The per-input bounded queues, priority-and-fairness multiplexed.
94    mux: EventMux<QueuedEvent>,
95    /// Producer-side handles, one per registered input.
96    inputs: Mutex<HashMap<DataId, InputHandle<QueuedEvent>>>,
97    /// The unbounded control lane.
98    control: Mutex<VecDeque<Event>>,
99    /// Wakes both blocking and async waiters.
100    signal: Signal,
101    /// Set when the session ends; no further events can arrive.
102    closed: AtomicBool,
103    /// Set when the node dropped its stream.
104    abandoned: AtomicBool,
105    /// Delivered counter.
106    delivered: AtomicU64,
107    /// Dropped counter.
108    dropped: AtomicU64,
109    /// Queue-signal counter.
110    signals: AtomicU64,
111    /// Run once when the node drops its stream (§7.3 `EventStreamDropped`).
112    abandon_hook: Mutex<Option<AbandonHook>>,
113    /// Per-input latency budgets declared via [`InputSpec::deadline`]
114    /// (§11.3). Registration is a no-op for an input with none — see
115    /// [`astrs_scheduler::DeadlineMonitor::register_from_spec`].
116    deadlines: DeadlineMonitor<DataId>,
117    /// The measurement currently open for each input that has one, keyed by
118    /// input — see this module's docs on why a later arrival supersedes
119    /// rather than queues behind an earlier one.
120    open_deadlines: Mutex<HashMap<DataId, DeadlineToken<DataId>>>,
121    /// Deadline-violation counter.
122    deadline_violations: AtomicU64,
123    /// The last instant each input's queue-drop signal actually reached the
124    /// process log, for [`DROP_LOG_WINDOW`] throttling — see
125    /// [`EventSource::log_drop_signal`].
126    last_drop_log: Mutex<HashMap<DataId, Instant>>,
127    /// Run once per violation [`EventSource::finish_deadlines`] finds
128    /// (§11.3), so the session can relay it to the daemon as
129    /// [`astrs_wire::NodeRequest::ReportDeadlineViolation`] — the
130    /// [`AbandonHook`] pattern, for the daemon-relay half of a deadline
131    /// violation rather than its local [`Event::Error`], which
132    /// `finish_deadlines` still pushes unconditionally.
133    deadline_violation_hook: Mutex<Option<DeadlineViolationHook>>,
134}
135
136/// What runs when a node drops its event stream.
137///
138/// The session installs one that sends
139/// [`astrs_wire::NodeRequest::EventStreamDropped`], which is what stops the
140/// daemon growing a queue for a node that stopped reading — dora's lesson,
141/// kept (§7.3).
142pub type AbandonHook = Box<dyn Fn() + Send + Sync>;
143
144/// What runs when [`EventSource::finish_deadlines`] finds a violation
145/// (§11.3): the input it was on, the budget, and the measured latency —
146/// everything [`astrs_wire::NodeRequest::ReportDeadlineViolation`] needs.
147///
148/// Not installed by default: without a hook, a violation is still counted
149/// ([`EventSource::stats`]) and still reported on the node's own control
150/// lane via [`Event::Error`] — this hook is *only* the daemon-relay half
151/// (§11.3's `astrs/status` fan-out and registered metric), which needs a live
152/// session to actually send anything over.
153pub type DeadlineViolationHook = Box<dyn Fn(DataId, Duration, Duration) + Send + Sync>;
154
155// Hand-written, and deliberately a summary: a payload can be megabytes, so a
156// `Debug` that tried to render queued events would be unusable exactly when a
157// node is being debugged. What a caller needs is which inputs exist, what the
158// counters say, and whether the session is still live.
159impl core::fmt::Debug for EventSource {
160    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
161        f.debug_struct("EventSource")
162            .field("inputs", &self.input_ids())
163            .field("stats", &self.stats())
164            .field("closed", &self.is_closed())
165            .field("abandoned", &self.is_abandoned())
166            .finish_non_exhaustive()
167    }
168}
169
170impl Default for EventSource {
171    fn default() -> Self {
172        Self::new()
173    }
174}
175
176impl EventSource {
177    /// An empty source with no inputs registered.
178    #[must_use]
179    pub fn new() -> Self {
180        Self {
181            mux: EventMux::new(),
182            inputs: Mutex::new(HashMap::new()),
183            control: Mutex::new(VecDeque::new()),
184            signal: Signal::new(),
185            closed: AtomicBool::new(false),
186            abandoned: AtomicBool::new(false),
187            delivered: AtomicU64::new(0),
188            dropped: AtomicU64::new(0),
189            signals: AtomicU64::new(0),
190            abandon_hook: Mutex::new(None),
191            deadlines: DeadlineMonitor::new(),
192            open_deadlines: Mutex::new(HashMap::new()),
193            deadline_violations: AtomicU64::new(0),
194            last_drop_log: Mutex::new(HashMap::new()),
195            deadline_violation_hook: Mutex::new(None),
196        }
197    }
198
199    /// Installs the hook that runs when the node drops its stream.
200    ///
201    /// Replaces any previous one; the session sets exactly one.
202    pub fn set_abandon_hook(&self, hook: AbandonHook) {
203        *lock(&self.abandon_hook) = Some(hook);
204    }
205
206    /// Installs the hook that runs once per deadline violation
207    /// [`EventSource::finish_deadlines`] finds (§11.3).
208    ///
209    /// Replaces any previous one; the session sets exactly one — see
210    /// [`DeadlineViolationHook`].
211    pub fn set_deadline_violation_hook(&self, hook: DeadlineViolationHook) {
212        *lock(&self.deadline_violation_hook) = Some(hook);
213    }
214
215    /// The wakeup both faces of the stream wait on.
216    #[must_use]
217    pub const fn signal(&self) -> &Signal {
218        &self.signal
219    }
220
221    /// Registers an input with the queue size, policy and lane its manifest
222    /// declares (§11.2, §11.3).
223    ///
224    /// Also registers `spec`'s input-to-output latency budget, when it
225    /// declares one ([`InputSpec::deadline`]) — a no-op, harmlessly, for the
226    /// common case of an input with none.
227    ///
228    /// # Errors
229    ///
230    /// [`crate::NodeError::Scheduler`] when the input is already registered or the
231    /// declared capacity is zero.
232    pub fn register_input(&self, spec: &InputSpec) -> Result<()> {
233        let handle = self.mux.register_from_spec(spec)?;
234        let _previous = lock(&self.inputs).insert(spec.id.clone(), handle);
235        let _has_deadline = self.deadlines.register_from_spec(spec.id.clone(), spec);
236        Ok(())
237    }
238
239    /// Registers an input explicitly, for a dynamic node with no manifest
240    /// specification to read from.
241    ///
242    /// # Errors
243    ///
244    /// As [`EventSource::register_input`].
245    pub fn register_raw(
246        &self,
247        id: DataId,
248        capacity: u32,
249        policy: QueuePolicy,
250        lane: PriorityLane,
251    ) -> Result<()> {
252        let handle = self
253            .mux
254            .register_input(id.clone(), capacity, policy, lane)?;
255        let _previous = lock(&self.inputs).insert(id, handle);
256        Ok(())
257    }
258
259    /// Whether `id` is a registered input.
260    #[must_use]
261    pub fn has_input(&self, id: &DataId) -> bool {
262        lock(&self.inputs).contains_key(id)
263    }
264
265    /// The registered input ids.
266    #[must_use]
267    pub fn input_ids(&self) -> Vec<DataId> {
268        let mut ids: Vec<DataId> = lock(&self.inputs).keys().cloned().collect();
269        ids.sort_unstable();
270        ids
271    }
272
273    /// A point-in-time reading of one input's queue.
274    #[must_use]
275    pub fn queue_snapshot(&self, id: &DataId) -> Option<QueueSnapshot> {
276        self.mux.queue_snapshot(id)
277    }
278
279    /// Pushes a data-lane event onto `id`'s queue, applying its policy.
280    ///
281    /// An unregistered input is reported on the control lane rather than
282    /// silently dropped: it means the daemon and the node disagree about the
283    /// wiring, which an operator needs to see.
284    pub fn push_input(&self, id: &DataId, event: QueuedEvent) {
285        let handle = lock(&self.inputs).get(id).cloned();
286        let Some(handle) = handle else {
287            self.push_control(Event::Error(format!(
288                "message for `{id}`, which this node does not declare as an input"
289            )));
290            return;
291        };
292        let report = handle.push(event);
293        if report.outcome == PushOutcome::DroppedIncoming
294            || report.outcome == PushOutcome::EnqueuedEvicting
295        {
296            self.dropped.fetch_add(1, Ordering::Relaxed);
297        }
298        if let Some(signal) = report.signal {
299            self.signals.fetch_add(1, Ordering::Relaxed);
300            self.push_control(Event::Error(describe_signal(id, signal)));
301            self.log_drop_signal(id, signal, Instant::now());
302            return;
303        }
304        self.signal.notify();
305    }
306
307    /// Writes `signal` to the process log as a `tracing::error!` (blueprint
308    /// §11.2: "a backpressure exhaustion is an ERROR-log-and-metric event
309    /// rather than a silent loss"), at most once per `id` per
310    /// [`DROP_LOG_WINDOW`].
311    ///
312    /// Rate-limited by input, not skipped after the first: a queue that goes
313    /// quiet for a while and then starts dropping again is a *new*
314    /// occurrence an operator needs to see, not a continuation of the first
315    /// one — throttling is only about not writing one line per message
316    /// during a single sustained overload.
317    fn log_drop_signal(&self, id: &DataId, signal: QueueSignal, now: Instant) {
318        {
319            let mut last = lock(&self.last_drop_log);
320            let elapsed_enough = match last.get(id) {
321                Some(&previous) => {
322                    now.checked_duration_since(previous)
323                        .unwrap_or(Duration::ZERO)
324                        >= DROP_LOG_WINDOW
325                }
326                None => true,
327            };
328            if !elapsed_enough {
329                return;
330            }
331            last.insert(id.clone(), now);
332        }
333        let message = describe_signal(id, signal);
334        tracing::error!(input = %id, "{message}");
335    }
336
337    /// Pushes a control-lane event, which is delivered before any data.
338    pub fn push_control(&self, event: Event) {
339        lock(&self.control).push_back(event);
340        self.signal.notify();
341    }
342
343    /// Marks the source finished: no further events will arrive, and waiting
344    /// consumers wake immediately.
345    ///
346    /// Idempotent.
347    pub fn close(&self) {
348        self.closed.store(true, Ordering::Release);
349        self.signal.close();
350    }
351
352    /// Whether the source has been closed.
353    #[must_use]
354    pub fn is_closed(&self) -> bool {
355        self.closed.load(Ordering::Acquire)
356    }
357
358    /// Records that the node dropped its event stream (§7.3).
359    ///
360    /// Idempotent: the hook runs once, however many times a stream is
361    /// abandoned.
362    pub fn abandon(&self) {
363        if self.abandoned.swap(true, Ordering::AcqRel) {
364            return;
365        }
366        self.signal.notify();
367        let hook = lock(&self.abandon_hook).take();
368        if let Some(hook) = hook {
369            hook();
370        }
371    }
372
373    /// Whether the node dropped its event stream.
374    #[must_use]
375    pub fn is_abandoned(&self) -> bool {
376        self.abandoned.load(Ordering::Acquire)
377    }
378
379    /// Takes the next event, control lane first, without waiting.
380    #[must_use]
381    pub fn try_next(&self) -> Option<Event> {
382        if let Some(event) = lock(&self.control).pop_front() {
383            self.delivered.fetch_add(1, Ordering::Relaxed);
384            return Some(event);
385        }
386        let (id, queued) = self.mux.try_recv()?;
387        self.delivered.fetch_add(1, Ordering::Relaxed);
388        // Only actual data starts an input-to-output measurement (§11.3) —
389        // `Closed`/`Recovered` are wind-down signals, not something a node
390        // is expected to answer with a publish.
391        if queued.is_message() {
392            self.start_deadline(&id, Instant::now());
393        }
394        Some(queued.into_event(id))
395    }
396
397    /// Opens an input-to-output latency measurement for `id`, if it has a
398    /// declared budget — a no-op otherwise. Supersedes (drops) whatever
399    /// measurement was already open for `id`; see this module's docs.
400    fn start_deadline(&self, id: &DataId, now: Instant) {
401        let Some(token) = self.deadlines.start(id, now) else {
402            return;
403        };
404        let _superseded = lock(&self.open_deadlines).insert(id.clone(), token);
405    }
406
407    /// Closes every open input-to-output measurement against `now` (§11.3),
408    /// counting and reporting each one that ran over budget.
409    ///
410    /// Called once per publish — see [`crate::session::SessionShared::send_request`]
411    /// — so the deadline check runs at the same cadence the daemon's own
412    /// `sends_inline`/`sends_zero_copy` counters do. A node with nothing open
413    /// (no deadline-tracked input has delivered since the last publish, or
414    /// none is configured at all) does nothing here.
415    pub fn finish_deadlines(&self, now: Instant) {
416        let open: Vec<DeadlineToken<DataId>> = {
417            let mut guard = lock(&self.open_deadlines);
418            guard.drain().map(|(_, token)| token).collect()
419        };
420        for token in open {
421            let id = token.key().clone();
422            if let DeadlineOutcome::Violated {
423                latency,
424                budget,
425                over_by,
426                ..
427            } = self.deadlines.finish(token, now)
428            {
429                self.deadline_violations.fetch_add(1, Ordering::Relaxed);
430                self.push_control(Event::Error(describe_deadline_violation(
431                    &id, budget, latency, over_by,
432                )));
433                if let Some(hook) = lock(&self.deadline_violation_hook).as_ref() {
434                    hook(id, budget, latency);
435                }
436            }
437        }
438    }
439
440    /// Whether anything is waiting to be delivered right now.
441    #[must_use]
442    pub fn is_empty(&self) -> bool {
443        lock(&self.control).is_empty() && self.mux.snapshot_all().iter().all(|(_, q)| q.depth == 0)
444    }
445
446    /// A snapshot of the stream's counters.
447    #[must_use]
448    pub fn stats(&self) -> StreamStats {
449        StreamStats {
450            delivered: self.delivered.load(Ordering::Relaxed),
451            dropped: self.dropped.load(Ordering::Relaxed),
452            control_depth: lock(&self.control).len() as u64,
453            queue_signals: self.signals.load(Ordering::Relaxed),
454            deadline_violations: self.deadline_violations.load(Ordering::Relaxed),
455        }
456    }
457}
458
459/// Renders a queue signal as the ERROR-lane message §11.2 asks for.
460fn describe_signal(id: &DataId, signal: QueueSignal) -> String {
461    match signal {
462        QueueSignal::BackpressureExhausted {
463            queue_size,
464            buffered,
465        } => format!(
466            "input `{id}`: backpressure buffer exhausted at {buffered} message(s) \
467             (queue_size {queue_size}); dropping"
468        ),
469        QueueSignal::ImmuneOverflow {
470            immune_count,
471            capacity,
472        } => format!(
473            "input `{id}`: {immune_count} eviction-immune message(s) over a capacity of \
474             {capacity}; the peer is not consuming its correlated replies"
475        ),
476    }
477}
478
479/// Renders a deadline violation as the ERROR-lane message §11.3 asks for —
480/// [`describe_signal`]'s §11.2 pattern, mirrored.
481fn describe_deadline_violation(
482    id: &DataId,
483    budget: std::time::Duration,
484    latency: std::time::Duration,
485    over_by: std::time::Duration,
486) -> String {
487    format!(
488        "input `{id}`: exceeded its {}ms input-to-output deadline — took {}ms ({}ms over)",
489        budget.as_millis(),
490        latency.as_millis(),
491        over_by.as_millis(),
492    )
493}
494
495/// Locks a mutex, recovering from a poisoning panic elsewhere.
496///
497/// Both guarded values are small collections with no multi-step invariant, so
498/// discarding them because an unrelated thread unwound would turn one bug into
499/// a wedged event loop — the same reasoning `astrs-scheduler` documents for
500/// its own locks.
501fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
502    mutex
503        .lock()
504        .unwrap_or_else(std::sync::PoisonError::into_inner)
505}
506
507#[cfg(test)]
508mod tests {
509    #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
510
511    use std::time::Duration;
512
513    use super::*;
514    use astrs_time::HlcTimestamp;
515    use astrs_wire::{DurationMs, Metadata, PortRef, RouteCloseReason, StopCause};
516
517    fn spec_with_deadline(name: &str, deadline_ms: Option<u64>) -> InputSpec {
518        let mut spec = InputSpec::new(
519            DataId::new(name).unwrap(),
520            PortRef::from_parts("camera", "image").unwrap(),
521        );
522        spec.deadline = deadline_ms.map(DurationMs::new);
523        spec
524    }
525
526    fn source_with_input(capacity: u32, policy: QueuePolicy) -> EventSource {
527        let source = EventSource::new();
528        source
529            .register_raw(
530                DataId::new("frames").unwrap(),
531                capacity,
532                policy,
533                PriorityLane::Data,
534            )
535            .unwrap();
536        source
537    }
538
539    fn message(seq: i64) -> QueuedEvent {
540        let mut metadata = Metadata::new(HlcTimestamp::new(1, 0));
541        metadata.set_seq(seq);
542        QueuedEvent::Input {
543            source: PortRef::from_parts("camera", "image").unwrap(),
544            metadata,
545            payload: Payload::inline(vec![0; 4]),
546        }
547    }
548
549    use crate::payload::Payload;
550
551    #[test]
552    fn events_come_out_in_order() {
553        let source = source_with_input(8, QueuePolicy::DropOldest);
554        let id = DataId::new("frames").unwrap();
555        for seq in 0..3 {
556            source.push_input(&id, message(seq));
557        }
558        for seq in 0..3 {
559            let event = source.try_next().unwrap();
560            assert_eq!(event.metadata().and_then(Metadata::seq), Some(seq));
561        }
562        assert!(source.try_next().is_none());
563        assert!(source.is_empty());
564        assert_eq!(source.stats().delivered, 3);
565    }
566
567    #[test]
568    fn the_control_lane_pre_empts_a_full_data_queue() {
569        let source = source_with_input(16, QueuePolicy::DropOldest);
570        let id = DataId::new("frames").unwrap();
571        for seq in 0..10 {
572            source.push_input(&id, message(seq));
573        }
574        source.push_control(Event::Stop(StopCause::Requested));
575        let event = source.try_next().unwrap();
576        assert!(event.is_stop(), "control first, whatever the backlog");
577    }
578
579    #[test]
580    fn drop_oldest_keeps_the_newest_message() {
581        let source = source_with_input(2, QueuePolicy::DropOldest);
582        let id = DataId::new("frames").unwrap();
583        for seq in 0..5 {
584            source.push_input(&id, message(seq));
585        }
586        let seqs: Vec<i64> = std::iter::from_fn(|| source.try_next())
587            .filter_map(|event| event.metadata().and_then(Metadata::seq))
588            .collect();
589        assert_eq!(seqs, vec![3, 4], "the two newest survive");
590        assert!(source.stats().dropped >= 3);
591    }
592
593    #[test]
594    fn correlated_messages_are_never_the_ones_dropped() {
595        let source = source_with_input(2, QueuePolicy::DropOldest);
596        let id = DataId::new("frames").unwrap();
597
598        let mut correlated = Metadata::new(HlcTimestamp::new(1, 0));
599        correlated.set_request_id("req-1");
600        correlated.set_seq(99);
601        source.push_input(
602            &id,
603            QueuedEvent::Input {
604                source: PortRef::from_parts("camera", "image").unwrap(),
605                metadata: correlated,
606                payload: Payload::empty(),
607            },
608        );
609        for seq in 0..6 {
610            source.push_input(&id, message(seq));
611        }
612        let seqs: Vec<i64> = std::iter::from_fn(|| source.try_next())
613            .filter_map(|event| event.metadata().and_then(Metadata::seq))
614            .collect();
615        assert!(
616            seqs.contains(&99),
617            "the correlated message survived: {seqs:?}"
618        );
619    }
620
621    #[test]
622    fn backpressure_exhaustion_is_reported_on_the_control_lane() {
623        let source = source_with_input(1, QueuePolicy::Backpressure);
624        let id = DataId::new("frames").unwrap();
625        // Ten times `queue_size` is the buffer; the eleventh push drops.
626        for seq in 0..12 {
627            source.push_input(&id, message(seq));
628        }
629        let mut errors = 0;
630        while let Some(event) = source.try_next() {
631            if matches!(event, Event::Error(_)) {
632                errors += 1;
633            }
634        }
635        assert!(errors > 0, "an exhausted buffer is an ERROR event");
636        assert!(source.stats().queue_signals > 0);
637    }
638
639    // A minimal `tracing::Subscriber` that records every event's level and
640    // rendered message — enough to assert a `tracing::error!` actually fired
641    // (and how many times), without a dev-dependency this crate does not
642    // otherwise need.
643    #[derive(Default)]
644    struct CaptureSubscriber {
645        events: Mutex<Vec<(tracing::Level, String)>>,
646    }
647
648    impl CaptureSubscriber {
649        fn events(&self) -> Vec<(tracing::Level, String)> {
650            lock(&self.events).clone()
651        }
652
653        fn error_count(&self) -> usize {
654            lock(&self.events)
655                .iter()
656                .filter(|(level, _)| *level == tracing::Level::ERROR)
657                .count()
658        }
659    }
660
661    #[derive(Default)]
662    struct MessageVisitor(String);
663
664    impl tracing::field::Visit for MessageVisitor {
665        fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
666            if field.name() == "message" {
667                self.0 = format!("{value:?}");
668            }
669        }
670    }
671
672    impl tracing::Subscriber for CaptureSubscriber {
673        fn enabled(&self, _metadata: &tracing::Metadata<'_>) -> bool {
674            true
675        }
676
677        fn new_span(&self, _span: &tracing::span::Attributes<'_>) -> tracing::span::Id {
678            tracing::span::Id::from_u64(1)
679        }
680
681        fn record(&self, _span: &tracing::span::Id, _values: &tracing::span::Record<'_>) {}
682
683        fn record_follows_from(&self, _span: &tracing::span::Id, _follows: &tracing::span::Id) {}
684
685        fn event(&self, event: &tracing::Event<'_>) {
686            let mut visitor = MessageVisitor::default();
687            event.record(&mut visitor);
688            lock(&self.events).push((*event.metadata().level(), visitor.0));
689        }
690
691        fn enter(&self, _span: &tracing::span::Id) {}
692        fn exit(&self, _span: &tracing::span::Id) {}
693    }
694
695    #[test]
696    fn a_backpressure_signal_logs_an_error_once_per_window_not_per_message() {
697        let source = source_with_input(1, QueuePolicy::Backpressure);
698        let id = DataId::new("frames").unwrap();
699        let subscriber = std::sync::Arc::new(CaptureSubscriber::default());
700        tracing::subscriber::with_default(subscriber.clone(), || {
701            // Far more than the ten-times-`queue_size` buffer, so many drops
702            // happen in this one burst — all within `DROP_LOG_WINDOW`.
703            for seq in 0..30 {
704                source.push_input(&id, message(seq));
705            }
706        });
707        assert_eq!(
708            subscriber.error_count(),
709            1,
710            "one ERROR per window, not one per dropped message: {:?}",
711            subscriber.events()
712        );
713        let (_, message) = &subscriber.events()[0];
714        assert!(message.contains("frames"), "{message}");
715    }
716
717    #[test]
718    fn an_input_within_budget_never_logs_an_error() {
719        let source = source_with_input(8, QueuePolicy::DropOldest);
720        let id = DataId::new("frames").unwrap();
721        let subscriber = std::sync::Arc::new(CaptureSubscriber::default());
722        tracing::subscriber::with_default(subscriber.clone(), || {
723            source.push_input(&id, message(0));
724        });
725        assert_eq!(subscriber.error_count(), 0, "{:?}", subscriber.events());
726    }
727
728    #[test]
729    fn the_drop_log_is_rate_limited_per_input_independently() {
730        let a = DataId::new("a").unwrap();
731        let b = DataId::new("b").unwrap();
732        let source = EventSource::new();
733        let signal = QueueSignal::BackpressureExhausted {
734            queue_size: 1,
735            buffered: 10,
736        };
737        let subscriber = std::sync::Arc::new(CaptureSubscriber::default());
738        tracing::subscriber::with_default(subscriber.clone(), || {
739            let t0 = Instant::now();
740            source.log_drop_signal(&a, signal, t0);
741            source.log_drop_signal(&a, signal, t0); // rate-limited: same input, same instant
742            source.log_drop_signal(&b, signal, t0); // a different input has its own window
743        });
744        assert_eq!(subscriber.error_count(), 2, "{:?}", subscriber.events());
745    }
746
747    #[test]
748    fn the_drop_log_fires_again_once_the_window_elapses() {
749        let id = DataId::new("frames").unwrap();
750        let source = EventSource::new();
751        let signal = QueueSignal::BackpressureExhausted {
752            queue_size: 1,
753            buffered: 10,
754        };
755        let subscriber = std::sync::Arc::new(CaptureSubscriber::default());
756        tracing::subscriber::with_default(subscriber.clone(), || {
757            let t0 = Instant::now();
758            source.log_drop_signal(&id, signal, t0);
759            // Still inside the window: rate-limited.
760            source.log_drop_signal(&id, signal, t0 + DROP_LOG_WINDOW - Duration::from_millis(1));
761            // The window has fully elapsed: a fresh occurrence, logged again.
762            source.log_drop_signal(&id, signal, t0 + DROP_LOG_WINDOW);
763        });
764        assert_eq!(subscriber.error_count(), 2, "{:?}", subscriber.events());
765    }
766
767    #[test]
768    fn a_message_for_an_unknown_input_is_reported_not_dropped() {
769        let source = EventSource::new();
770        source.push_input(&DataId::new("nope").unwrap(), message(0));
771        let event = source.try_next().unwrap();
772        let Event::Error(message) = event else {
773            panic!("expected an error event");
774        };
775        assert!(message.contains("nope"), "{message}");
776    }
777
778    #[test]
779    fn wind_down_events_keep_their_place_behind_queued_frames() {
780        let source = source_with_input(8, QueuePolicy::DropOldest);
781        let id = DataId::new("frames").unwrap();
782        source.push_input(&id, message(1));
783        source.push_input(
784            &id,
785            QueuedEvent::Closed {
786                source: PortRef::from_parts("camera", "image").unwrap(),
787                reason: RouteCloseReason::ProducerFinished,
788            },
789        );
790        assert!(
791            source.try_next().unwrap().is_input(),
792            "the frame comes first"
793        );
794        assert!(matches!(
795            source.try_next().unwrap(),
796            Event::InputClosed { .. }
797        ));
798    }
799
800    #[test]
801    fn registration_is_reported_and_queryable() {
802        let source = EventSource::new();
803        let spec = InputSpec::new(
804            DataId::new("frames").unwrap(),
805            PortRef::from_parts("camera", "image").unwrap(),
806        );
807        source.register_input(&spec).unwrap();
808        assert!(source.has_input(&spec.id));
809        assert_eq!(source.input_ids(), vec![spec.id.clone()]);
810        assert!(source.queue_snapshot(&spec.id).is_some());
811        assert!(
812            source.register_input(&spec).is_err(),
813            "no double registration"
814        );
815        assert!(!source.has_input(&DataId::new("other").unwrap()));
816    }
817
818    #[test]
819    fn the_abandon_hook_runs_exactly_once() {
820        use std::sync::Arc;
821        use std::sync::atomic::AtomicUsize;
822
823        let source = EventSource::new();
824        let calls = Arc::new(AtomicUsize::new(0));
825        let counter = Arc::clone(&calls);
826        source.set_abandon_hook(Box::new(move || {
827            let _ = counter.fetch_add(1, Ordering::Relaxed);
828        }));
829        source.abandon();
830        source.abandon();
831        assert_eq!(calls.load(Ordering::Relaxed), 1);
832    }
833
834    #[test]
835    fn closing_and_abandoning_are_observable_and_idempotent() {
836        let source = EventSource::new();
837        assert!(!source.is_closed());
838        assert!(!source.is_abandoned());
839        source.abandon();
840        assert!(source.is_abandoned());
841        source.close();
842        assert!(source.is_closed());
843        source.close();
844        assert!(source.is_closed());
845        assert!(source.signal().is_closed());
846    }
847
848    // ------------------------------------------------------ deadlines (§11.3)
849
850    #[test]
851    fn an_input_with_no_declared_deadline_never_reports_a_violation() {
852        let source = EventSource::new();
853        let spec = spec_with_deadline("frames", None);
854        source.register_input(&spec).unwrap();
855        source.push_input(&spec.id, message(0));
856        assert!(source.try_next().unwrap().is_input());
857
858        // Even an absurdly-late "publish" has no budget to have missed.
859        source.finish_deadlines(Instant::now() + Duration::from_secs(3600));
860        assert_eq!(source.stats().deadline_violations, 0);
861        assert!(source.try_next().is_none(), "nothing was queued");
862    }
863
864    #[test]
865    fn a_publish_comfortably_inside_the_budget_is_silent() {
866        let source = EventSource::new();
867        let spec = spec_with_deadline("frames", Some(50));
868        source.register_input(&spec).unwrap();
869        source.push_input(&spec.id, message(0));
870        assert!(source.try_next().unwrap().is_input());
871
872        source.finish_deadlines(Instant::now() + Duration::from_millis(5));
873        assert_eq!(source.stats().deadline_violations, 0);
874        assert!(source.try_next().is_none());
875    }
876
877    #[test]
878    fn a_publish_past_the_budget_is_counted_and_reported() {
879        let source = EventSource::new();
880        let spec = spec_with_deadline("frames", Some(10));
881        source.register_input(&spec).unwrap();
882        source.push_input(&spec.id, message(0));
883        assert!(source.try_next().unwrap().is_input());
884
885        source.finish_deadlines(Instant::now() + Duration::from_secs(1));
886        assert_eq!(source.stats().deadline_violations, 1);
887
888        let event = source.try_next().expect("the violation was queued");
889        let Event::Error(message) = event else {
890            panic!("expected an error event, got {event:?}");
891        };
892        assert!(message.contains("frames"), "{message}");
893        assert!(message.contains("deadline"), "{message}");
894    }
895
896    #[test]
897    fn a_violation_runs_the_relay_hook_with_the_input_budget_and_latency() {
898        let source = EventSource::new();
899        let spec = spec_with_deadline("frames", Some(10));
900        source.register_input(&spec).unwrap();
901
902        let calls: std::sync::Arc<Mutex<Vec<(DataId, Duration, Duration)>>> =
903            std::sync::Arc::new(Mutex::new(Vec::new()));
904        let recorded = calls.clone();
905        source.set_deadline_violation_hook(Box::new(move |input, budget, latency| {
906            lock(&recorded).push((input, budget, latency));
907        }));
908
909        source.push_input(&spec.id, message(0));
910        assert!(source.try_next().unwrap().is_input());
911        source.finish_deadlines(Instant::now() + Duration::from_millis(80));
912
913        let calls = lock(&calls);
914        assert_eq!(calls.len(), 1, "{calls:?}");
915        let (input, budget, latency) = &calls[0];
916        assert_eq!(input, &spec.id);
917        assert_eq!(*budget, Duration::from_millis(10));
918        assert!(*latency >= Duration::from_millis(80), "{latency:?}");
919    }
920
921    #[test]
922    fn a_publish_inside_the_budget_never_runs_the_relay_hook() {
923        let source = EventSource::new();
924        let spec = spec_with_deadline("frames", Some(50));
925        source.register_input(&spec).unwrap();
926
927        let calls: std::sync::Arc<Mutex<Vec<(DataId, Duration, Duration)>>> =
928            std::sync::Arc::new(Mutex::new(Vec::new()));
929        let recorded = calls.clone();
930        source.set_deadline_violation_hook(Box::new(move |input, budget, latency| {
931            lock(&recorded).push((input, budget, latency));
932        }));
933
934        source.push_input(&spec.id, message(0));
935        assert!(source.try_next().unwrap().is_input());
936        source.finish_deadlines(Instant::now() + Duration::from_millis(5));
937
938        assert!(
939            lock(&calls).is_empty(),
940            "the quiet path must not run the relay hook either"
941        );
942    }
943
944    #[test]
945    fn a_second_delivery_supersedes_the_first_open_measurement() {
946        let source = EventSource::new();
947        let spec = spec_with_deadline("frames", Some(10));
948        source.register_input(&spec).unwrap();
949
950        source.push_input(&spec.id, message(0));
951        assert!(source.try_next().unwrap().is_input());
952        // A second delivery on the same input, before anything published,
953        // replaces the first open measurement rather than queuing behind
954        // it — see this module's docs.
955        source.push_input(&spec.id, message(1));
956        assert!(source.try_next().unwrap().is_input());
957
958        source.finish_deadlines(Instant::now() + Duration::from_secs(1));
959        assert_eq!(
960            source.stats().deadline_violations,
961            1,
962            "one measurement was open, not two"
963        );
964    }
965
966    #[test]
967    fn finishing_deadlines_with_nothing_open_does_nothing() {
968        let source = EventSource::new();
969        source.finish_deadlines(Instant::now());
970        assert_eq!(source.stats().deadline_violations, 0);
971        assert!(source.try_next().is_none());
972    }
973
974    #[test]
975    fn a_wind_down_signal_does_not_open_a_deadline_measurement() {
976        let source = EventSource::new();
977        let spec = spec_with_deadline("frames", Some(10));
978        source.register_input(&spec).unwrap();
979        source.push_input(
980            &spec.id,
981            QueuedEvent::Closed {
982                source: PortRef::from_parts("camera", "image").unwrap(),
983                reason: RouteCloseReason::ProducerFinished,
984            },
985        );
986        assert!(matches!(
987            source.try_next().unwrap(),
988            Event::InputClosed { .. }
989        ));
990
991        source.finish_deadlines(Instant::now() + Duration::from_secs(1));
992        assert_eq!(
993            source.stats().deadline_violations,
994            0,
995            "a wind-down signal is not data the node must answer with output"
996        );
997    }
998}