Skip to main content

astrs_cli/command/
log_stream.rs

1//! Streaming a running dataflow's output to a terminal (blueprint §17:
2//! `astrs run` "streams logs to the terminal"; `astrs logs -f` does the
3//! same over the coordinator link).
4//!
5//! ```text
6//!   embedded daemon ──DaemonEvent──► TerminalLogSink ──bounded──► printer task
7//!   (event loop)                     (never blocks,               (owns stdout)
8//!                                     counts drops)
9//!
10//!   coordinator ──LogFrame──────────────────────────────────────► same printer
11//! ```
12//!
13//! # The sink must never block the event loop
14//!
15//! [`astrs_daemon::health::ReportSink::report`] is called from *inside* the
16//! daemon's merged event loop, and its contract says so: *"reporting must
17//! never block the event loop and must never fail it"*. A sink that took a
18//! lock around a `write!` to a terminal would stall every node on the
19//! machine behind a slow pipe. [`TerminalLogSink`] therefore does the one
20//! thing that cannot stall: a `try_send` into a bounded channel, with a
21//! counter for what did not fit. A separate task owns the sink and does the
22//! formatting and the writing.
23//!
24//! Bounded rather than unbounded on purpose: an unbounded channel in front
25//! of a terminal that has stopped being read is an out-of-memory bug with
26//! extra steps. Dropping the oldest news under sustained overload — and
27//! saying how much was dropped when the stream ends — is the behavior a
28//! log tail should have.
29
30use std::io::Write;
31use std::sync::atomic::{AtomicU64, Ordering};
32
33use astrs_daemon::health::ReportSink;
34use astrs_wire::{DaemonEvent, LogLevel, LogRecord, NodeExitCause, NodeId, SpawnOutcome};
35use tokio::sync::mpsc;
36
37/// How many items may be queued between the daemon's event loop and the
38/// printer before the oldest news starts being dropped.
39///
40/// Sized for a burst, not a backlog: a node that logs a thousand lines in
41/// one scheduler tick should not lose any, while a terminal that has been
42/// wedged for a minute should not cost the daemon a megabyte of memory.
43pub const STREAM_CAPACITY: usize = 4096;
44
45/// One thing worth printing.
46#[derive(Debug, Clone, PartialEq, Eq)]
47pub enum StreamItem {
48    /// A log record from a node (captured stdout/stderr, or a structured
49    /// record the node emitted).
50    Record(Box<LogRecord>),
51    /// Something the CLI itself observed about the run: a node started, a
52    /// node exited, a restart. Not a node's own output, and marked so.
53    Notice {
54        /// How serious it is.
55        level: LogLevel,
56        /// The node it concerns, when it concerns one.
57        node: Option<NodeId>,
58        /// The text.
59        message: String,
60    },
61}
62
63impl StreamItem {
64    /// This item's severity, whatever shape it has.
65    #[must_use]
66    pub fn level(&self) -> LogLevel {
67        match self {
68            Self::Record(record) => record.level,
69            Self::Notice { level, .. } => *level,
70        }
71    }
72
73    /// The node this item is about, when it is about one.
74    #[must_use]
75    pub fn node(&self) -> Option<&NodeId> {
76        match self {
77            Self::Record(record) => record.node.as_ref(),
78            Self::Notice { node, .. } => node.as_ref(),
79        }
80    }
81
82    /// The text to print.
83    #[must_use]
84    pub fn message(&self) -> &str {
85        match self {
86            Self::Record(record) => &record.message,
87            Self::Notice { message, .. } => message,
88        }
89    }
90}
91
92/// Which items reach the terminal.
93#[derive(Debug, Clone)]
94pub struct LogFilter {
95    /// Hide anything less severe than this.
96    pub min_level: LogLevel,
97    /// Show only this node's items, when set.
98    pub node: Option<NodeId>,
99}
100
101impl Default for LogFilter {
102    fn default() -> Self {
103        Self {
104            min_level: LogLevel::Trace,
105            node: None,
106        }
107    }
108}
109
110impl LogFilter {
111    /// A filter that hides nothing.
112    #[must_use]
113    pub fn new() -> Self {
114        Self::default()
115    }
116
117    /// Hides anything below `level`.
118    #[must_use]
119    pub const fn with_min_level(mut self, level: LogLevel) -> Self {
120        self.min_level = level;
121        self
122    }
123
124    /// Restricts the stream to one node.
125    #[must_use]
126    pub fn with_node(mut self, node: Option<NodeId>) -> Self {
127        self.node = node;
128        self
129    }
130
131    /// Whether `item` should be printed.
132    ///
133    /// A [`StreamItem::Notice`] with no node survives a `--node` filter:
134    /// "the dataflow failed" is not one node's news, and hiding it because
135    /// the user asked to watch one node would hide the reason the stream
136    /// ended.
137    #[must_use]
138    pub fn accepts(&self, item: &StreamItem) -> bool {
139        if !level_at_least(item.level(), self.min_level) {
140            return false;
141        }
142        match (&self.node, item.node()) {
143            (Some(wanted), Some(actual)) => wanted == actual,
144            (Some(_), None) => matches!(item, StreamItem::Notice { .. }),
145            (None, _) => true,
146        }
147    }
148}
149
150/// Whether `level` is at least as severe as `floor`.
151///
152/// [`LogLevel`]'s own ordering runs `Trace` (least) to `Error` (most), and
153/// this is the one place that fact is relied on — a comparison spelled out
154/// here rather than repeated at each call site.
155#[must_use]
156pub fn level_at_least(level: LogLevel, floor: LogLevel) -> bool {
157    severity(level) >= severity(floor)
158}
159
160/// A total order over severities, most severe highest.
161const fn severity(level: LogLevel) -> u8 {
162    match level {
163        LogLevel::Trace => 0,
164        LogLevel::Debug => 1,
165        LogLevel::Info => 2,
166        LogLevel::Warn => 3,
167        LogLevel::Error => 4,
168        // `LogLevel` is `#[non_exhaustive]` (the protocol may grow a
169        // severity); an unknown one is treated as the most severe rather
170        // than silently filtered out, because news the CLI cannot classify
171        // is exactly the news a user must not miss.
172        _ => u8::MAX,
173    }
174}
175
176/// Parses a `--level` argument, accepting the spellings a user types.
177///
178/// # Errors
179///
180/// [`crate::error::CliError::BadArgument`] naming the accepted values.
181pub fn parse_level(flag: &'static str, text: &str) -> Result<LogLevel, crate::error::CliError> {
182    match text.trim().to_ascii_lowercase().as_str() {
183        "trace" => Ok(LogLevel::Trace),
184        "debug" => Ok(LogLevel::Debug),
185        "info" => Ok(LogLevel::Info),
186        "warn" | "warning" => Ok(LogLevel::Warn),
187        "error" | "err" => Ok(LogLevel::Error),
188        _ => Err(crate::error::CliError::BadArgument {
189            flag,
190            value: text.to_owned(),
191            reason: "expected one of trace, debug, info, warn, error".to_owned(),
192        }),
193    }
194}
195
196/// The [`ReportSink`] `astrs run` installs on its embedded daemon.
197///
198/// Turns the daemon's upward traffic into [`StreamItem`]s a printer task
199/// consumes, without ever blocking or failing the event loop.
200#[derive(Debug)]
201pub struct TerminalLogSink {
202    /// The bounded queue into the printer.
203    sender: mpsc::Sender<StreamItem>,
204    /// How many items did not fit.
205    dropped: AtomicU64,
206}
207
208impl TerminalLogSink {
209    /// Builds a sink and the receiver a printer task drains.
210    #[must_use]
211    pub fn new() -> (Self, mpsc::Receiver<StreamItem>) {
212        let (sender, receiver) = mpsc::channel(STREAM_CAPACITY);
213        (
214            Self {
215                sender,
216                dropped: AtomicU64::new(0),
217            },
218            receiver,
219        )
220    }
221
222    /// Queues one item, counting it as dropped if there is no room.
223    fn offer(&self, item: StreamItem) {
224        if self.sender.try_send(item).is_err() {
225            self.dropped.fetch_add(1, Ordering::Relaxed);
226        }
227    }
228}
229
230impl ReportSink for TerminalLogSink {
231    fn report(&self, event: DaemonEvent) {
232        match event {
233            DaemonEvent::Log { records, .. } => {
234                for record in records {
235                    self.offer(StreamItem::Record(Box::new(record)));
236                }
237            }
238            DaemonEvent::SpawnResult { node, outcome, .. } => {
239                if let Some(message) = spawn_failure_text(&outcome) {
240                    self.offer(StreamItem::Notice {
241                        level: LogLevel::Error,
242                        node: Some(node),
243                        message,
244                    });
245                }
246            }
247            DaemonEvent::NodeStopped {
248                node,
249                cause,
250                restarting,
251                ..
252            } => {
253                self.offer(StreamItem::Notice {
254                    level: exit_level(&cause),
255                    node: Some(node),
256                    message: exit_text(&cause, restarting),
257                });
258            }
259            // Heartbeats, metric batches, tapped topic frames, catch-up
260            // acknowledgements: real events, but not *news a terminal wants*.
261            // Deliberately dropped here rather than filtered downstream, so
262            // a 2 s metrics tick never costs a channel slot a log line could
263            // have used.
264            _ => {}
265        }
266    }
267
268    fn dropped(&self) -> u64 {
269        self.dropped.load(Ordering::Relaxed)
270    }
271
272    fn is_open(&self) -> bool {
273        !self.sender.is_closed()
274    }
275}
276
277/// The message for a spawn that did not produce a running node, or `None`
278/// when it did.
279fn spawn_failure_text(outcome: &SpawnOutcome) -> Option<String> {
280    match outcome {
281        SpawnOutcome::Spawned { .. } | SpawnOutcome::AwaitingDynamic => None,
282        SpawnOutcome::Failed { message, errno } => Some(match errno {
283            Some(number) => format!("failed to spawn: {message} (errno {number})"),
284            None => format!("failed to spawn: {message}"),
285        }),
286        SpawnOutcome::Cancelled => Some("spawn cancelled: the dataflow stopped first".to_owned()),
287        other => Some(format!("spawn did not start the node: {other:?}")),
288    }
289}
290
291/// How severe a node's exit is.
292fn exit_level(cause: &NodeExitCause) -> LogLevel {
293    if cause.is_failure() {
294        LogLevel::Error
295    } else {
296        LogLevel::Info
297    }
298}
299
300/// The one-line summary of a node's exit.
301fn exit_text(cause: &NodeExitCause, restarting: bool) -> String {
302    if restarting {
303        format!("exited ({cause}); restarting")
304    } else {
305        format!("exited ({cause})")
306    }
307}
308
309/// How the terminal is painted.
310#[derive(Debug, Clone, Copy)]
311pub struct LogStyle {
312    /// Whether to emit ANSI SGR codes.
313    pub color: bool,
314    /// How wide the `[node]` column is, so messages line up.
315    pub prefix_width: usize,
316    /// Whether to prefix each line with the seconds since the stream
317    /// started.
318    pub elapsed: bool,
319}
320
321impl Default for LogStyle {
322    fn default() -> Self {
323        Self {
324            color: false,
325            prefix_width: 0,
326            elapsed: true,
327        }
328    }
329}
330
331/// Formats one item as the line a terminal shows.
332///
333/// `elapsed` is the time since the stream started, used only when
334/// [`LogStyle::elapsed`] is set — the wall clock a user cares about during a
335/// run is "how long has this been going", not a date, and deriving it here
336/// keeps the formatter free of any dependency on how the caller measures
337/// time.
338#[must_use]
339pub fn render(item: &StreamItem, style: LogStyle, elapsed: std::time::Duration) -> String {
340    let mut line = String::with_capacity(item.message().len() + 32);
341    if style.elapsed {
342        line.push_str(&format!("{:>8.3}s ", elapsed.as_secs_f64()));
343    }
344    let name = item
345        .node()
346        .map_or_else(|| "astrs".to_owned(), |node| node.as_str().to_owned());
347    let padded = if name.len() >= style.prefix_width {
348        name.clone()
349    } else {
350        format!("{name:<width$}", width = style.prefix_width)
351    };
352    if style.color {
353        line.push_str(&format!("{}[{padded}]{} ", node_color(&name), ansi::RESET));
354    } else {
355        line.push_str(&format!("[{padded}] "));
356    }
357    let level = item.level();
358    if level_at_least(level, LogLevel::Warn) {
359        if style.color {
360            line.push_str(&format!(
361                "{}{}{} ",
362                level_color(level),
363                level_tag(level),
364                ansi::RESET
365            ));
366        } else {
367            line.push_str(&format!("{} ", level_tag(level)));
368        }
369    }
370    line.push_str(item.message());
371    line
372}
373
374/// The uppercase tag shown for a warning or an error.
375const fn level_tag(level: LogLevel) -> &'static str {
376    match level {
377        LogLevel::Trace => "TRACE",
378        LogLevel::Debug => "DEBUG",
379        LogLevel::Info => "INFO",
380        LogLevel::Warn => "WARN",
381        LogLevel::Error => "ERROR",
382        _ => "LOG",
383    }
384}
385
386/// Plain ANSI SGR codes — the same handful [`crate::diagnostic`] uses, kept
387/// here rather than shared because these are *foreground palette* entries
388/// for node names, not severity colors.
389mod ansi {
390    /// Ends any styling.
391    pub const RESET: &str = "\x1b[0m";
392    /// The per-node palette, chosen for legibility on both light and dark
393    /// terminals (no bright yellow, no pure blue).
394    pub const PALETTE: &[&str] = &[
395        "\x1b[36m", // cyan
396        "\x1b[35m", // magenta
397        "\x1b[32m", // green
398        "\x1b[34m", // blue
399        "\x1b[33m", // yellow
400        "\x1b[31m", // red
401    ];
402    /// Warnings.
403    pub const YELLOW: &str = "\x1b[33m";
404    /// Errors.
405    pub const RED: &str = "\x1b[31m";
406}
407
408/// A stable color for a node name.
409///
410/// Stable across the whole run *and* across runs: the same graph always
411/// paints `camera` the same color, so a user reading two terminals side by
412/// side can match them up. A hash rather than an assignment counter is what
413/// makes that true even though nodes register in a nondeterministic order.
414fn node_color(name: &str) -> &'static str {
415    let mut hash: u64 = 0xcbf2_9ce4_8422_2325;
416    for byte in name.as_bytes() {
417        hash ^= u64::from(*byte);
418        hash = hash.wrapping_mul(0x0000_0100_0000_01b3);
419    }
420    let index = usize::try_from(hash % ansi::PALETTE.len() as u64).unwrap_or(0);
421    ansi::PALETTE.get(index).copied().unwrap_or(ansi::RESET)
422}
423
424/// The color a severity tag is painted.
425const fn level_color(level: LogLevel) -> &'static str {
426    match level {
427        LogLevel::Error => ansi::RED,
428        _ => ansi::YELLOW,
429    }
430}
431
432/// Drains `receiver` onto `out` until the stream ends, returning how many
433/// items were printed.
434///
435/// Ends when every sender is dropped — which, for `astrs run`, is when the
436/// daemon and the sink it held are both gone, i.e. exactly when the run is
437/// over and there is nothing left to print.
438pub async fn print_stream(
439    receiver: &mut mpsc::Receiver<StreamItem>,
440    out: &mut (dyn Write + Send),
441    filter: &LogFilter,
442    style: LogStyle,
443    started: std::time::Instant,
444) -> usize {
445    let mut printed = 0;
446    while let Some(item) = receiver.recv().await {
447        if !filter.accepts(&item) {
448            continue;
449        }
450        let line = render(&item, style, started.elapsed());
451        if writeln!(out, "{line}").is_err() {
452            // The terminal went away (a closed pipe). Nothing useful is left
453            // to do, and failing the run because `head -5` exited would be
454            // absurd; stop printing and let the run finish on its own terms.
455            break;
456        }
457        let _ = out.flush();
458        printed += 1;
459    }
460    printed
461}
462
463/// The widest node name in a manifest, so the `[node]` column is sized
464/// before the first line is printed rather than jittering as nodes appear.
465#[must_use]
466pub fn prefix_width<'a>(names: impl IntoIterator<Item = &'a str>) -> usize {
467    names
468        .into_iter()
469        .map(str::len)
470        .max()
471        .unwrap_or(0)
472        .max("astrs".len())
473}
474
475#[cfg(test)]
476mod tests {
477    #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
478
479    use super::*;
480    use astrs_time::HlcTimestamp;
481
482    fn node(name: &str) -> NodeId {
483        NodeId::new(name).unwrap()
484    }
485
486    fn record(level: LogLevel, name: &str, message: &str) -> StreamItem {
487        StreamItem::Record(Box::new(
488            LogRecord::new(HlcTimestamp::new(1, 0), level, message).with_node(node(name)),
489        ))
490    }
491
492    #[test]
493    fn severities_are_ordered_trace_to_error() {
494        assert!(level_at_least(LogLevel::Error, LogLevel::Trace));
495        assert!(level_at_least(LogLevel::Info, LogLevel::Info));
496        assert!(!level_at_least(LogLevel::Debug, LogLevel::Info));
497    }
498
499    #[test]
500    fn a_level_filter_hides_what_is_below_it() {
501        let filter = LogFilter::new().with_min_level(LogLevel::Warn);
502        assert!(!filter.accepts(&record(LogLevel::Info, "cam", "hi")));
503        assert!(filter.accepts(&record(LogLevel::Warn, "cam", "hi")));
504        assert!(filter.accepts(&record(LogLevel::Error, "cam", "hi")));
505    }
506
507    #[test]
508    fn a_node_filter_keeps_only_that_node_but_never_hides_a_global_notice() {
509        let filter = LogFilter::new().with_node(Some(node("cam")));
510        assert!(filter.accepts(&record(LogLevel::Info, "cam", "mine")));
511        assert!(!filter.accepts(&record(LogLevel::Info, "detect", "theirs")));
512        assert!(filter.accepts(&StreamItem::Notice {
513            level: LogLevel::Error,
514            node: None,
515            message: "the dataflow failed".to_owned(),
516        }));
517    }
518
519    #[test]
520    fn parse_level_accepts_the_spellings_a_user_types() {
521        assert_eq!(parse_level("level", "WARN").unwrap(), LogLevel::Warn);
522        assert_eq!(parse_level("level", "warning").unwrap(), LogLevel::Warn);
523        assert_eq!(parse_level("level", " error ").unwrap(), LogLevel::Error);
524        let error = parse_level("level", "loud").unwrap_err();
525        assert!(error.to_string().contains("trace"), "{error}");
526    }
527
528    #[test]
529    fn rendering_pads_the_node_column_and_omits_a_tag_below_warn() {
530        let style = LogStyle {
531            color: false,
532            prefix_width: 8,
533            elapsed: false,
534        };
535        let line = render(
536            &record(LogLevel::Info, "cam", "frame 1"),
537            style,
538            std::time::Duration::ZERO,
539        );
540        assert_eq!(line, "[cam     ] frame 1");
541
542        let line = render(
543            &record(LogLevel::Error, "cam", "boom"),
544            style,
545            std::time::Duration::ZERO,
546        );
547        assert_eq!(line, "[cam     ] ERROR boom");
548    }
549
550    #[test]
551    fn rendering_with_color_wraps_the_prefix_and_resets() {
552        let style = LogStyle {
553            color: true,
554            prefix_width: 0,
555            elapsed: false,
556        };
557        let line = render(
558            &record(LogLevel::Info, "cam", "hi"),
559            style,
560            std::time::Duration::ZERO,
561        );
562        assert!(line.contains("\x1b["), "{line}");
563        assert!(line.ends_with("hi"), "{line}");
564        assert!(line.contains(ansi::RESET));
565    }
566
567    #[test]
568    fn elapsed_is_rendered_when_asked_for() {
569        let style = LogStyle {
570            color: false,
571            prefix_width: 0,
572            elapsed: true,
573        };
574        let line = render(
575            &record(LogLevel::Info, "cam", "hi"),
576            style,
577            std::time::Duration::from_millis(1500),
578        );
579        assert!(line.starts_with("   1.500s "), "{line}");
580    }
581
582    #[test]
583    fn a_node_keeps_one_color_across_calls() {
584        assert_eq!(node_color("camera"), node_color("camera"));
585    }
586
587    #[test]
588    fn the_prefix_width_never_shrinks_below_the_cli_s_own_name() {
589        assert_eq!(prefix_width(["a", "bb"]), "astrs".len());
590        assert_eq!(prefix_width(["a-very-long-node"]), "a-very-long-node".len());
591        assert_eq!(prefix_width(std::iter::empty()), "astrs".len());
592    }
593
594    #[tokio::test]
595    async fn the_sink_turns_daemon_log_events_into_stream_items() {
596        let (sink, mut receiver) = TerminalLogSink::new();
597        sink.report(DaemonEvent::Log {
598            request: None,
599            records: vec![
600                LogRecord::new(HlcTimestamp::new(1, 0), LogLevel::Info, "one")
601                    .with_node(node("cam")),
602                LogRecord::new(HlcTimestamp::new(2, 0), LogLevel::Error, "two")
603                    .with_node(node("cam")),
604            ],
605            truncated: false,
606        });
607        assert_eq!(receiver.recv().await.unwrap().message(), "one");
608        assert_eq!(receiver.recv().await.unwrap().message(), "two");
609        assert_eq!(sink.dropped(), 0);
610    }
611
612    #[tokio::test]
613    async fn the_sink_summarizes_a_node_exit_and_marks_a_restart() {
614        let (sink, mut receiver) = TerminalLogSink::new();
615        sink.report(DaemonEvent::NodeStopped {
616            dataflow: astrs_wire::DataflowId::from_u128(1),
617            node: node("cam"),
618            generation: 0,
619            cause: NodeExitCause::ExitCode { code: 3 },
620            restarting: true,
621        });
622        let item = receiver.recv().await.unwrap();
623        assert_eq!(item.level(), LogLevel::Error);
624        assert!(item.message().contains("restarting"), "{item:?}");
625    }
626
627    #[tokio::test]
628    async fn the_sink_ignores_traffic_a_terminal_does_not_want() {
629        let (sink, mut receiver) = TerminalLogSink::new();
630        sink.report(DaemonEvent::StateCatchUpAck { seq: 1, applied: 0 });
631        assert!(
632            receiver.try_recv().is_err(),
633            "a catch-up acknowledgement is not news"
634        );
635    }
636
637    #[tokio::test]
638    async fn the_sink_counts_what_does_not_fit_instead_of_blocking() {
639        let (sink, receiver) = TerminalLogSink::new();
640        for index in 0..(STREAM_CAPACITY + 8) {
641            sink.report(DaemonEvent::Log {
642                request: None,
643                records: vec![LogRecord::new(
644                    HlcTimestamp::new(1, 0),
645                    LogLevel::Info,
646                    format!("line {index}"),
647                )],
648                truncated: false,
649            });
650        }
651        assert_eq!(sink.dropped(), 8, "the overflow is counted, not queued");
652        drop(receiver);
653        assert!(!sink.is_open());
654    }
655
656    #[tokio::test]
657    async fn printing_applies_the_filter_and_stops_when_the_senders_go() {
658        let (sink, mut receiver) = TerminalLogSink::new();
659        sink.report(DaemonEvent::Log {
660            request: None,
661            records: vec![
662                LogRecord::new(HlcTimestamp::new(1, 0), LogLevel::Debug, "quiet")
663                    .with_node(node("cam")),
664                LogRecord::new(HlcTimestamp::new(2, 0), LogLevel::Error, "loud")
665                    .with_node(node("cam")),
666            ],
667            truncated: false,
668        });
669        drop(sink);
670
671        let mut out: Vec<u8> = Vec::new();
672        let filter = LogFilter::new().with_min_level(LogLevel::Warn);
673        let style = LogStyle {
674            color: false,
675            prefix_width: 0,
676            elapsed: false,
677        };
678        let printed = print_stream(
679            &mut receiver,
680            &mut out,
681            &filter,
682            style,
683            std::time::Instant::now(),
684        )
685        .await;
686        assert_eq!(printed, 1);
687        let text = String::from_utf8(out).unwrap();
688        assert!(text.contains("loud"), "{text}");
689        assert!(!text.contains("quiet"), "{text}");
690    }
691}