Skip to main content

subc_daemon/
stderr_tail.rs

1//! Bounded per-module stderr capture.
2//!
3//! # Why this exists
4//!
5//! `last_exit_code` survives a respawn because the supervisor holds it in memory.
6//! Stderr had no such path: it went to the daemon's inherited fd, and from there
7//! to whatever rotates or evicts it. On the box this was written for, that window
8//! was about three hours; on another it was bounded by a log file reaching 908 MB
9//! with one module accounting for 98% of it. Two hosts, two mechanisms, the same
10//! outcome -- the text explaining a crash is gone by the time anyone asks.
11//!
12//! So this keeps the last few lines where `last_exit` already lives: in supervisor
13//! memory, immune to whatever happens to the log.
14//!
15//! # What it is not
16//!
17//! Not a log. The ring is deliberately small and lossy, and callers are expected
18//! to know they are reading a tail rather than a history. The daemon log keeps
19//! doing its job; this exists because that job has a time limit.
20
21use std::collections::{BTreeMap, VecDeque};
22use std::io::{self, Write};
23use std::path::{Path, PathBuf};
24use std::sync::{
25    atomic::{AtomicBool, Ordering},
26    Arc, Mutex,
27};
28
29use tokio::io::AsyncReadExt;
30
31/// Longest single line admitted to the ring before truncation.
32///
33/// A module emitting a 40 MB backtrace on one line satisfies any line-count cap
34/// while evicting everything else -- the pathological emitter wins twice, once by
35/// filling the ring and once by being unreadable itself. Truncating on the way in
36/// costs that emitter one line instead of the whole tail.
37pub const DEFAULT_MAX_LINE_BYTES: usize = 2048;
38
39/// Lines retained per module.
40pub const DEFAULT_MAX_LINES: usize = 200;
41
42/// Total bytes retained per module, across all lines.
43///
44/// Both this and [`DEFAULT_MAX_LINES`] apply; whichever binds first wins. A line
45/// cap alone is satisfied by 200 lines of 2 KB, which is not a budget worth
46/// holding for fifteen modules.
47pub const DEFAULT_MAX_BYTES: usize = 64 * 1024;
48
49/// Whether a module's stderr is being captured, and if not, why not.
50///
51/// Typed rather than nullable so `NotCaptured` has to be handled rather than
52/// defaulted past. An empty tail and an uncaptured one send an operator in
53/// opposite directions -- one says the module printed nothing before dying, the
54/// other says nobody was listening -- and rendering them alike is the defect this
55/// module exists to fix, reproduced one layer up.
56#[derive(Debug, Clone, PartialEq, Eq)]
57pub enum CaptureState {
58    /// A reader is attached, or was attached and reached clean EOF.
59    Captured,
60    /// Retained entries are valid, but capture ended before clean EOF, or the
61    /// pipe of a process the supervisor has already moved on from has not
62    /// reached EOF yet. The second kind clears when that pipe does reach EOF,
63    /// because from then on nothing that process wrote is missing.
64    Incomplete { reason: String },
65    /// No reader was attached. The tail says nothing about what the module wrote.
66    NotCaptured { reason: String },
67}
68
69/// One retained entry.
70///
71/// Boundaries are in-band rather than a separate field because their position
72/// relative to the lines is the whole point: "these three lines came from the
73/// process that died, those came from its replacement" is unanswerable from a
74/// count.
75#[derive(Debug, Clone, PartialEq, Eq)]
76pub enum TailEntry {
77    Line {
78        text: String,
79        /// This line was cut at the per-line cap.
80        truncated: bool,
81    },
82    /// The supervisor spawned a new process for this module. Lines after this
83    /// entry come from the new one.
84    ProcessStart,
85}
86
87impl TailEntry {
88    fn cost(&self) -> usize {
89        match self {
90            Self::Line { text, .. } => text.len(),
91            Self::ProcessStart => 0,
92        }
93    }
94}
95
96/// One stored entry. Unlike [`TailEntry`], a boundary remembers which process
97/// generation it starts, so a line that arrives late from an older process can
98/// be put back in that process's section instead of after its successor's
99/// boundary.
100#[derive(Debug, Clone, PartialEq, Eq)]
101enum Slot {
102    Line { text: String, truncated: bool },
103    ProcessStart { generation: u64 },
104}
105
106impl Slot {
107    fn cost(&self) -> usize {
108        match self {
109            Self::Line { text, .. } => text.len(),
110            Self::ProcessStart { .. } => 0,
111        }
112    }
113}
114
115/// Where the stderr reader of one process generation stands.
116#[derive(Debug, Clone, PartialEq, Eq)]
117enum PumpPhase {
118    /// The process is the supervisor's current concern; its lines go at the end.
119    Attached,
120    /// The supervisor has moved on from the process, but its pipe is still open.
121    /// Lines it still delivers belong in its own section.
122    Retired,
123    /// Retired, and the pipe was still open when the supervisor stopped waiting
124    /// for it. Reported as `Incomplete` until the pipe reaches EOF.
125    Late { reason: String },
126}
127
128#[derive(Debug, Clone, Copy, PartialEq, Eq)]
129pub struct StderrTailConfig {
130    max_lines: usize,
131    max_bytes: usize,
132    max_line_bytes: usize,
133}
134
135impl StderrTailConfig {
136    /// Keeps every retained line within the ring's total byte budget.
137    /// Clamp rather than reject so diagnostics degrade without blocking supervisor startup.
138    pub const fn new(max_lines: usize, max_bytes: usize, max_line_bytes: usize) -> Self {
139        Self {
140            max_lines,
141            max_bytes,
142            max_line_bytes: if max_line_bytes > max_bytes {
143                max_bytes
144            } else {
145                max_line_bytes
146            },
147        }
148    }
149}
150
151impl Default for StderrTailConfig {
152    fn default() -> Self {
153        Self::new(DEFAULT_MAX_LINES, DEFAULT_MAX_BYTES, DEFAULT_MAX_LINE_BYTES)
154    }
155}
156
157/// A module's retained stderr, oldest first.
158#[derive(Debug, Clone, PartialEq, Eq)]
159pub struct StderrTailSnapshot {
160    pub capture: CaptureState,
161    pub entries: Vec<TailEntry>,
162    /// Lines evicted since the module was first supervised.
163    ///
164    /// Non-zero means the tail starts mid-stream. That is the ring working as
165    /// intended, but a reader diagnosing a crash needs to know the first retained
166    /// line is not the first line the module wrote -- otherwise an absent cause
167    /// reads as a module that never explained itself.
168    pub dropped_lines: u64,
169}
170
171impl StderrTailSnapshot {
172    /// The uncaptured case, for a module whose stderr was never piped.
173    pub fn not_captured(reason: impl Into<String>) -> Self {
174        Self {
175            capture: CaptureState::NotCaptured {
176                reason: reason.into(),
177            },
178            entries: Vec::new(),
179            dropped_lines: 0,
180        }
181    }
182}
183
184/// Bounded ring of a single module's stderr lines.
185///
186/// Survives respawn deliberately. The stderr explaining an exit is written
187/// *before* that exit, so clearing on restart would discard the lines exactly
188/// when they become the thing being asked for. [`TailEntry::ProcessStart`] keeps
189/// the generations distinguishable instead.
190///
191/// A process's stderr reader can outlive the supervisor's interest in it: the
192/// reader may not have been scheduled yet when the process exited, or a
193/// descendant may still hold the pipe open. Lines such a reader delivers after
194/// the next process started are placed before that next process's boundary, so
195/// the section a line appears in always names the process that wrote it.
196#[derive(Debug)]
197pub struct StderrRing {
198    config: StderrTailConfig,
199    entries: VecDeque<Slot>,
200    // Count of `TailEntry::Line` entries, kept running because eviction checks
201    // it on every push and recounting would walk the whole ring under the
202    // mutex each time.
203    lines: usize,
204    bytes: usize,
205    dropped_lines: u64,
206    capture: CaptureState,
207    /// Generation of the newest process boundary; 0 before the first.
208    generation: u64,
209    /// Readers that have not reached EOF yet, by the generation they read for.
210    pumps: BTreeMap<u64, PumpPhase>,
211    /// Highest generation whose boundary was evicted from the front. A late line
212    /// from an older generation belongs in front of that boundary, which is
213    /// evicted territory, so it is counted as dropped rather than stored.
214    evicted_through: u64,
215}
216
217impl StderrRing {
218    pub fn new(config: StderrTailConfig) -> Self {
219        Self {
220            config,
221            entries: VecDeque::new(),
222            lines: 0,
223            bytes: 0,
224            dropped_lines: 0,
225            // Until a reader attaches, the honest answer is that nothing is
226            // listening -- not that the module has been quiet.
227            capture: CaptureState::NotCaptured {
228                reason: "stderr reader has not started".to_string(),
229            },
230            generation: 0,
231            pumps: BTreeMap::new(),
232            evicted_through: 0,
233        }
234    }
235
236    /// Generation of the newest process boundary.
237    pub fn generation(&self) -> u64 {
238        self.generation
239    }
240
241    pub fn mark_captured(&mut self) {
242        if matches!(self.capture, CaptureState::NotCaptured { .. }) {
243            self.capture = CaptureState::Captured;
244        }
245    }
246
247    pub fn mark_incomplete(&mut self, reason: impl Into<String>) {
248        self.capture = CaptureState::Incomplete {
249            reason: reason.into(),
250        };
251    }
252
253    pub fn mark_not_captured(&mut self, reason: impl Into<String>) {
254        self.capture = CaptureState::NotCaptured {
255            reason: reason.into(),
256        };
257    }
258
259    /// Record that a new process was spawned for this module, returning the
260    /// generation number its lines are attributed to.
261    ///
262    /// A boundary separates output on either side of it, so one with nothing
263    /// before it separates nothing: on the FIRST spawn it would make a module
264    /// that printed nothing render as a marker rather than as empty, and the
265    /// caller then has to decide whether a one-marker tail counts as silence.
266    /// The boundary is still stored, because a late line from an older process
267    /// needs it to find its section, but [`Self::snapshot`] shows it only once
268    /// there is output before it to divide, which keeps "captured and empty"
269    /// literally empty.
270    pub fn push_process_start(&mut self) -> u64 {
271        self.generation += 1;
272        let generation = self.generation;
273        // Two boundaries in a row mean the process between them has written
274        // nothing retained. The earlier one can go only if no reader from its
275        // generation onward is still open: such a reader may yet deliver a line
276        // that belongs between the two. Without this a module that restarts
277        // silently would grow the ring by one boundary per restart forever.
278        if let Some(Slot::ProcessStart {
279            generation: previous,
280        }) = self.entries.back()
281        {
282            if self.pumps.range(*previous..).next().is_none() {
283                self.entries.pop_back();
284            }
285        }
286        self.push_entry(Slot::ProcessStart { generation });
287        generation
288    }
289
290    /// [`Self::push_process_start`] for a process whose stderr reader is
291    /// attached: the reader is tracked until it calls [`Self::finish_pump`].
292    pub(crate) fn begin_process(&mut self) -> u64 {
293        let generation = self.push_process_start();
294        self.pumps.insert(generation, PumpPhase::Attached);
295        generation
296    }
297
298    /// The supervisor has moved on from this generation's process. Its reader
299    /// keeps running, and any line it still delivers is kept in its own section.
300    pub(crate) fn retire_pump(&mut self, generation: u64) {
301        if let Some(phase @ PumpPhase::Attached) = self.pumps.get_mut(&generation) {
302            *phase = PumpPhase::Retired;
303        }
304    }
305
306    /// The supervisor stopped waiting for this generation's reader before its
307    /// pipe reached EOF. The capture reads as `Incomplete` with `reason` until
308    /// the reader finishes. A reader that already finished is left alone: it
309    /// got everything.
310    pub(crate) fn mark_pump_late(&mut self, generation: u64, reason: impl Into<String>) {
311        if let Some(phase) = self.pumps.get_mut(&generation) {
312            *phase = PumpPhase::Late {
313                reason: reason.into(),
314            };
315        }
316    }
317
318    /// This generation's reader has stopped: at EOF, or on a read error that
319    /// has already been recorded with [`Self::mark_incomplete`].
320    pub(crate) fn finish_pump(&mut self, generation: u64) {
321        self.pumps.remove(&generation);
322    }
323
324    /// Admit one complete line, truncating it if it exceeds the per-line cap.
325    ///
326    /// `line` must not contain a trailing newline; the reader strips it so the
327    /// stored text and the byte accounting agree.
328    pub fn push_line(&mut self, line: &str) {
329        self.push_line_from(self.generation, line);
330    }
331
332    /// [`Self::push_line`] for a line read from `generation`'s pipe.
333    ///
334    /// A line from a retired process that arrives after a newer process started
335    /// goes in front of the first boundary newer than its own generation. Lines
336    /// from a process the supervisor has not retired (the incumbent during a
337    /// swap's overlap) go at the end, as they arrive.
338    pub(crate) fn push_line_from(&mut self, generation: u64, line: &str) {
339        let (text, truncated) = truncate_line(line, self.config.max_line_bytes);
340        let slot = Slot::Line { text, truncated };
341        let retired = matches!(
342            self.pumps.get(&generation),
343            Some(PumpPhase::Retired | PumpPhase::Late { .. })
344        );
345        if !retired || generation >= self.generation {
346            self.push_entry(slot);
347            return;
348        }
349        if self.evicted_through > generation {
350            // The section this line belongs to has been evicted, so the line is
351            // older than everything retained.
352            self.dropped_lines += 1;
353            return;
354        }
355        let index = self.entries.iter().position(
356            |slot| matches!(slot, Slot::ProcessStart { generation: start } if *start > generation),
357        );
358        match index {
359            Some(index) => self.insert_entry(index, slot),
360            None => self.push_entry(slot),
361        }
362    }
363
364    fn push_entry(&mut self, entry: Slot) {
365        self.insert_entry(self.entries.len(), entry);
366    }
367
368    fn insert_entry(&mut self, index: usize, entry: Slot) {
369        self.bytes += entry.cost();
370        if matches!(entry, Slot::Line { .. }) {
371            self.lines += 1;
372        }
373        self.entries.insert(index, entry);
374        self.evict_to_fit();
375    }
376
377    fn evict_to_fit(&mut self) {
378        while self.lines > self.config.max_lines
379            || (self.bytes > self.config.max_bytes && self.entries.len() > 1)
380        {
381            let Some(evicted) = self.entries.pop_front() else {
382                break;
383            };
384            self.bytes -= evicted.cost();
385            match evicted {
386                Slot::Line { .. } => {
387                    self.lines -= 1;
388                    self.dropped_lines += 1;
389                }
390                Slot::ProcessStart { generation } => {
391                    self.evicted_through = self.evicted_through.max(generation);
392                }
393            }
394        }
395    }
396
397    /// The most recent entries, oldest first, bounded by the caller's limits.
398    ///
399    /// `max_lines`/`max_bytes` narrow the ring's own caps; they cannot widen them.
400    pub fn snapshot(
401        &self,
402        max_lines: Option<usize>,
403        max_bytes: Option<usize>,
404    ) -> StderrTailSnapshot {
405        let line_limit = max_lines.unwrap_or(self.config.max_lines);
406        let byte_limit = max_bytes.unwrap_or(self.config.max_bytes);
407
408        // The stored boundaries, reduced to the ones worth showing: a boundary
409        // with no output before it (retained or evicted) divides nothing, and
410        // two in a row say no more than one.
411        let mut visible: Vec<TailEntry> = Vec::with_capacity(self.entries.len());
412        let mut output_before = self.dropped_lines > 0;
413        for slot in &self.entries {
414            match slot {
415                Slot::Line { text, truncated } => {
416                    visible.push(TailEntry::Line {
417                        text: text.clone(),
418                        truncated: *truncated,
419                    });
420                    output_before = true;
421                }
422                Slot::ProcessStart { .. } => {
423                    if output_before && !matches!(visible.last(), Some(TailEntry::ProcessStart)) {
424                        visible.push(TailEntry::ProcessStart);
425                    }
426                }
427            }
428        }
429
430        let mut taken: Vec<TailEntry> = Vec::new();
431        let mut bytes = 0usize;
432        let mut lines = 0usize;
433        // Walk backwards: a tail is anchored at the newest end, so a caller
434        // asking for 20 lines wants the last 20, not the first 20.
435        for entry in visible.iter().rev() {
436            match entry {
437                TailEntry::Line { .. } => {
438                    if lines >= line_limit {
439                        break;
440                    }
441                    let cost = entry.cost();
442                    if lines > 0 && bytes + cost > byte_limit {
443                        break;
444                    }
445                    bytes += cost;
446                    lines += 1;
447                    taken.push(entry.clone());
448                }
449                TailEntry::ProcessStart if lines > 0 => taken.push(entry.clone()),
450                TailEntry::ProcessStart => {}
451            }
452        }
453        taken.reverse();
454
455        let withheld = self.lines.saturating_sub(lines);
456
457        // A retired process whose pipe the supervisor stopped waiting for may
458        // still be writing; until its reader reaches EOF the tail cannot claim
459        // to hold everything. A permanent state (a read failure, no pipe at
460        // all) is the more specific fact and is reported as is.
461        let late = self.pumps.values().find_map(|phase| match phase {
462            PumpPhase::Late { reason } => Some(reason),
463            _ => None,
464        });
465        let capture = match (&self.capture, late) {
466            (CaptureState::Captured, Some(reason)) => CaptureState::Incomplete {
467                reason: reason.clone(),
468            },
469            (capture, _) => capture.clone(),
470        };
471
472        StderrTailSnapshot {
473            capture,
474            entries: taken,
475            // Lines the ring evicted plus lines this request's own limits held
476            // back. Both mean the same thing to the reader -- the text above is
477            // not the beginning -- and separating them would invite treating a
478            // narrow request as evidence of a quiet module.
479            dropped_lines: self.dropped_lines + withheld as u64,
480        }
481    }
482}
483
484/// Reassembly buffer ceiling for a line with no newline in sight.
485///
486/// The ring truncates what it stores, but the READER has to hold the bytes until
487/// it finds a delimiter. A module emitting a gigabyte with no newline would grow
488/// this buffer without bound and take the daemon down with it -- a module fault
489/// escalating into a fleet fault, which is exactly what supervision exists to
490/// prevent. At this ceiling the pending bytes are flushed as a line and
491/// reassembly restarts.
492const MAX_PENDING_LINE_BYTES: usize = 1024 * 1024;
493
494/// Read a child's stderr to EOF, retaining the bounded crash tail and forwarding
495/// every complete line to the selected capture sink.
496pub async fn pump_stderr<R>(source: R, ring: Arc<Mutex<StderrRing>>)
497where
498    R: AsyncReadExt + Unpin,
499{
500    pump_stderr_into(source, ring, &mut StderrSink).await
501}
502
503/// Shared destination for a child's stdout and stderr pumps.
504///
505/// The file keeps the historical `.stderr.log` name even though it carries both
506/// streams; the stable name is part of the operator contract. Both pumps share
507/// one mutex, and cortexkit-log writes each framed line in one call, so partial
508/// lines from the two pipes cannot interleave.
509#[derive(Clone)]
510pub(crate) enum ChildOutputSink {
511    File {
512        sink: Arc<Mutex<cortexkit_log::LineSink>>,
513        path: Arc<PathBuf>,
514        failure_reported: Arc<AtomicBool>,
515    },
516    Stderr,
517}
518
519impl ChildOutputSink {
520    pub(crate) fn open(path: &Path, retention: cortexkit_log::Retention) -> io::Result<Self> {
521        Ok(Self::File {
522            sink: Arc::new(Mutex::new(cortexkit_log::LineSink::open(path, retention)?)),
523            path: Arc::new(path.to_path_buf()),
524            failure_reported: Arc::new(AtomicBool::new(false)),
525        })
526    }
527}
528
529/// Where forwarded complete lines go. It exists so tests can observe framing
530/// and so production can serialize the two child pipes through one file sink.
531pub trait OutputSink {
532    fn write_line(&mut self, line: &[u8]);
533}
534
535struct StderrSink;
536
537impl OutputSink for StderrSink {
538    fn write_line(&mut self, line: &[u8]) {
539        let stderr = std::io::stderr();
540        let mut handle = stderr.lock();
541        let _ = handle.write_all(line);
542    }
543}
544
545impl OutputSink for ChildOutputSink {
546    fn write_line(&mut self, line: &[u8]) {
547        match self {
548            Self::File {
549                sink,
550                path,
551                failure_reported,
552            } => {
553                let result = sink
554                    .lock()
555                    .unwrap_or_else(|poisoned| poisoned.into_inner())
556                    .write_line(line);
557                if let Err(error) = result {
558                    if !failure_reported.swap(true, Ordering::Relaxed) {
559                        tracing::warn!(
560                            path = %path.display(),
561                            error = %error,
562                            "child output capture write failed; later failures are suppressed"
563                        );
564                    }
565                }
566            }
567            Self::Stderr => StderrSink.write_line(line),
568        }
569    }
570}
571
572/// Read one process generation's stderr to EOF. `generation` is the value
573/// [`StderrRing::begin_process`] returned for that process; it decides which
574/// section of the ring a line lands in if the reader outlives the process.
575pub(crate) async fn pump_stderr_to<R, S>(
576    source: R,
577    ring: Arc<Mutex<StderrRing>>,
578    generation: u64,
579    mut sink: S,
580) where
581    R: AsyncReadExt + Unpin,
582    S: OutputSink,
583{
584    pump_lines_into(source, Some((&ring, generation)), &mut sink, "stderr").await;
585}
586
587pub(crate) async fn pump_stdout_to<R>(source: R, mut sink: ChildOutputSink)
588where
589    R: AsyncReadExt + Unpin,
590{
591    pump_lines_into(source, None, &mut sink, "stdout").await;
592}
593
594/// Read stderr for whichever process generation is newest when the reader
595/// starts.
596async fn pump_stderr_into<R, S>(source: R, ring: Arc<Mutex<StderrRing>>, sink: &mut S)
597where
598    R: AsyncReadExt + Unpin,
599    S: OutputSink,
600{
601    let generation = lock_ring(&ring).generation();
602    pump_lines_into(source, Some((&ring, generation)), sink, "stderr").await;
603}
604
605async fn pump_lines_into<R, S>(
606    mut source: R,
607    ring: Option<(&Arc<Mutex<StderrRing>>, u64)>,
608    sink: &mut S,
609    stream_name: &str,
610) where
611    R: AsyncReadExt + Unpin,
612    S: OutputSink,
613{
614    if let Some((ring, _)) = ring {
615        lock_ring(ring).mark_captured();
616    }
617
618    let mut pending: Vec<u8> = Vec::new();
619    // Bytes before `scanned_upto` are already known to hold no newline; searching
620    // them again would rescan the whole buffer on every chunk -- for a line with
621    // no newline that is about 64 MiB examined per MiB of module output.
622    let mut scanned_upto = 0usize;
623    // Bytes before `cursor` were emitted as complete lines. They are removed in
624    // one compaction per chunk rather than shifting the buffer once per line.
625    let mut cursor = 0usize;
626    let mut chunk = [0u8; 8192];
627    loop {
628        let read = match source.read(&mut chunk).await {
629            Ok(0) => break,
630            Ok(n) => n,
631            Err(error) => {
632                if let Some((ring, generation)) = ring {
633                    let mut ring = lock_ring(ring);
634                    ring.mark_incomplete(format!("{stream_name} read failed: {error}"));
635                    ring.finish_pump(generation);
636                } else {
637                    tracing::warn!(stream = stream_name, error = %error, "child output capture read failed");
638                }
639                return;
640            }
641        };
642        pending.extend_from_slice(&chunk[..read]);
643
644        while let Some(relative) = find_newline(&pending[scanned_upto..]) {
645            let newline = scanned_upto + relative;
646            emit_line(ring, sink, &pending[cursor..newline], true);
647            cursor = newline + 1;
648            scanned_upto = cursor;
649        }
650        scanned_upto = pending.len();
651
652        if cursor > 0 {
653            pending.drain(..cursor);
654            scanned_upto -= cursor;
655            cursor = 0;
656        }
657
658        if pending.len() >= MAX_PENDING_LINE_BYTES {
659            let line = std::mem::take(&mut pending);
660            emit_line(ring, sink, &line, false);
661            scanned_upto = 0;
662        }
663    }
664
665    if !pending.is_empty() {
666        emit_line(ring, sink, &pending, false);
667    }
668    if let Some((ring, generation)) = ring {
669        lock_ring(ring).finish_pump(generation);
670    }
671}
672
673// Bytes examined by newline searches, summed across a pump. Tests use this to
674// assert the reader does not rescan bytes it already knows contain no newline.
675// Per thread, not process-wide: other tests pump concurrently on their own
676// threads, and a shared counter picked up their searches too, failing the
677// bound by a few bytes under a parallel run. `#[tokio::test]` runs its body
678// and every future it awaits on one thread, so a pump's searches land on the
679// test's own counter.
680#[cfg(test)]
681thread_local! {
682    static SCANNED_BYTES: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
683}
684
685#[cfg(test)]
686fn take_scanned_bytes() -> usize {
687    SCANNED_BYTES.with(|scanned| scanned.replace(0))
688}
689
690/// Locate the next newline in `haystack`, counting the bytes examined so a
691/// test can observe how much of the pending buffer each search walks.
692fn find_newline(haystack: &[u8]) -> Option<usize> {
693    let found = memchr::memchr(b'\n', haystack);
694    #[cfg(test)]
695    SCANNED_BYTES.with(|scanned| {
696        scanned.set(scanned.get() + found.map(|index| index + 1).unwrap_or(haystack.len()));
697    });
698    found
699}
700
701fn emit_line<S: OutputSink>(
702    ring: Option<(&Arc<Mutex<StderrRing>>, u64)>,
703    sink: &mut S,
704    raw: &[u8],
705    terminated: bool,
706) {
707    if let Some((ring, generation)) = ring {
708        lock_ring(ring).push_line_from(generation, &String::from_utf8_lossy(raw));
709    }
710
711    // Framed and written in ONE call. Two writes would let the other pipe land
712    // between the body and newline.
713    if terminated {
714        let mut framed = Vec::with_capacity(raw.len() + 1);
715        framed.extend_from_slice(raw);
716        framed.push(b'\n');
717        sink.write_line(&framed);
718    } else {
719        sink.write_line(raw);
720    }
721}
722
723fn lock_ring(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
724    ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
725}
726
727/// Cut `line` to at most `max_bytes`, reporting whether it was shortened.
728///
729/// Cuts on a char boundary: slicing a multi-byte sequence would produce invalid
730/// UTF-8, and a panic while capturing a crash message is the worst possible time
731/// to discover that.
732fn truncate_line(line: &str, max_bytes: usize) -> (String, bool) {
733    if line.len() <= max_bytes {
734        return (line.to_string(), false);
735    }
736    let mut end = max_bytes;
737    while end > 0 && !line.is_char_boundary(end) {
738        end -= 1;
739    }
740    (line[..end].to_string(), true)
741}
742
743#[cfg(test)]
744mod tests {
745    use std::{
746        io,
747        pin::Pin,
748        task::{Context, Poll},
749    };
750
751    use super::*;
752    use tokio::io::{AsyncRead, ReadBuf};
753
754    fn ring(max_lines: usize, max_bytes: usize, max_line_bytes: usize) -> StderrRing {
755        StderrRing::new(StderrTailConfig::new(max_lines, max_bytes, max_line_bytes))
756    }
757
758    fn lines(snapshot: &StderrTailSnapshot) -> Vec<String> {
759        snapshot
760            .entries
761            .iter()
762            .filter_map(|entry| match entry {
763                TailEntry::Line { text, .. } => Some(text.clone()),
764                TailEntry::ProcessStart => None,
765            })
766            .collect()
767    }
768
769    #[test]
770    fn a_fresh_ring_reports_not_captured_rather_than_empty() {
771        // The distinction this whole module exists for: "nobody was listening"
772        // must not render as "the module said nothing".
773        let ring = ring(10, 1024, 128);
774        let snapshot = ring.snapshot(None, None);
775        assert!(matches!(snapshot.capture, CaptureState::NotCaptured { .. }));
776        assert!(snapshot.entries.is_empty());
777    }
778
779    #[test]
780    fn a_captured_module_that_printed_nothing_is_distinguishable_from_an_uncaptured_one() {
781        let mut captured = ring(10, 1024, 128);
782        captured.mark_captured();
783        let uncaptured = ring(10, 1024, 128);
784
785        let captured = captured.snapshot(None, None);
786        let uncaptured = uncaptured.snapshot(None, None);
787
788        // Both are empty. Only the capture state separates them, which is the
789        // point -- an assertion on emptiness alone would pass either way.
790        assert!(captured.entries.is_empty());
791        assert!(uncaptured.entries.is_empty());
792        assert_eq!(captured.capture, CaptureState::Captured);
793        assert!(matches!(
794            uncaptured.capture,
795            CaptureState::NotCaptured { .. }
796        ));
797    }
798
799    #[test]
800    fn the_line_cap_evicts_oldest_first_and_counts_what_it_dropped() {
801        let mut ring = ring(3, 10_000, 128);
802        ring.mark_captured();
803        for i in 0..6 {
804            ring.push_line(&format!("line{i}"));
805        }
806        let snapshot = ring.snapshot(None, None);
807        assert_eq!(lines(&snapshot), vec!["line3", "line4", "line5"]);
808        // Without this the tail silently becomes "the last lines that happened
809        // to survive" and reads as complete.
810        assert_eq!(snapshot.dropped_lines, 3);
811    }
812
813    #[test]
814    fn the_byte_cap_binds_before_the_line_cap_when_lines_are_large() {
815        // 100 lines allowed, but only ~30 bytes of them.
816        let mut ring = ring(100, 30, 128);
817        ring.mark_captured();
818        for i in 0..10 {
819            ring.push_line(&format!("{i}--------")); // 9 bytes each
820        }
821        let snapshot = ring.snapshot(None, None);
822        assert!(
823            snapshot.entries.len() < 10,
824            "byte cap did not bind: {} entries retained",
825            snapshot.entries.len()
826        );
827        let retained: usize = lines(&snapshot).iter().map(String::len).sum();
828        assert!(
829            retained <= 30,
830            "retained {retained} bytes over a 30 byte cap"
831        );
832        assert!(snapshot.dropped_lines > 0);
833    }
834
835    #[test]
836    fn one_enormous_line_is_truncated_rather_than_evicting_the_tail() {
837        // The pathological-emitter case: without per-line truncation this single
838        // line would evict every other line AND be unreadable itself.
839        let mut ring = ring(10, 10_000, 64);
840        ring.mark_captured();
841        ring.push_line("context line that must survive");
842        ring.push_line(&"x".repeat(40_000));
843
844        let snapshot = ring.snapshot(None, None);
845        let kept = &snapshot.entries;
846        assert!(matches!(
847            &kept[0],
848            TailEntry::Line { text, truncated: false }
849                if text == "context line that must survive"
850        ));
851        let TailEntry::Line { text, truncated } = &kept[1] else {
852            panic!("expected a truncated line");
853        };
854        assert_eq!(text, &"x".repeat(64));
855        assert!(*truncated);
856    }
857
858    #[test]
859    fn truncation_is_visible_so_a_cut_line_is_not_mistaken_for_a_short_one() {
860        let mut ring = ring(10, 10_000, 16);
861        ring.mark_captured();
862        ring.push_line("0123456789abcdefghij");
863        ring.push_line("short");
864
865        let snapshot = ring.snapshot(None, None);
866        let TailEntry::Line { truncated, .. } = &snapshot.entries[0] else {
867            panic!("expected a line");
868        };
869        assert!(truncated);
870        let TailEntry::Line { truncated, .. } = &snapshot.entries[1] else {
871            panic!("expected a line");
872        };
873        assert!(!truncated, "a short line must not be reported as truncated");
874    }
875
876    #[test]
877    fn truncation_cuts_on_a_char_boundary_rather_than_splitting_utf8() {
878        // A panic message with non-ASCII in it is not exotic, and slicing mid
879        // sequence would panic while capturing a crash.
880        let mut ring = ring(10, 10_000, 5);
881        ring.mark_captured();
882        ring.push_line("aa€€€€");
883        let snapshot = ring.snapshot(None, None);
884        let TailEntry::Line { text, truncated } = &snapshot.entries[0] else {
885            panic!("expected a line");
886        };
887        assert!(truncated);
888        assert!(text.starts_with("aa"));
889    }
890
891    #[test]
892    fn a_restart_boundary_keeps_generations_distinguishable() {
893        let mut ring = ring(10, 10_000, 128);
894        ring.mark_captured();
895        ring.push_line("before the crash");
896        ring.push_process_start();
897        ring.push_line("after the respawn");
898
899        let snapshot = ring.snapshot(None, None);
900        assert_eq!(
901            snapshot.entries,
902            vec![
903                TailEntry::Line {
904                    text: "before the crash".to_string(),
905                    truncated: false
906                },
907                TailEntry::ProcessStart,
908                TailEntry::Line {
909                    text: "after the respawn".to_string(),
910                    truncated: false
911                },
912            ]
913        );
914    }
915
916    #[test]
917    fn the_ring_survives_respawn_because_the_cause_is_written_before_the_exit() {
918        // Clearing on restart would discard the lines at the exact moment they
919        // become the thing being asked for.
920        let mut ring = ring(10, 10_000, 128);
921        ring.mark_captured();
922        ring.push_line("Error: storage section missing");
923        ring.push_process_start();
924
925        let snapshot = ring.snapshot(None, None);
926        assert!(lines(&snapshot).contains(&"Error: storage section missing".to_string()));
927    }
928
929    #[test]
930    fn a_caller_limit_returns_the_newest_lines_not_the_oldest() {
931        let mut ring = ring(100, 100_000, 128);
932        ring.mark_captured();
933        for i in 0..10 {
934            ring.push_line(&format!("line{i}"));
935        }
936        let snapshot = ring.snapshot(Some(3), None);
937        assert_eq!(lines(&snapshot), vec!["line7", "line8", "line9"]);
938    }
939
940    #[test]
941    fn a_caller_line_limit_keeps_the_boundary_before_the_selected_line() {
942        let mut ring = ring(100, 100_000, 128);
943        ring.mark_captured();
944        ring.push_line("before restart");
945        ring.push_process_start();
946        ring.push_line("after restart");
947
948        let snapshot = ring.snapshot(Some(1), None);
949        assert_eq!(
950            snapshot.entries,
951            vec![
952                TailEntry::ProcessStart,
953                TailEntry::Line {
954                    text: "after restart".to_string(),
955                    truncated: false,
956                },
957            ]
958        );
959    }
960
961    #[test]
962    fn a_caller_line_limit_omits_a_trailing_boundary_after_the_selected_line() {
963        let mut ring = ring(100, 100_000, 128);
964        ring.mark_captured();
965        ring.push_line("before restart");
966        ring.push_process_start();
967
968        let snapshot = ring.snapshot(Some(1), None);
969        assert_eq!(
970            snapshot.entries,
971            vec![TailEntry::Line {
972                text: "before restart".to_string(),
973                truncated: false,
974            }]
975        );
976    }
977
978    #[test]
979    fn a_caller_limit_reports_what_it_withheld_rather_than_looking_complete() {
980        let mut ring = ring(100, 100_000, 128);
981        ring.mark_captured();
982        for i in 0..10 {
983            ring.push_line(&format!("line{i}"));
984        }
985        // Nothing was evicted; the narrowing is the caller's own. It still has to
986        // be reported, or a 3-line request reads as a module that wrote 3 lines.
987        assert_eq!(ring.snapshot(Some(3), None).dropped_lines, 7);
988        assert_eq!(ring.snapshot(None, None).dropped_lines, 0);
989    }
990
991    #[test]
992    fn a_caller_limit_cannot_widen_the_rings_own_caps() {
993        let mut ring = ring(2, 10_000, 128);
994        ring.mark_captured();
995        for i in 0..5 {
996            ring.push_line(&format!("line{i}"));
997        }
998        let snapshot = ring.snapshot(Some(1000), Some(1_000_000));
999        assert_eq!(lines(&snapshot).len(), 2);
1000    }
1001
1002    fn shared(max_lines: usize, max_bytes: usize, max_line_bytes: usize) -> Arc<Mutex<StderrRing>> {
1003        Arc::new(Mutex::new(ring(max_lines, max_bytes, max_line_bytes)))
1004    }
1005
1006    /// Records each forwarded write separately, so a test can tell one write of
1007    /// `b"abc\n"` from two writes of `b"abc"` and `b"\n"`.
1008    #[derive(Default)]
1009    struct RecordingSink {
1010        writes: Vec<Vec<u8>>,
1011    }
1012
1013    impl OutputSink for RecordingSink {
1014        fn write_line(&mut self, line: &[u8]) {
1015            self.writes.push(line.to_vec());
1016        }
1017    }
1018
1019    /// Yields predetermined chunks, one per read, so a test controls exactly
1020    /// where the byte stream is split.
1021    struct ChunkedReader {
1022        chunks: VecDeque<Vec<u8>>,
1023    }
1024
1025    impl AsyncRead for ChunkedReader {
1026        fn poll_read(
1027            mut self: Pin<&mut Self>,
1028            _cx: &mut Context<'_>,
1029            buf: &mut ReadBuf<'_>,
1030        ) -> Poll<io::Result<()>> {
1031            match self.chunks.pop_front() {
1032                None => Poll::Ready(Ok(())),
1033                Some(chunk) => {
1034                    buf.put_slice(&chunk);
1035                    Poll::Ready(Ok(()))
1036                }
1037            }
1038        }
1039    }
1040
1041    struct FailingReader {
1042        bytes: Vec<u8>,
1043        emitted: bool,
1044    }
1045
1046    impl AsyncRead for FailingReader {
1047        fn poll_read(
1048            mut self: Pin<&mut Self>,
1049            _cx: &mut Context<'_>,
1050            buf: &mut ReadBuf<'_>,
1051        ) -> Poll<io::Result<()>> {
1052            if self.emitted {
1053                return Poll::Ready(Err(io::Error::other("reader failed")));
1054            }
1055            self.emitted = true;
1056            buf.put_slice(&self.bytes);
1057            Poll::Ready(Ok(()))
1058        }
1059    }
1060
1061    #[tokio::test]
1062    async fn the_pump_splits_on_newlines_and_keeps_a_trailing_fragment() {
1063        let ring = shared(10, 10_000, 128);
1064        // No trailing newline on the last line: a crashing process routinely dies
1065        // mid-line, and that fragment is often the message worth reading.
1066        let source = std::io::Cursor::new(b"one\ntwo\nthree".to_vec());
1067        let mut sink = RecordingSink::default();
1068        pump_stderr_into(source, Arc::clone(&ring), &mut sink).await;
1069
1070        let snapshot = lock_ring(&ring).snapshot(None, None);
1071        assert_eq!(lines(&snapshot), vec!["one", "two", "three"]);
1072        assert_eq!(snapshot.capture, CaptureState::Captured);
1073        assert_eq!(
1074            sink.writes,
1075            vec![b"one\n".to_vec(), b"two\n".to_vec(), b"three".to_vec()]
1076        );
1077    }
1078
1079    #[tokio::test]
1080    async fn a_read_failure_keeps_prior_lines_and_marks_the_capture_incomplete() {
1081        let ring = shared(10, 10_000, 128);
1082        let source = FailingReader {
1083            bytes: b"crash cause\n".to_vec(),
1084            emitted: false,
1085        };
1086        let mut sink = RecordingSink::default();
1087        pump_stderr_into(source, Arc::clone(&ring), &mut sink).await;
1088
1089        let snapshot = lock_ring(&ring).snapshot(None, None);
1090        assert_eq!(lines(&snapshot), vec!["crash cause"]);
1091        assert!(matches!(
1092            snapshot.capture,
1093            CaptureState::Incomplete { ref reason } if reason.contains("reader failed")
1094        ));
1095        assert_eq!(sink.writes, vec![b"crash cause\n".to_vec()]);
1096    }
1097
1098    #[tokio::test]
1099    async fn every_captured_line_is_also_forwarded() {
1100        // Forwarding is not optional. The daemon log is overwhelmingly module
1101        // output; a tap that captured without forwarding would leave it nearly
1102        // empty and every existing reader would report clean on nothing.
1103        let ring = shared(10, 10_000, 128);
1104        let source = std::io::Cursor::new(b"alpha\nbeta\n".to_vec());
1105        let mut sink = RecordingSink::default();
1106        pump_stderr_into(source, Arc::clone(&ring), &mut sink).await;
1107
1108        assert_eq!(sink.writes, vec![b"alpha\n".to_vec(), b"beta\n".to_vec()]);
1109    }
1110
1111    #[tokio::test]
1112    async fn each_forwarded_line_is_exactly_one_write() {
1113        // Inheriting the fd gave line atomicity for free. Reading a pipe and
1114        // re-emitting can split a line that used to be atomic, so the framing
1115        // must be one syscall per complete line -- asserted as one write per
1116        // line, not merely as correct bytes.
1117        let ring = shared(10, 10_000, 128);
1118        let source = std::io::Cursor::new(b"first\nsecond\nthird\n".to_vec());
1119        let mut sink = RecordingSink::default();
1120        pump_stderr_into(source, Arc::clone(&ring), &mut sink).await;
1121
1122        assert_eq!(sink.writes.len(), 3);
1123        for write in &sink.writes {
1124            assert_eq!(
1125                write.iter().filter(|byte| **byte == b'\n').count(),
1126                1,
1127                "a write carried something other than exactly one complete line"
1128            );
1129            assert_eq!(*write.last().unwrap(), b'\n');
1130        }
1131    }
1132
1133    #[test]
1134    fn the_first_process_start_is_not_recorded_because_it_divides_nothing() {
1135        // Otherwise a module that printed nothing renders as a lone boundary
1136        // marker, and every caller has to decide whether that counts as silence.
1137        let mut ring = ring(10, 10_000, 128);
1138        ring.push_process_start();
1139        assert!(ring.snapshot(None, None).entries.is_empty());
1140
1141        ring.push_line("first process said this");
1142        ring.push_process_start();
1143        assert!(
1144            matches!(ring.entries.back(), Some(Slot::ProcessStart { .. })),
1145            "a boundary with output before it must be recorded"
1146        );
1147        ring.push_line("second process said this");
1148        assert_eq!(
1149            ring.snapshot(None, None).entries,
1150            vec![
1151                TailEntry::Line {
1152                    text: "first process said this".to_string(),
1153                    truncated: false
1154                },
1155                TailEntry::ProcessStart,
1156                TailEntry::Line {
1157                    text: "second process said this".to_string(),
1158                    truncated: false
1159                },
1160            ],
1161            "only the boundary with output before it may be shown"
1162        );
1163    }
1164
1165    #[test]
1166    fn a_process_start_is_recorded_when_only_dropped_lines_precede_it() {
1167        // The ring can be non-empty in the sense that matters -- lines were
1168        // written and evicted -- while `entries` is empty. Suppressing the
1169        // boundary there would attribute surviving output to the wrong process.
1170        let mut ring = ring(1, 10_000, 128);
1171        ring.push_line("evicted");
1172        ring.push_line("also evicted");
1173        // Emptying `entries` by hand must also zero the running totals kept
1174        // beside it, or the ring holds counts for lines it no longer has and
1175        // any later eviction decision is made against the stale numbers.
1176        ring.entries.clear();
1177        ring.lines = 0;
1178        ring.bytes = 0;
1179        ring.push_process_start();
1180        ring.push_line("survivor");
1181        assert_eq!(
1182            ring.snapshot(None, None).entries,
1183            vec![
1184                TailEntry::ProcessStart,
1185                TailEntry::Line {
1186                    text: "survivor".to_string(),
1187                    truncated: false
1188                },
1189            ]
1190        );
1191    }
1192
1193    fn line(text: &str) -> TailEntry {
1194        TailEntry::Line {
1195            text: text.to_string(),
1196            truncated: false,
1197        }
1198    }
1199
1200    #[test]
1201    fn a_late_line_from_a_retired_process_lands_in_that_processs_section() {
1202        // The reader of a process that already exited may deliver its last
1203        // lines after the next process started. Appending them would put the
1204        // crash's own explanation under its successor's boundary.
1205        let mut ring = ring(10, 10_000, 128);
1206        ring.mark_captured();
1207        let old = ring.begin_process();
1208        ring.push_line_from(old, "old: booting");
1209        ring.retire_pump(old);
1210        let new = ring.begin_process();
1211        ring.push_line_from(new, "new: booting");
1212        ring.push_line_from(old, "old: config error");
1213
1214        assert_eq!(
1215            ring.snapshot(None, None).entries,
1216            vec![
1217                line("old: booting"),
1218                line("old: config error"),
1219                TailEntry::ProcessStart,
1220                line("new: booting"),
1221            ]
1222        );
1223    }
1224
1225    #[test]
1226    fn a_line_from_a_process_that_was_not_retired_is_appended_as_it_arrives() {
1227        // A swap runs the incumbent alongside its candidate; until the
1228        // supervisor retires it, the incumbent is live and its lines are news.
1229        let mut ring = ring(10, 10_000, 128);
1230        ring.mark_captured();
1231        let incumbent = ring.begin_process();
1232        ring.push_line_from(incumbent, "incumbent: before");
1233        let candidate = ring.begin_process();
1234        ring.push_line_from(candidate, "candidate: booting");
1235        ring.push_line_from(incumbent, "incumbent: still serving");
1236
1237        assert_eq!(
1238            ring.snapshot(None, None).entries,
1239            vec![
1240                line("incumbent: before"),
1241                TailEntry::ProcessStart,
1242                line("candidate: booting"),
1243                line("incumbent: still serving"),
1244            ]
1245        );
1246    }
1247
1248    #[test]
1249    fn a_late_line_keeps_its_section_when_the_process_had_printed_nothing_before() {
1250        // A process whose reader had delivered nothing when its successor
1251        // started has a boundary with nothing after it. Dropping that boundary
1252        // as redundant would file the late line under the process before.
1253        let mut ring = ring(10, 10_000, 128);
1254        ring.mark_captured();
1255        let first = ring.begin_process();
1256        ring.push_line_from(first, "first: done");
1257        ring.finish_pump(first);
1258        let old = ring.begin_process();
1259        ring.retire_pump(old);
1260        let new = ring.begin_process();
1261        ring.push_line_from(new, "new: booting");
1262        ring.push_line_from(old, "old: config error");
1263
1264        assert_eq!(
1265            ring.snapshot(None, None).entries,
1266            vec![
1267                line("first: done"),
1268                TailEntry::ProcessStart,
1269                line("old: config error"),
1270                TailEntry::ProcessStart,
1271                line("new: booting"),
1272            ]
1273        );
1274    }
1275
1276    #[test]
1277    fn a_late_line_whose_section_was_evicted_counts_as_dropped() {
1278        let mut ring = ring(2, 10_000, 128);
1279        ring.mark_captured();
1280        let old = ring.begin_process();
1281        ring.push_line_from(old, "old");
1282        ring.retire_pump(old);
1283        let new = ring.begin_process();
1284        for text in ["new 1", "new 2", "new 3"] {
1285            ring.push_line_from(new, text);
1286        }
1287        // The old section and the boundary after it are gone; the late line
1288        // belongs in front of everything retained.
1289        ring.push_line_from(old, "old, late");
1290
1291        let snapshot = ring.snapshot(None, None);
1292        assert_eq!(snapshot.entries, vec![line("new 2"), line("new 3")]);
1293        assert_eq!(snapshot.dropped_lines, 3);
1294    }
1295
1296    #[test]
1297    fn a_late_reader_reads_incomplete_until_its_pipe_reaches_eof() {
1298        let mut ring = ring(10, 10_000, 128);
1299        ring.mark_captured();
1300        let old = ring.begin_process();
1301        ring.retire_pump(old);
1302        ring.mark_pump_late(old, "still open");
1303        ring.begin_process();
1304        assert_eq!(
1305            ring.snapshot(None, None).capture,
1306            CaptureState::Incomplete {
1307                reason: "still open".to_string()
1308            }
1309        );
1310
1311        ring.finish_pump(old);
1312        assert_eq!(ring.snapshot(None, None).capture, CaptureState::Captured);
1313    }
1314
1315    #[test]
1316    fn silent_restarts_do_not_grow_the_ring() {
1317        let mut ring = ring(10, 10_000, 128);
1318        ring.mark_captured();
1319        ring.push_line("once");
1320        for _ in 0..100 {
1321            let generation = ring.begin_process();
1322            ring.finish_pump(generation);
1323        }
1324        assert_eq!(ring.entries.len(), 2);
1325    }
1326
1327    #[tokio::test]
1328    async fn the_pump_marks_captured_even_when_the_module_writes_nothing() {
1329        // Clean EOF with no output is a module that was quiet, not one nobody
1330        // listened to -- and the two must not render alike.
1331        let ring = shared(10, 10_000, 128);
1332        let source = std::io::Cursor::new(Vec::new());
1333        let mut sink = RecordingSink::default();
1334        pump_stderr_into(source, Arc::clone(&ring), &mut sink).await;
1335
1336        let snapshot = lock_ring(&ring).snapshot(None, None);
1337        assert!(snapshot.entries.is_empty());
1338        assert_eq!(snapshot.capture, CaptureState::Captured);
1339        assert!(sink.writes.is_empty());
1340    }
1341
1342    #[tokio::test]
1343    async fn a_line_with_no_newline_cannot_grow_the_reader_without_bound() {
1344        // A module fault must not become a daemon fault: without the pending
1345        // ceiling this buffer grows to whatever the module writes.
1346        let ring = shared(10, 10_000_000, 4 * 1024 * 1024);
1347        let source = std::io::Cursor::new(vec![b'x'; MAX_PENDING_LINE_BYTES + 4096]);
1348        let mut sink = RecordingSink::default();
1349        pump_stderr_into(source, Arc::clone(&ring), &mut sink).await;
1350
1351        let snapshot = lock_ring(&ring).snapshot(None, None);
1352        assert_eq!(
1353            lines(&snapshot).len(),
1354            2,
1355            "expected a forced flush at the ceiling plus the remainder"
1356        );
1357        assert_eq!(
1358            sink.writes,
1359            vec![vec![b'x'; MAX_PENDING_LINE_BYTES], vec![b'x'; 4096],],
1360            "forced flushes and EOF fragments must not invent delimiters"
1361        );
1362    }
1363
1364    #[tokio::test]
1365    async fn boundaries_truncation_and_framing_do_not_depend_on_chunk_splits() {
1366        // The same stream split at hostile boundaries -- mid-line, between a CR
1367        // and its LF, and a line sitting exactly on the per-line cap -- must
1368        // produce the same ring entries and forwarded bytes as any other split.
1369        let ring = shared(100, 100_000, 8);
1370        let source = ChunkedReader {
1371            chunks: vec![
1372                b"fir".to_vec(),
1373                b"st\nsec".to_vec(),
1374                b"ond\ncarry\r".to_vec(),
1375                b"\nover\n".to_vec(),
1376                b"12345678\n".to_vec(),
1377                b"1234567".to_vec(),
1378                b"89\n".to_vec(),
1379                b"tail".to_vec(),
1380            ]
1381            .into_iter()
1382            .collect(),
1383        };
1384        let mut sink = RecordingSink::default();
1385        pump_stderr_into(source, Arc::clone(&ring), &mut sink).await;
1386
1387        let snapshot = lock_ring(&ring).snapshot(None, None);
1388        assert_eq!(snapshot.capture, CaptureState::Captured);
1389        assert_eq!(
1390            snapshot.entries,
1391            vec![
1392                TailEntry::Line {
1393                    text: "first".to_string(),
1394                    truncated: false
1395                },
1396                TailEntry::Line {
1397                    text: "second".to_string(),
1398                    truncated: false
1399                },
1400                // The pump delimits on '\n' alone; a CR belongs to the line body.
1401                TailEntry::Line {
1402                    text: "carry\r".to_string(),
1403                    truncated: false
1404                },
1405                TailEntry::Line {
1406                    text: "over".to_string(),
1407                    truncated: false
1408                },
1409                // Exactly at the per-line cap: kept whole.
1410                TailEntry::Line {
1411                    text: "12345678".to_string(),
1412                    truncated: false
1413                },
1414                // One byte past the cap: cut, and marked as cut.
1415                TailEntry::Line {
1416                    text: "12345678".to_string(),
1417                    truncated: true
1418                },
1419                TailEntry::Line {
1420                    text: "tail".to_string(),
1421                    truncated: false
1422                },
1423            ]
1424        );
1425        assert_eq!(
1426            sink.writes,
1427            vec![
1428                b"first\n".to_vec(),
1429                b"second\n".to_vec(),
1430                b"carry\r\n".to_vec(),
1431                b"over\n".to_vec(),
1432                b"12345678\n".to_vec(),
1433                b"123456789\n".to_vec(),
1434                b"tail".to_vec(),
1435            ]
1436        );
1437    }
1438
1439    #[tokio::test]
1440    async fn a_line_with_no_newline_is_not_rescanned_from_byte_zero_on_every_chunk() {
1441        // One 1 MiB line arrives in 8192-byte reads. Searching the whole
1442        // pending buffer for a newline on every chunk scans each byte once per
1443        // chunk that arrived after it -- about 64 MiB examined per MiB of
1444        // output. Searching only the bytes that arrived since the last search
1445        // scans each byte once.
1446        let input = vec![b'x'; MAX_PENDING_LINE_BYTES + 4096];
1447        let ring = shared(10, 10_000_000, 4 * 1024 * 1024);
1448        let source = std::io::Cursor::new(input.clone());
1449        let mut sink = RecordingSink::default();
1450
1451        take_scanned_bytes();
1452        pump_stderr_into(source, Arc::clone(&ring), &mut sink).await;
1453        let scanned = take_scanned_bytes();
1454
1455        assert!(
1456            scanned <= 2 * input.len(),
1457            "newline searches examined {scanned} bytes for {} bytes of input; \
1458             each chunk must search only newly arrived bytes",
1459            input.len()
1460        );
1461    }
1462
1463    #[test]
1464    fn a_byte_limit_smaller_than_one_line_still_returns_that_line() {
1465        // Returning nothing would be indistinguishable from a quiet module, which
1466        // is the failure this module exists to prevent.
1467        let mut ring = ring(10, 10_000, 128);
1468        ring.mark_captured();
1469        ring.push_line("a line considerably longer than the request limit");
1470        let snapshot = ring.snapshot(None, Some(4));
1471        assert_eq!(snapshot.entries.len(), 1);
1472    }
1473
1474    #[test]
1475    fn an_incoherent_config_clamps_the_line_cap_and_keeps_its_restart_boundary() {
1476        let config = StderrTailConfig::new(2, 10, 100);
1477        assert_eq!(config.max_line_bytes, config.max_bytes);
1478        let mut ring = StderrRing::new(config);
1479        ring.mark_captured();
1480        ring.push_line("old");
1481        ring.push_process_start();
1482        ring.push_line("new process line longer than the ring byte cap");
1483
1484        assert_eq!(
1485            ring.snapshot(None, None).entries,
1486            vec![
1487                TailEntry::ProcessStart,
1488                TailEntry::Line {
1489                    text: "new proces".to_string(),
1490                    truncated: true,
1491                },
1492            ]
1493        );
1494    }
1495}