Skip to main content

runnerd_core/api/jobs/
log.rs

1//! A job's log as structure, for every client (ADR 0133): each stored line styled, the outline the store keeps as
2//! the lines arrive, what a failed job's log says of its failure, and a literal search. The plain log
3//! (`GET /v1/jobs/{id}/logs`) and `log.line` events carry the lines as stored; these types are the only way the log
4//! reaches a client as structure, and no escape sequence or control character is left in them.
5//!
6//! A styled line has its timestamp split off, the runner's `##[…]` command classified, and its SGR colours mapped
7//! to a closed set; every other escape and control character is dropped. Pure functions: the store feeds
8//! [`Outliner`] and keeps what it returns, and the daemon, the demo's fake daemon and the fixtures assemble the same
9//! answers from it.
10
11use serde::{Deserialize, Serialize};
12
13mod outline;
14mod search;
15
16pub(crate) use outline::TITLE_SOURCE_MAX;
17pub(crate) use outline::title;
18pub use outline::{
19    FailureSummary, LineRange, LogEvidence, LogGroup, LogMark, LogOutline, MarkKind, OutlineChange,
20    OutlineParts, Outliner, SavedOutline, StepLines, StepLog, StoredLog,
21};
22pub use search::{LogNeedle, LogSearch, LogSearchQuery, SEARCH_QUERY_MAX, Search};
23
24/// `GET /v1/jobs/{id}/log/lines`: lines a page holds unless `limit` says otherwise.
25pub const LINES_DEFAULT: u32 = 1000;
26
27/// `GET /v1/jobs/{id}/log/lines`: lines a page holds at most.
28pub const LINES_MAX: u32 = 10_000;
29
30/// The most of a stored line's text a styled line shows, in bytes; the rest is counted in
31/// [`StyledLine::omitted`].
32pub const LINE_BYTES_MAX: u64 = 64 << 10;
33
34/// The most runs a styled line holds; the text after them is counted in [`StyledLine::omitted`].
35pub const RUNS_MAX: usize = 1000;
36
37/// How the runner prints a line's command: `##[error]…`.
38const COMMAND: &str = "##[";
39
40/// Stored lines to read: up to `limit` from line `from` (past a gap pruning left, too), each cut to its first
41/// `prefix` bytes, stopping after the line that brings the text read to `bytes`.
42#[derive(Debug, Clone, Copy, PartialEq, Eq)]
43pub struct LineRead {
44    /// The first line.
45    pub from: u64,
46    /// How many lines at most.
47    pub limit: u64,
48    /// How much of each line's text, in bytes.
49    pub prefix: u64,
50    /// How much text in all, in bytes: the line that reaches it is the last.
51    pub bytes: u64,
52}
53
54impl LineRead {
55    /// What it reads of `lines`, a whole log held in memory, as the store reads it: for fixtures and the demo.
56    pub fn of(self, lines: &[String]) -> Vec<StoredLine> {
57        let mut read = Vec::new();
58        let mut bytes = 0;
59        let first = usize::try_from(self.from).unwrap_or(usize::MAX);
60        let numbered = (self.from..).zip(lines.iter().skip(first));
61        for (line, text) in numbered.take(usize::try_from(self.limit).unwrap_or(usize::MAX)) {
62            if bytes >= self.bytes {
63                break;
64            }
65            let prefix = usize::try_from(self.prefix).unwrap_or(usize::MAX);
66            let cut = &text.as_bytes()[..text.len().min(prefix)];
67            let stored = StoredLine::cut(line, cut, text.len() as u64);
68            bytes += stored.text.len() as u64;
69            read.push(stored);
70        }
71        read
72    }
73}
74
75/// A stored line as a [`LineRead`] gives it: its number, and its text, cut to the bytes the read asked for.
76#[derive(Debug, Clone, PartialEq, Eq)]
77pub struct StoredLine {
78    /// Its number in the stored log, from 0.
79    pub line: u64,
80    /// Its text, or the first of it; bytes that are not UTF-8 replaced.
81    pub text: String,
82    /// The stored bytes not in `text`.
83    pub omitted: u64,
84}
85
86impl StoredLine {
87    /// Line `line` from the first `read` of its `stored` bytes: a character the cut splits is left out.
88    pub fn cut(line: u64, read: &[u8], stored: u64) -> StoredLine {
89        let whole = if (read.len() as u64) < stored {
90            whole_chars(read)
91        } else {
92            read
93        };
94        StoredLine {
95            line,
96            text: String::from_utf8_lossy(whole).into_owned(),
97            omitted: stored.saturating_sub(whole.len() as u64),
98        }
99    }
100
101    /// Styled, what it omits counted.
102    pub fn styled(&self) -> StyledLine {
103        let mut styled = StyledLine::parse(self.line, &self.text);
104        styled.omitted += self.omitted;
105        styled
106    }
107}
108
109/// `bytes` without a UTF-8 character cut off at its end.
110fn whole_chars(bytes: &[u8]) -> &[u8] {
111    // The last character starts at most 3 bytes before the end, at a byte that is not a continuation byte.
112    let Some(start) = (bytes.len().saturating_sub(4)..bytes.len())
113        .rev()
114        .find(|&at| bytes[at] & 0xC0 != 0x80)
115    else {
116        return bytes;
117    };
118    let width = match bytes[start] {
119        lead if lead >= 0xF0 => 4,
120        lead if lead >= 0xE0 => 3,
121        lead if lead >= 0xC0 => 2,
122        _ => 1,
123    };
124    if start + width > bytes.len() {
125        &bytes[..start]
126    } else {
127        bytes
128    }
129}
130
131/// The first `max` bytes of `text`, or fewer so a character is not cut.
132pub(crate) fn prefix(text: &str, max: usize) -> &str {
133    if text.len() <= max {
134        return text;
135    }
136    let mut end = max;
137    while !text.is_char_boundary(end) {
138        end -= 1;
139    }
140    &text[..end]
141}
142
143/// `GET /v1/jobs/{id}/log/lines` query parameters. The response is `application/x-ndjson`: one [`StyledLine`] per
144/// line, in order.
145#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
146pub struct LogLinesQuery {
147    /// Start at this line, 0-based.
148    pub from: Option<u64>,
149    /// At most this many lines; default [`LINES_DEFAULT`], at most [`LINES_MAX`]. Not read with `follow`.
150    pub limit: Option<u32>,
151    /// Keep the response open and send lines as they are stored, until the job finishes, as the plain log does.
152    #[serde(default)]
153    pub follow: bool,
154}
155
156impl LogLinesQuery {
157    /// How many lines a page without `follow` holds.
158    pub fn page(&self) -> u64 {
159        u64::from(self.limit.unwrap_or(LINES_DEFAULT).clamp(1, LINES_MAX))
160    }
161}
162
163/// What a line is, from the command the runner prefixed it with.
164#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
165#[serde(rename_all = "kebab-case")]
166pub enum LineKind {
167    /// What a step printed.
168    Output,
169    /// `##[group]`: a foldable group starts; the runs are its title.
170    Group,
171    /// `##[endgroup]`: the innermost open group ends.
172    EndGroup,
173    /// `##[error]`.
174    Error,
175    /// `##[warning]`.
176    Warning,
177    /// `##[notice]`.
178    Notice,
179    /// `##[debug]`.
180    Debug,
181    /// `##[command]`: a command an action ran, such as `/usr/bin/git version`.
182    Command,
183    /// A kind a newer daemon names; this client draws it as output.
184    #[serde(other)]
185    Unknown,
186}
187
188/// A colour a run of text was printed in, from the closed set a log is drawn with. Bright and dark shades are one
189/// colour; white, default and 256-colour or RGB colours beyond the first 16 have none.
190#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
191#[serde(rename_all = "kebab-case")]
192pub enum Color {
193    /// Red.
194    Red,
195    /// Green.
196    Green,
197    /// Yellow.
198    Yellow,
199    /// Blue.
200    Blue,
201    /// Magenta.
202    Magenta,
203    /// Cyan.
204    Cyan,
205    /// Black or bright black: dimmed text.
206    Grey,
207    /// A colour a newer daemon names; this client draws it uncoloured.
208    #[serde(other)]
209    Unknown,
210}
211
212#[expect(
213    clippy::trivially_copy_pass_by_ref,
214    reason = "serde's skip_serializing_if passes a reference"
215)]
216fn is_false(value: &bool) -> bool {
217    !value
218}
219
220/// A run of a line's text in one style.
221#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
222pub struct Run {
223    /// The text, without escapes or control characters but tabs.
224    pub text: String,
225    /// Its colour; none for the default.
226    #[serde(default, skip_serializing_if = "Option::is_none")]
227    pub color: Option<Color>,
228    /// Printed bold.
229    #[serde(default, skip_serializing_if = "is_false")]
230    pub bold: bool,
231    /// A search's match: the text the query found.
232    #[serde(default, skip_serializing_if = "is_false")]
233    pub mark: bool,
234}
235
236/// One stored line of a job's log, styled.
237#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
238pub struct StyledLine {
239    /// Its number in the stored log, from 0: the same as the plain log's.
240    pub line: u64,
241    /// When the runner wrote it, in seconds since the Unix epoch; absent for a line without a timestamp.
242    #[serde(default, skip_serializing_if = "Option::is_none")]
243    pub at: Option<u64>,
244    /// What it is.
245    pub kind: LineKind,
246    /// Its text after the timestamp and the command, in order: at most [`RUNS_MAX`] runs of the line's first
247    /// [`LINE_BYTES_MAX`] bytes.
248    pub runs: Vec<Run>,
249    /// Bytes of the stored line not shown: past [`LINE_BYTES_MAX`] or [`RUNS_MAX`], or past what a read took.
250    #[serde(default, skip_serializing_if = "is_zero")]
251    pub omitted: u64,
252}
253
254#[expect(
255    clippy::trivially_copy_pass_by_ref,
256    reason = "serde's skip_serializing_if passes a reference"
257)]
258fn is_zero(value: &u64) -> bool {
259    *value == 0
260}
261
262impl StyledLine {
263    /// Styles stored line number `line`, whose text is `text`: its first [`LINE_BYTES_MAX`] bytes, in at most
264    /// [`RUNS_MAX`] runs.
265    pub fn parse(line: u64, text: &str) -> StyledLine {
266        let shown = prefix(text, usize::try_from(LINE_BYTES_MAX).unwrap_or(usize::MAX));
267        let (at, rest) = split_time(shown);
268        let (kind, rest) = command(rest);
269        let (runs, read) = runs(rest, RUNS_MAX);
270        StyledLine {
271            line,
272            at: at.map(|ms| ms / 1000),
273            kind,
274            runs,
275            omitted: (text.len() - shown.len() + rest.len() - read) as u64,
276        }
277    }
278
279    /// Its text, without styles.
280    pub fn text(&self) -> String {
281        self.runs.iter().map(|run| run.text.as_str()).collect()
282    }
283
284    /// What a reader is told of the bytes it does not show: `80.0K not shown`; `None` when it shows them all. The
285    /// plain log (`GET /v1/jobs/{id}/logs`) has the whole line.
286    pub fn omitted_words(&self) -> Option<String> {
287        (self.omitted > 0).then(|| format!("{} not shown", crate::config::ByteSize(self.omitted)))
288    }
289}
290
291/// A line's timestamp in ms, when it starts with one as the runner writes it (`2026-10-23T01:08:56.1234567Z `),
292/// and the rest.
293fn split_time(text: &str) -> (Option<u64>, &str) {
294    match text.split_once(' ') {
295        Some((stamp, rest)) => match crate::time::parse_ms(stamp) {
296            Some(at) => (Some(at), rest),
297            None => (None, text),
298        },
299        None => match crate::time::parse_ms(text) {
300            Some(at) => (Some(at), ""),
301            None => (None, text),
302        },
303    }
304}
305
306/// A line's command, after its timestamp, and the rest; output for a command the runner does not name.
307fn command(text: &str) -> (LineKind, &str) {
308    let Some((name, rest)) = text
309        .strip_prefix(COMMAND)
310        .and_then(|after| after.split_once(']'))
311    else {
312        return (LineKind::Output, text);
313    };
314    let kind = match name {
315        "group" => LineKind::Group,
316        "endgroup" => LineKind::EndGroup,
317        "error" => LineKind::Error,
318        "warning" => LineKind::Warning,
319        "notice" => LineKind::Notice,
320        "debug" => LineKind::Debug,
321        "command" => LineKind::Command,
322        _ => return (LineKind::Output, text),
323    };
324    (kind, rest)
325}
326
327/// What a line is and its text after the timestamp and command, unstyled: what the outline reads.
328pub(crate) fn classify(text: &str) -> (LineKind, &str) {
329    command(split_time(text).1)
330}
331
332/// A line's time in seconds, when it has one.
333pub(crate) fn time_of(text: &str) -> Option<u64> {
334    split_time(text).0.map(|ms| ms / 1000)
335}
336
337/// Text without escapes or control characters but tabs, as a group's title or a search compares it.
338pub(crate) fn plain(text: &str) -> String {
339    runs(text, usize::MAX)
340        .0
341        .into_iter()
342        .map(|run| run.text)
343        .collect()
344}
345
346/// The style a run is printed in.
347#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
348struct Style {
349    color: Option<Color>,
350    bold: bool,
351}
352
353/// One of the eight basic colours, 0 to 7, as a log is drawn: white has none.
354fn basic(index: u16) -> Option<Color> {
355    match index {
356        0 => Some(Color::Grey),
357        1 => Some(Color::Red),
358        2 => Some(Color::Green),
359        3 => Some(Color::Yellow),
360        4 => Some(Color::Blue),
361        5 => Some(Color::Magenta),
362        6 => Some(Color::Cyan),
363        _ => None,
364    }
365}
366
367impl Style {
368    /// The style after an SGR sequence with `params`, such as `1;31`.
369    fn apply(self, params: &str) -> Style {
370        let codes: Vec<u16> = params
371            .split([';', ':'])
372            .map(|code| code.parse().unwrap_or(0))
373            .collect();
374        let mut style = self;
375        let mut codes = codes.into_iter();
376        while let Some(code) = codes.next() {
377            match code {
378                0 => style = Style::default(),
379                1 => style.bold = true,
380                22 => style.bold = false,
381                30..=37 => style.color = basic(code - 30),
382                90..=97 => style.color = basic(code - 90),
383                39 => style.color = None,
384                38 | 48 => {
385                    let color = match codes.next() {
386                        Some(5) => codes.next().and_then(|index| {
387                            basic(if index >= 8 { index - 8 } else { index }).filter(|_| index < 16)
388                        }),
389                        Some(2) => {
390                            codes.by_ref().take(3).for_each(drop);
391                            None
392                        }
393                        _ => None,
394                    };
395                    if code == 38 {
396                        style.color = color;
397                    }
398                }
399                _ => {}
400            }
401        }
402        style
403    }
404}
405
406/// A line's text as at most `max` runs, and how many of its bytes they cover: SGR sequences become styles, every
407/// other escape sequence and control character but a tab is dropped, and adjacent runs of one style are one.
408fn runs(text: &str, max: usize) -> (Vec<Run>, usize) {
409    let mut runs: Vec<Run> = Vec::new();
410    let mut style = Style::default();
411    let mut current = String::new();
412    // Where `current`'s text starts in `text`.
413    let mut began = 0;
414    let mut chars = text.char_indices().peekable();
415    // Adds `current` in `style`; false when that takes a run past `max`.
416    let flush = |runs: &mut Vec<Run>, current: &mut String, style: Style| {
417        if current.is_empty() {
418            return true;
419        }
420        let full = runs.len() >= max;
421        match runs.last_mut() {
422            Some(last) if last.color == style.color && last.bold == style.bold => {
423                last.text.push_str(current);
424                current.clear();
425            }
426            _ if full => return false,
427            _ => runs.push(Run {
428                text: std::mem::take(current),
429                color: style.color,
430                bold: style.bold,
431                mark: false,
432            }),
433        }
434        true
435    };
436    while let Some((at, c)) = chars.next() {
437        match c {
438            '\u{1b}' => match chars.next().map(|(_, c)| c) {
439                // CSI: parameters, then a final byte from `@` to `~`; only `m` (SGR) is kept, as a style.
440                Some('[') => {
441                    let mut params = String::new();
442                    let mut last = None;
443                    for (_, c) in chars.by_ref() {
444                        if ('@'..='~').contains(&c) {
445                            last = Some(c);
446                            break;
447                        }
448                        params.push(c);
449                    }
450                    if last == Some('m') {
451                        let next = style.apply(&params);
452                        if next != style {
453                            if !flush(&mut runs, &mut current, style) {
454                                return (runs, began);
455                            }
456                            style = next;
457                        }
458                    }
459                }
460                // OSC, such as a hyperlink or a title: up to BEL or ST.
461                Some(']') => {
462                    while let Some((_, c)) = chars.next() {
463                        if c == '\u{7}' {
464                            break;
465                        }
466                        if c == '\u{1b}' {
467                            chars.next_if(|&(_, c)| c == '\\');
468                            break;
469                        }
470                    }
471                }
472                // Any other escape is two characters.
473                _ => {}
474            },
475            c if c == '\t' || !c.is_control() => {
476                if current.is_empty() {
477                    began = at;
478                }
479                current.push(c);
480            }
481            _ => {}
482        }
483    }
484    if flush(&mut runs, &mut current, style) {
485        (runs, text.len())
486    } else {
487        (runs, began)
488    }
489}
490
491// The tests read the wire format, JSON, which only the host and client features read.
492#[cfg(test)]
493#[cfg(any(feature = "host", feature = "client"))]
494mod tests;