Skip to main content

runnerd_core/
record.rs

1//! Records of failed jobs: which runner containers may be kept, the files a record must never hold, which records go
2//! to keep within `[records]`, what a shell in a record shows first, and whether a failed job's log says its runner
3//! failed it (`runner_fault`). Pure functions: the engine
4//! (`scheduler::engine`) commits, lists and removes the images, and the daemon serves the shell.
5//!
6//! A record is the runner container's own layer, committed as the image `runnerd-record:<job>` once the runner
7//! exited: the workspace and everything else the job wrote outside the slot's volumes. When the runner exits, the
8//! runner image's entrypoint deletes what the job added or changed among the runner's dot-files and in the temporary
9//! directories, and the step scripts (`image/scrub.sh`); before a commit, `secrets_left` checks the container's
10//! changed paths and the engine keeps no record that still holds one.
11
12use std::collections::BTreeSet;
13use std::fmt::Write as _;
14
15use crate::api;
16use crate::config;
17
18/// The repository of every record image; the tag is the job's ID.
19pub const IMAGE: &str = "runnerd-record";
20
21/// The label every record image carries, with the job's ID.
22pub const LABEL: &str = "runnerd.record";
23
24/// The label every shell container carries, with the job's ID.
25pub const SHELL_LABEL: &str = "runnerd.shell";
26
27/// The image of a job's record.
28pub fn image(job: u64) -> String {
29    format!("{IMAGE}:{job}")
30}
31
32/// What the runner prints when its job failed. Its exit code says nothing about the job.
33const FAILED: &str = "completed with result: Failed";
34
35/// Whether a runner's output says its job failed.
36pub fn failed(runner_log: &str) -> bool {
37    runner_log.lines().any(|line| line.ends_with(FAILED))
38}
39
40/// The runner's home, whose dot-files (`.runner`, `.credentials`, `.gitconfig`, `.aws`, `.gnupg`, …) a job's changes
41/// must not leave in a record: the runner keeps its registration and credentials there, and tools their logins and
42/// keys.
43const HOME: &str = "/home/runner/";
44
45/// Directories nothing a job changed in may be kept: the JIT config's, the step scripts' (`GITHUB_ENV` and the
46/// credentials `actions/checkout` and `setup-node` leave), and the temporary directories, where a job may put a key
47/// or a kubeconfig.
48const SECRET_DIRS: &[&str] = &[
49    "/run/runnerd",
50    "/home/runner/_work/_temp",
51    "/tmp",
52    "/var/tmp",
53];
54
55/// Directories among the runner's dot-files a record keeps, as the runner image's scrub does: the Rust toolchains
56/// rustup and cargo install (`dtolnay/rust-toolchain`), which a shell needs to re-run a step, and cargo's downloaded
57/// crates and git checkouts, which hold nothing a workspace's `Cargo.lock` does not name. Not `~/.cargo`'s
58/// `credentials.toml` or `config.toml`, which may hold a registry's token.
59const HARMLESS: &[&str] = &[
60    "/home/runner/.rustup",
61    "/home/runner/.cargo/bin",
62    "/home/runner/.cargo/registry",
63    "/home/runner/.cargo/git",
64];
65
66/// Whether `path` is in a `HARMLESS` directory, or is one of them or a parent of one that `changed` shows a path in:
67/// a directory's own entry holds nothing.
68fn harmless(path: &str, changed: &[&str]) -> bool {
69    let below = |dir: &str, path: &str| {
70        path.strip_prefix(dir)
71            .is_some_and(|rest| rest.starts_with('/'))
72    };
73    let on_the_way = HARMLESS.iter().any(|dir| *dir == path || below(path, dir));
74    HARMLESS.iter().any(|dir| below(dir, path))
75        || on_the_way && changed.iter().any(|other| below(path, other))
76}
77
78/// The changed paths of a runner container that a record must not hold: anything added or modified among the
79/// runner's dot-files but the `HARMLESS` toolchains, in the temporary directories, or where the runner keeps the
80/// JIT config and step scripts. A record of it must not be made. `changed` are the paths Docker reports added or
81/// modified.
82pub fn secrets_left<'path>(changed: impl IntoIterator<Item = &'path str>) -> Vec<String> {
83    let changed: Vec<&str> = changed.into_iter().collect();
84    changed
85        .iter()
86        .copied()
87        .filter(|path| !harmless(path, &changed))
88        .filter(|path| {
89            path.strip_prefix(HOME)
90                .is_some_and(|rest| rest.starts_with('.'))
91                || SECRET_DIRS.iter().any(|dir| {
92                    path.strip_prefix(dir)
93                        .is_some_and(|rest| rest.starts_with('/'))
94                })
95        })
96        .map(str::to_owned)
97        .collect()
98}
99
100/// The first few of `paths`, and how many more there are: a container without the scrub may have changed thousands.
101pub fn listed(paths: &[String]) -> String {
102    const SHOWN: usize = 5;
103    let mut text = paths
104        .iter()
105        .take(SHOWN)
106        .map(String::as_str)
107        .collect::<Vec<_>>()
108        .join(", ");
109    if paths.len() > SHOWN {
110        write!(text, " and {} more", paths.len() - SHOWN).ok();
111    }
112    text
113}
114
115/// Which records go, and why, before a new record of `incoming` bytes is made; or why it is not made. Records in
116/// `open` have a shell open and stay. `free` is the disk Docker has left, when known. Expired records go too.
117pub fn make_room(
118    records: &[api::Record],
119    open: &BTreeSet<u64>,
120    limits: &config::Records,
121    incoming: u64,
122    free: Option<u64>,
123    now: u64,
124) -> Result<Vec<(u64, api::RecordRemoval)>, String> {
125    if incoming > limits.max_total.0 {
126        return Err(format!(
127            "it would take {}, more than records.max_total ({})",
128            config::ByteSize(incoming),
129            limits.max_total
130        ));
131    }
132    let mut removed = expired(records, open, limits, now);
133    let mut kept: Vec<&api::Record> = records
134        .iter()
135        .filter(|record| !removed.iter().any(|(job, _)| *job == record.job))
136        .collect();
137    kept.sort_by_key(|record| (record.created_at, record.job));
138    let mut removable = kept
139        .clone()
140        .into_iter()
141        .filter(|record| !open.contains(&record.job));
142    let max_count = usize::try_from(limits.max_count).unwrap_or(usize::MAX);
143    let mut count = kept.len();
144    let mut total: u64 = kept.iter().map(|record| record.bytes).sum();
145    while count >= max_count || total + incoming > limits.max_total.0 {
146        let why = if count >= max_count {
147            api::RecordRemoval::Count
148        } else {
149            api::RecordRemoval::Size
150        };
151        let Some(oldest) = removable.next() else {
152            return Err("the records kept within the limits have shells open".to_owned());
153        };
154        removed.push((oldest.job, why));
155        count -= 1;
156        total -= oldest.bytes;
157    }
158    if let Some(free) = free {
159        let freed: u64 = records
160            .iter()
161            .filter(|record| removed.iter().any(|(job, _)| *job == record.job))
162            .map(|record| record.bytes)
163            .sum();
164        if (free + freed).saturating_sub(incoming) < limits.min_free.0 {
165            return Err(format!(
166                "Docker has {} free; a record of {} would leave less than records.min_free ({})",
167                config::ByteSize(free),
168                config::ByteSize(incoming),
169                limits.min_free
170            ));
171        }
172    }
173    Ok(removed)
174}
175
176/// The records older than `max_age` at `now`, but those with a shell open.
177pub fn expired(
178    records: &[api::Record],
179    open: &BTreeSet<u64>,
180    limits: &config::Records,
181    now: u64,
182) -> Vec<(u64, api::RecordRemoval)> {
183    records
184        .iter()
185        .filter(|record| {
186            !open.contains(&record.job)
187                && now.saturating_sub(record.created_at) > limits.max_age.as_secs()
188        })
189        .map(|record| (record.job, api::RecordRemoval::Age))
190        .collect()
191}
192
193/// The directory a job's steps run in by default: `GITHUB_WORKSPACE`.
194pub fn workspace(repository: &str) -> String {
195    let name = repository.rsplit('/').next().unwrap_or(repository);
196    format!("/home/runner/_work/{name}/{name}")
197}
198
199/// What a shell in a job's record prints before its prompt: the job, the step (`step`, or the one that failed),
200/// what the step ran as its log shows it, and what the record lacks. `log` is the job's stored log. Lines end in
201/// `\r\n`, for a terminal in raw mode.
202pub fn banner(job: &api::Job, log: &[String], step: Option<u32>) -> String {
203    let summary = &job.summary;
204    let mut text = format!(
205        "runnerd: the record of job {} of {}",
206        summary.id, summary.repository
207    );
208    match &summary.workflow {
209        Some(workflow) => write!(text, ", {workflow} / {}", summary.name),
210        None => write!(text, ", {}", summary.name),
211    }
212    .ok();
213    text.push('\n');
214    let failed = job
215        .steps
216        .iter()
217        .find(|candidate| candidate.conclusion.as_deref() == Some("failure"));
218    let shown = match step {
219        Some(number) => job
220            .steps
221            .iter()
222            .find(|candidate| candidate.number == number),
223        None => failed,
224    };
225    let sections = sections(log);
226    match shown {
227        Some(shown) => {
228            let verb = if failed.is_some_and(|failed| failed.number == shown.number) {
229                "failed at"
230            } else {
231                "shows"
232            };
233            write!(text, "It {verb} step {}: {}", shown.number, shown.name).ok();
234            let section = section_of(job, &sections, shown);
235            if let Some(code) = section.and_then(|section| section.exit_code) {
236                write!(text, " (exit code {code})").ok();
237            }
238            text.push('\n');
239            match section {
240                Some(section) => {
241                    text.push_str("Its log shows it ran:\n");
242                    for line in &section.lines {
243                        writeln!(text, "    {line}").ok();
244                    }
245                }
246                None => text.push_str("Its log does not show what it ran.\n"),
247            }
248        }
249        None if step.is_some() => text.push_str("The job has no such step.\n"),
250        None => text.push_str("The job's steps do not say which one failed.\n"),
251    }
252    writeln!(
253        text,
254        "You are in {}, as the job left it. Not kept: the job's Docker containers and images; what it changed \
255         in ~/.cache, the tool cache and the Android SDK, which show the profile's snapshot; and what it added or \
256         changed among your home's dot-files and in /tmp (logins, and tools in ~/.local or ~/.nvm), except \
257         ~/.rustup and ~/.cargo's bin, registry and git. Exit to remove this container.",
258        workspace(&summary.repository)
259    )
260    .ok();
261    text.replace('\n', "\r\n")
262}
263
264/// A top-level `##[group]Run …` section of a job's log: one step that ran.
265#[derive(Debug, Default)]
266struct Section {
267    /// Its title without `Run `.
268    title: String,
269    /// The group's lines: what the step ran, its shell and its environment.
270    lines: Vec<String>,
271    /// What the step printed after its group, up to `Post job cleanup.`.
272    output: Vec<String>,
273    /// From `##[error]Process completed with exit code N.`.
274    exit_code: Option<i64>,
275}
276
277/// The `##[group]Run …` sections of a log, in order.
278fn sections(log: &[String]) -> Vec<Section> {
279    let mut sections: Vec<Section> = Vec::new();
280    let (mut in_group, mut cleaning_up) = (false, false);
281    for line in log {
282        let text = plain(line);
283        if let Some(title) = text.strip_prefix("##[group]Run ") {
284            sections.push(Section {
285                title: title.to_owned(),
286                ..Section::default()
287            });
288            (in_group, cleaning_up) = (true, false);
289        } else if text.starts_with("##[endgroup]") {
290            in_group = false;
291        } else if text == "Post job cleanup." {
292            // The post steps print after the last step, and are none of its.
293            cleaning_up = true;
294        } else if let Some(section) = sections.last_mut().filter(|_| !cleaning_up) {
295            if in_group {
296                section.lines.push(text);
297            } else {
298                if let Some(code) = text
299                    .strip_prefix("##[error]Process completed with exit code ")
300                    .and_then(|code| code.strip_suffix('.'))
301                    .and_then(|code| code.parse().ok())
302                {
303                    section.exit_code.get_or_insert(code);
304                }
305                section.output.push(text);
306            }
307        }
308    }
309    sections
310}
311
312/// The section of a step: the one titled as the step is named (`Run cargo test`), else for a failed step the one
313/// that ended with an exit code, else the one at the step's place among the steps that ran after `Set up job`.
314fn section_of<'log>(
315    job: &api::Job,
316    sections: &'log [Section],
317    step: &api::Step,
318) -> Option<&'log Section> {
319    if let Some(title) = step.name.strip_prefix("Run ")
320        && let Some(section) = sections.iter().find(|section| section.title == title)
321    {
322        return Some(section);
323    }
324    if step.conclusion.as_deref() == Some("failure")
325        && let Some(section) = sections.iter().rfind(|section| section.exit_code.is_some())
326    {
327        return Some(section);
328    }
329    let place = job
330        .steps
331        .iter()
332        .filter(|candidate| {
333            candidate.number > 1
334                && !matches!(candidate.conclusion.as_deref(), Some("skipped") | None)
335        })
336        .position(|candidate| candidate.number == step.number)?;
337    sections.get(place)
338}
339
340/// Whether a failed job's log says its runner failed it, not its code: the output of the step GitHub reports as
341/// `failure` has a line with `function not implemented` on a mount, umount or openat2, or Sysbox's `unsafe procfs
342/// detected … invalid argument`. Sysbox answers a trapped call with `ENOSYS` once it lost the runner's seccomp
343/// listener, and with `EINVAL` when a descriptor it injected was interrupted; no retry in the same runner helps, and
344/// the job's code is not at fault. The step is found as `banner` finds the failed step's section, and only what it
345/// printed counts: not the script and environment the runner echoes before it, not the output of a step that
346/// passed or was allowed to fail, nor what a post step printed. A step that prints its own `::group::Run …` line cuts
347/// its output short there. `log` is the stored lines.
348pub fn runner_fault(job: &api::Job, log: &[String]) -> bool {
349    let Some(failed) = job
350        .steps
351        .iter()
352        .find(|step| step.conclusion.as_deref() == Some("failure"))
353    else {
354        return false;
355    };
356    section_of(job, &sections(log), failed)
357        .is_some_and(|section| section.output.iter().any(|line| sysbox_fault(line)))
358}
359
360/// What failed a failed job apart from its code, as its log shows it, read only where the runner reports it
361/// (actions/runner 2.338's `ActionManager` and `JobExtension`):
362///
363/// - [`api::FailureCause::ActionArchive`]: `Set up job` failed, and the last error it printed, the one the runner ends
364///   it with, says an action could not be downloaded or unpacked (`Failed to download archive '…' after 3 attempts.`,
365///   `Can't use 'tar -xzf'`, `Can't un-zip archive file`, a download that timed out).
366/// - [`api::FailureCause::ActionResolve`]: `Set up job` failed, and that last error says an action could not be
367///   resolved (`Failed to resolve action download info.`, `Unable to resolve action`).
368/// - [`api::FailureCause::Checkout`]: the step that failed is `actions/checkout`, by its own section of the log: the
369///   section at its place among the job's steps, when the log has one section a step, that runs the action, not a
370///   script, and that printed the checkout's `Syncing repository:` and an error.
371///
372/// A warning a retry printed is no cause: a retry that succeeded prints the same, and what fails after it is not
373/// this. `None` for anything else: a job that did not fail, a post step, or a log that does not say. `log` is the
374/// stored lines.
375pub fn failure_cause(job: &api::Job, log: &[String]) -> Option<api::FailureCause> {
376    let failed = job
377        .steps
378        .iter()
379        .find(|step| step.conclusion.as_deref() == Some("failure"))?;
380    if failed.name == SET_UP {
381        return setup_failure(&setup_lines(log));
382    }
383    let section = own_section(job, log, failed)?;
384    let runs_checkout = section
385        .title
386        .to_ascii_lowercase()
387        .strip_prefix("actions/checkout@")
388        .is_some_and(|reference| !reference.is_empty() && !reference.contains(' '));
389    let script = section
390        .lines
391        .iter()
392        .any(|line| line.trim_start().starts_with("shell: "));
393    let fetched = section
394        .output
395        .iter()
396        .any(|line| line.starts_with("Syncing repository: "));
397    let errored = section
398        .output
399        .iter()
400        .any(|line| line.starts_with("##[error]"));
401    (runs_checkout && !script && fetched && errored).then_some(api::FailureCause::Checkout)
402}
403
404/// The name GitHub gives the step a job sets up in: the runner prepares its actions there.
405const SET_UP: &str = "Set up job";
406
407/// The plain lines a log has before its first step's group: what `Set up job` printed.
408fn setup_lines(log: &[String]) -> Vec<String> {
409    log.iter()
410        .map(|line| plain(line))
411        .take_while(|text| !text.starts_with("##[group]Run ") && text != "Post job cleanup.")
412        .collect()
413}
414
415/// The cause of a failed `Set up job`, from the last error it printed (`failure_cause`).
416fn setup_failure(lines: &[String]) -> Option<api::FailureCause> {
417    let error = lines
418        .iter()
419        .rev()
420        .find(|text| text.starts_with("##[error]"))?;
421    let archive = [
422        "Failed to download archive '",
423        "Failed to download action '",
424        "Can't use 'tar -xzf'",
425        "Can't un-zip archive file",
426        "download has timed out",
427    ];
428    let resolve = [
429        "Failed to resolve action download info",
430        "Unable to resolve action",
431    ];
432    if archive.iter().any(|sign| error.contains(sign)) {
433        Some(api::FailureCause::ActionArchive)
434    } else if resolve.iter().any(|sign| error.contains(sign)) {
435        Some(api::FailureCause::ActionResolve)
436    } else {
437        None
438    }
439}
440
441/// The section of the log a failed main step printed, by its place: only when the log has exactly one `##[group]Run`
442/// section for each step that ran between `Set up job` and the post steps, as a composite action's inner steps or a
443/// step's own groups would shift the places. `None` for a post step, or when the places do not line up.
444fn own_section(job: &api::Job, log: &[String], failed: &api::Step) -> Option<Section> {
445    if failed.name.starts_with("Post ") {
446        return None;
447    }
448    let main: Vec<&api::Step> = job
449        .steps
450        .iter()
451        .filter(|step| {
452            step.name != SET_UP
453                && !step.name.starts_with("Post ")
454                && !step.name.starts_with("Complete job")
455                && !matches!(step.conclusion.as_deref(), Some("skipped") | None)
456        })
457        .collect();
458    let place = main.iter().position(|step| step.number == failed.number)?;
459    let mut sections = sections(&main_lines(log));
460    if sections.len() != main.len() {
461        return None;
462    }
463    Some(sections.swap_remove(place))
464}
465
466/// The lines of a log before its post steps: everything up to its first `Post job cleanup.`.
467fn main_lines(log: &[String]) -> Vec<String> {
468    log.iter()
469        .take_while(|line| plain(line) != "Post job cleanup.")
470        .cloned()
471        .collect()
472}
473
474/// What a call's name is in a line of Sysbox's trouble: the words of `mount`'s family and `openat2`.
475const FAULT_CALLS: [&str; 6] = [
476    "mount", "umount", "umount2", "unmount", "mounting", "openat2",
477];
478
479/// Whether a log line is what a runner whose Sysbox lost its seccomp listener, or failed an injection, prints. The
480/// call is a word of its own, between spaces, colons, brackets, quotes and commas: a path or a name that holds
481/// `mount`, as `amount-api`, `mounted`, `mount-api/a.txt`, `mount.c` or `test_mount`, is not one.
482fn sysbox_fault(text: &str) -> bool {
483    let text = text.to_lowercase();
484    (text.contains("function not implemented")
485        && text
486            .split(|c: char| c.is_whitespace() || ":()\"',".contains(c))
487            .any(|word| FAULT_CALLS.contains(&word)))
488        || (text.contains("unsafe procfs detected") && text.contains("invalid argument"))
489}
490
491/// The repositories `lines`, stored lines of a job's log, show the runner downloading as an immutable action package
492/// (actions/runner 2.338's `##[group]Download immutable action package 'owner/repo@ref'`), each once, as written.
493/// The runner prints it before it looks in an action archive cache, so it shows a package resolution whether or not
494/// it then read an archive in the package's place.
495pub fn immutable_packages(lines: &[String]) -> Vec<String> {
496    let mut found: Vec<String> = Vec::new();
497    for line in lines {
498        let text = plain(line);
499        let Some(rest) = text.strip_prefix("##[group]Download immutable action package '") else {
500            continue;
501        };
502        let Some((name, _)) = rest.split_once('@') else {
503            continue;
504        };
505        let mut parts = name.split('/');
506        let (Some(owner), Some(repo)) = (parts.next(), parts.next()) else {
507            continue;
508        };
509        let repository = format!("{owner}/{repo}");
510        if !found.contains(&repository) {
511            found.push(repository);
512        }
513    }
514    found
515}
516
517/// A stored log line without its timestamp and colour codes.
518fn plain(line: &str) -> String {
519    let text = match line.split_once(' ') {
520        Some((stamp, rest)) if stamp.ends_with('Z') && stamp.contains('T') => rest,
521        _ => line,
522    };
523    let mut plain = String::with_capacity(text.len());
524    let mut chars = text.chars();
525    while let Some(c) = chars.next() {
526        if c == '\u{1b}' {
527            // An SGR sequence: ESC [ … m.
528            for c in chars.by_ref() {
529                if c == 'm' {
530                    break;
531                }
532            }
533        } else {
534            plain.push(c);
535        }
536    }
537    plain
538}
539
540#[cfg(test)]
541mod tests;