Skip to main content

runnerd_core/
record.rs

1//! Records of failed jobs: which runner containers may be kept, the files a record must never hold, which records go
2//! to keep within `[records]`, what a shell in a record shows first, and whether a failed job's log says its runner
3//! failed it (`runner_fault`). Pure functions: the engine
4//! (`scheduler::engine`) commits, lists and removes the images, and the daemon serves the shell.
5//!
6//! A record is the runner container's own layer, committed as the image `runnerd-record:<job>` once the runner
7//! exited: the workspace and everything else the job wrote outside the slot's volumes. When the runner exits, the
8//! runner image's entrypoint deletes what the job added or changed among the runner's dot-files and in the temporary
9//! directories, and the step scripts (`image/scrub.sh`); before a commit, `secrets_left` checks the container's
10//! changed paths and the engine keeps no record that still holds one.
11
12use std::collections::BTreeSet;
13use std::fmt::Write as _;
14
15use crate::api;
16use crate::config;
17
18/// The repository of every record image; the tag is the job's ID.
19pub const IMAGE: &str = "runnerd-record";
20
21/// The label every record image carries, with the job's ID.
22pub const LABEL: &str = "runnerd.record";
23
24/// The label every shell container carries, with the job's ID.
25pub const SHELL_LABEL: &str = "runnerd.shell";
26
27/// The image of a job's record.
28pub fn image(job: u64) -> String {
29    format!("{IMAGE}:{job}")
30}
31
32/// What the runner prints when its job failed. Its exit code says nothing about the job.
33const FAILED: &str = "completed with result: Failed";
34
35/// Whether a runner's output says its job failed.
36pub fn failed(runner_log: &str) -> bool {
37    runner_log.lines().any(|line| line.ends_with(FAILED))
38}
39
40/// The runner's home, whose dot-files (`.runner`, `.credentials`, `.gitconfig`, `.aws`, `.gnupg`, …) a job's changes
41/// must not leave in a record: the runner keeps its registration and credentials there, and tools their logins and
42/// keys.
43const HOME: &str = "/home/runner/";
44
45/// Directories nothing a job changed in may be kept: the JIT config's, the step scripts' (`GITHUB_ENV` and the
46/// credentials `actions/checkout` and `setup-node` leave), and the temporary directories, where a job may put a key
47/// or a kubeconfig.
48const SECRET_DIRS: &[&str] = &[
49    "/run/runnerd",
50    "/home/runner/_work/_temp",
51    "/tmp",
52    "/var/tmp",
53];
54
55/// Directories among the runner's dot-files a record keeps, as the runner image's scrub does: the Rust toolchains
56/// rustup and cargo install (`dtolnay/rust-toolchain`), which a shell needs to re-run a step, and cargo's downloaded
57/// crates and git checkouts, which hold nothing a workspace's `Cargo.lock` does not name. Not `~/.cargo`'s
58/// `credentials.toml` or `config.toml`, which may hold a registry's token.
59const HARMLESS: &[&str] = &[
60    "/home/runner/.rustup",
61    "/home/runner/.cargo/bin",
62    "/home/runner/.cargo/registry",
63    "/home/runner/.cargo/git",
64];
65
66/// Whether `path` is in a `HARMLESS` directory, or is one of them or a parent of one that `changed` shows a path in:
67/// a directory's own entry holds nothing.
68fn harmless(path: &str, changed: &[&str]) -> bool {
69    let below = |dir: &str, path: &str| {
70        path.strip_prefix(dir)
71            .is_some_and(|rest| rest.starts_with('/'))
72    };
73    let on_the_way = HARMLESS.iter().any(|dir| *dir == path || below(path, dir));
74    HARMLESS.iter().any(|dir| below(dir, path))
75        || on_the_way && changed.iter().any(|other| below(path, other))
76}
77
78/// The changed paths of a runner container that a record must not hold: anything added or modified among the
79/// runner's dot-files but the `HARMLESS` toolchains, in the temporary directories, or where the runner keeps the
80/// JIT config and step scripts. A record of it must not be made. `changed` are the paths Docker reports added or
81/// modified.
82pub fn secrets_left<'path>(changed: impl IntoIterator<Item = &'path str>) -> Vec<String> {
83    let changed: Vec<&str> = changed.into_iter().collect();
84    changed
85        .iter()
86        .copied()
87        .filter(|path| !harmless(path, &changed))
88        .filter(|path| {
89            path.strip_prefix(HOME)
90                .is_some_and(|rest| rest.starts_with('.'))
91                || SECRET_DIRS.iter().any(|dir| {
92                    path.strip_prefix(dir)
93                        .is_some_and(|rest| rest.starts_with('/'))
94                })
95        })
96        .map(str::to_owned)
97        .collect()
98}
99
100/// The first few of `paths`, and how many more there are: a container without the scrub may have changed thousands.
101pub fn listed(paths: &[String]) -> String {
102    const SHOWN: usize = 5;
103    let mut text = paths
104        .iter()
105        .take(SHOWN)
106        .map(String::as_str)
107        .collect::<Vec<_>>()
108        .join(", ");
109    if paths.len() > SHOWN {
110        write!(text, " and {} more", paths.len() - SHOWN).ok();
111    }
112    text
113}
114
115/// Which records go, and why, before a new record of `incoming` bytes is made; or why it is not made. Records in
116/// `open` have a shell open and stay. `free` is the disk Docker has left, when known. Expired records go too.
117pub fn make_room(
118    records: &[api::Record],
119    open: &BTreeSet<u64>,
120    limits: &config::Records,
121    incoming: u64,
122    free: Option<u64>,
123    now: u64,
124) -> Result<Vec<(u64, api::RecordRemoval)>, String> {
125    if incoming > limits.max_total.0 {
126        return Err(format!(
127            "it would take {}, more than records.max_total ({})",
128            config::ByteSize(incoming),
129            limits.max_total
130        ));
131    }
132    let mut removed = expired(records, open, limits, now);
133    let mut kept: Vec<&api::Record> = records
134        .iter()
135        .filter(|record| !removed.iter().any(|(job, _)| *job == record.job))
136        .collect();
137    kept.sort_by_key(|record| (record.created_at, record.job));
138    let mut removable = kept
139        .clone()
140        .into_iter()
141        .filter(|record| !open.contains(&record.job));
142    let max_count = usize::try_from(limits.max_count).unwrap_or(usize::MAX);
143    let mut count = kept.len();
144    let mut total: u64 = kept.iter().map(|record| record.bytes).sum();
145    while count >= max_count || total + incoming > limits.max_total.0 {
146        let why = if count >= max_count {
147            api::RecordRemoval::Count
148        } else {
149            api::RecordRemoval::Size
150        };
151        let Some(oldest) = removable.next() else {
152            return Err("the records kept within the limits have shells open".to_owned());
153        };
154        removed.push((oldest.job, why));
155        count -= 1;
156        total -= oldest.bytes;
157    }
158    if let Some(free) = free {
159        let freed: u64 = records
160            .iter()
161            .filter(|record| removed.iter().any(|(job, _)| *job == record.job))
162            .map(|record| record.bytes)
163            .sum();
164        if (free + freed).saturating_sub(incoming) < limits.min_free.0 {
165            return Err(format!(
166                "Docker has {} free; a record of {} would leave less than records.min_free ({})",
167                config::ByteSize(free),
168                config::ByteSize(incoming),
169                limits.min_free
170            ));
171        }
172    }
173    Ok(removed)
174}
175
176/// The records older than `max_age` at `now`, but those with a shell open.
177pub fn expired(
178    records: &[api::Record],
179    open: &BTreeSet<u64>,
180    limits: &config::Records,
181    now: u64,
182) -> Vec<(u64, api::RecordRemoval)> {
183    records
184        .iter()
185        .filter(|record| {
186            !open.contains(&record.job)
187                && now.saturating_sub(record.created_at) > limits.max_age.as_secs()
188        })
189        .map(|record| (record.job, api::RecordRemoval::Age))
190        .collect()
191}
192
193/// The directory a job's steps run in by default: `GITHUB_WORKSPACE`.
194pub fn workspace(repository: &str) -> String {
195    let name = repository.rsplit('/').next().unwrap_or(repository);
196    format!("/home/runner/_work/{name}/{name}")
197}
198
199/// What a shell in a job's record prints before its prompt: the job, the step (`step`, or the one that failed),
200/// what the step ran as its log shows it, and what the record lacks. `log` is the job's stored log. Lines end in
201/// `\r\n`, for a terminal in raw mode.
202pub fn banner(job: &api::Job, log: &[String], step: Option<u32>) -> String {
203    let summary = &job.summary;
204    let mut text = format!(
205        "runnerd: the record of job {} of {}",
206        summary.id, summary.repository
207    );
208    match &summary.workflow {
209        Some(workflow) => write!(text, ", {workflow} / {}", summary.name),
210        None => write!(text, ", {}", summary.name),
211    }
212    .ok();
213    text.push('\n');
214    let failed = job
215        .steps
216        .iter()
217        .find(|candidate| candidate.conclusion.as_deref() == Some("failure"));
218    let shown = match step {
219        Some(number) => job
220            .steps
221            .iter()
222            .find(|candidate| candidate.number == number),
223        None => failed,
224    };
225    let sections = sections(log);
226    match shown {
227        Some(shown) => {
228            let verb = if failed.is_some_and(|failed| failed.number == shown.number) {
229                "failed at"
230            } else {
231                "shows"
232            };
233            write!(text, "It {verb} step {}: {}", shown.number, shown.name).ok();
234            let section = section_of(job, &sections, shown);
235            if let Some(code) = section.and_then(|section| section.exit_code) {
236                write!(text, " (exit code {code})").ok();
237            }
238            text.push('\n');
239            match section {
240                Some(section) => {
241                    text.push_str("Its log shows it ran:\n");
242                    for line in &section.lines {
243                        writeln!(text, "    {line}").ok();
244                    }
245                }
246                None => text.push_str("Its log does not show what it ran.\n"),
247            }
248        }
249        None if step.is_some() => text.push_str("The job has no such step.\n"),
250        None => text.push_str("The job's steps do not say which one failed.\n"),
251    }
252    writeln!(
253        text,
254        "You are in {}, as the job left it. Not kept: the job's Docker containers and images; what it changed \
255         in ~/.cache, the tool cache and the Android SDK, which show the profile's snapshot; and what it added or \
256         changed among your home's dot-files and in /tmp (logins, and tools in ~/.local or ~/.nvm), except \
257         ~/.rustup and ~/.cargo's bin, registry and git. Exit to remove this container.",
258        workspace(&summary.repository)
259    )
260    .ok();
261    text.replace('\n', "\r\n")
262}
263
264/// A top-level `##[group]Run …` section of a job's log: one step that ran.
265#[derive(Debug, Default)]
266struct Section {
267    /// Its title without `Run `.
268    title: String,
269    /// The group's lines: what the step ran, its shell and its environment.
270    lines: Vec<String>,
271    /// What the step printed after its group, up to `Post job cleanup.`.
272    output: Vec<String>,
273    /// From `##[error]Process completed with exit code N.`.
274    exit_code: Option<i64>,
275}
276
277/// The `##[group]Run …` sections of a log, in order.
278fn sections(log: &[String]) -> Vec<Section> {
279    let mut sections: Vec<Section> = Vec::new();
280    let (mut in_group, mut cleaning_up) = (false, false);
281    for line in log {
282        let text = plain(line);
283        if let Some(title) = text.strip_prefix("##[group]Run ") {
284            sections.push(Section {
285                title: title.to_owned(),
286                ..Section::default()
287            });
288            (in_group, cleaning_up) = (true, false);
289        } else if text.starts_with("##[endgroup]") {
290            in_group = false;
291        } else if text == "Post job cleanup." {
292            // The post steps print after the last step, and are none of its.
293            cleaning_up = true;
294        } else if let Some(section) = sections.last_mut().filter(|_| !cleaning_up) {
295            if in_group {
296                section.lines.push(text);
297            } else {
298                if let Some(code) = text
299                    .strip_prefix("##[error]Process completed with exit code ")
300                    .and_then(|code| code.strip_suffix('.'))
301                    .and_then(|code| code.parse().ok())
302                {
303                    section.exit_code.get_or_insert(code);
304                }
305                section.output.push(text);
306            }
307        }
308    }
309    sections
310}
311
312/// The section of a step: the one titled as the step is named (`Run cargo test`), else for a failed step the one
313/// that ended with an exit code, else the one at the step's place among the steps that ran after `Set up job`.
314fn section_of<'log>(
315    job: &api::Job,
316    sections: &'log [Section],
317    step: &api::Step,
318) -> Option<&'log Section> {
319    if let Some(title) = step.name.strip_prefix("Run ")
320        && let Some(section) = sections.iter().find(|section| section.title == title)
321    {
322        return Some(section);
323    }
324    if step.conclusion.as_deref() == Some("failure")
325        && let Some(section) = sections.iter().rfind(|section| section.exit_code.is_some())
326    {
327        return Some(section);
328    }
329    let place = job
330        .steps
331        .iter()
332        .filter(|candidate| {
333            candidate.number > 1
334                && !matches!(candidate.conclusion.as_deref(), Some("skipped") | None)
335        })
336        .position(|candidate| candidate.number == step.number)?;
337    sections.get(place)
338}
339
340/// Whether a failed job's log says its runner failed it, not its code: the output of the step GitHub reports as
341/// `failure` has a line with `function not implemented` on a mount, umount or openat2, or Sysbox's `unsafe procfs
342/// detected … invalid argument`. Sysbox answers a trapped call with `ENOSYS` once it lost the runner's seccomp
343/// listener, and with `EINVAL` when a descriptor it injected was interrupted; no retry in the same runner helps, and
344/// the job's code is not at fault. The step is found as `banner` finds the failed step's section, and only what it
345/// printed counts: not the script and environment the runner echoes before it, not the output of a step that
346/// passed or was allowed to fail, nor what a post step printed. A step that prints its own `::group::Run …` line cuts
347/// its output short there. `log` is the stored lines.
348pub fn runner_fault(job: &api::Job, log: &[String]) -> bool {
349    let Some(failed) = job
350        .steps
351        .iter()
352        .find(|step| step.conclusion.as_deref() == Some("failure"))
353    else {
354        return false;
355    };
356    section_of(job, &sections(log), failed)
357        .is_some_and(|section| section.output.iter().any(|line| sysbox_fault(line)))
358}
359
360/// What a call's name is in a line of Sysbox's trouble: the words of `mount`'s family and `openat2`.
361const FAULT_CALLS: [&str; 6] = [
362    "mount", "umount", "umount2", "unmount", "mounting", "openat2",
363];
364
365/// Whether a log line is what a runner whose Sysbox lost its seccomp listener, or failed an injection, prints. The
366/// call is a word of its own, between spaces, colons, brackets, quotes and commas: a path or a name that holds
367/// `mount`, as `amount-api`, `mounted`, `mount-api/a.txt`, `mount.c` or `test_mount`, is not one.
368fn sysbox_fault(text: &str) -> bool {
369    let text = text.to_lowercase();
370    (text.contains("function not implemented")
371        && text
372            .split(|c: char| c.is_whitespace() || ":()\"',".contains(c))
373            .any(|word| FAULT_CALLS.contains(&word)))
374        || (text.contains("unsafe procfs detected") && text.contains("invalid argument"))
375}
376
377/// A stored log line without its timestamp and colour codes.
378fn plain(line: &str) -> String {
379    let text = match line.split_once(' ') {
380        Some((stamp, rest)) if stamp.ends_with('Z') && stamp.contains('T') => rest,
381        _ => line,
382    };
383    let mut plain = String::with_capacity(text.len());
384    let mut chars = text.chars();
385    while let Some(c) = chars.next() {
386        if c == '\u{1b}' {
387            // An SGR sequence: ESC [ … m.
388            for c in chars.by_ref() {
389                if c == 'm' {
390                    break;
391                }
392            }
393        } else {
394            plain.push(c);
395        }
396    }
397    plain
398}
399
400#[cfg(test)]
401mod tests;