isothrum_runner/
checks.rs

1//! The run's checks: the manifest's `setup` steps, then each runner's step under its own timeout
2//! (Isoloom computes it from the checks' waits, so a long-waiting run isn't cut off at the
3//! launcher's old 5-minute cap, R11), its lines read with the `isoloom-check/1` grammar, then the
4//! `teardown`. A check its runner never reported fails; a runner with no `END` is unfinished.
5
6use isoloom_manifest::{Check, CheckRunner};
7use serde::{Deserialize, Serialize};
8
9use crate::Result;
10use crate::error::Error;
11use crate::events::EventKind;
12use crate::events::WarningCode;
13use crate::executor::{Cx, run_argv, run_steps};
14
15/// One check's outcome.
16#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
17#[non_exhaustive]
18pub struct CheckResult {
19    /// The runner that ran it.
20    pub runner: String,
21    /// Where it ran from (`from web`).
22    pub from: String,
23    /// The check's name.
24    pub name: String,
25    /// Whether it passed.
26    pub ok: bool,
27    /// Why it failed (empty when it passed).
28    pub detail: String,
29}
30
31/// Every check's outcome.
32#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
33#[non_exhaustive]
34pub struct CheckReport {
35    /// The run has checks, every one ran, and every one passed.
36    pub ok: bool,
37    /// Every check, in the order it ran.
38    pub results: Vec<CheckResult>,
39    /// The runners that didn't run to their end (timed out, failed before `END`).
40    pub unfinished: Vec<String>,
41}
42
43/// One line of a runner's output.
44#[derive(Debug, Clone, PartialEq, Eq)]
45pub(crate) enum Line {
46    Pass(String),
47    Fail(String, String),
48    End(u32, u32),
49}
50
51/// Reads a runner's line, anywhere in it (tools prefix their output: `web-1  | isoloom-check: …`).
52/// Ported from Isoloom's `isoloom-plan` (`checks/runner.rs`, `parse_line`).
53pub(crate) fn parse_line(line: &str) -> Option<Line> {
54    let (_, rest) = line.split_once("isoloom-check: ")?;
55    if let Some(n) = rest.strip_prefix("PASS ") {
56        return Some(Line::Pass(n.trim_end().to_owned()));
57    }
58    if let Some(r) = rest.strip_prefix("FAIL ") {
59        let (name, why) = r.split_once(": ").unwrap_or((r, ""));
60        return Some(Line::Fail(name.to_owned(), why.trim_end().to_owned()));
61    }
62    if let Some(r) = rest.strip_prefix("END ") {
63        let mut nums = r.split_whitespace().filter_map(|w| w.parse::<u32>().ok());
64        return Some(Line::End(nums.next()?, nums.next()?));
65    }
66    None
67}
68
69/// Runs the run's checks.
70pub(crate) async fn run(cx: &mut Cx<'_>) -> Result<CheckReport> {
71    let checks = &cx.run.manifest().checks;
72    let mut report = CheckReport::default();
73    let setup = run_steps(cx, "checks.setup", &checks.setup).await;
74    if setup.is_ok() {
75        for runner in &checks.runners {
76            run_runner(cx, runner, &mut report).await;
77        }
78    }
79    let teardown = run_steps(cx, "checks.teardown", &checks.teardown).await;
80    setup?;
81    teardown?;
82    report.ok = !checks.runners.is_empty() && report.unfinished.is_empty() && report.results.iter().all(|r| r.ok);
83    Ok(report)
84}
85
86/// Runs one check runner, reading its results into `report`.
87async fn run_runner(cx: &mut Cx<'_>, runner: &CheckRunner, report: &mut CheckReport) {
88    let id = runner.id.as_str();
89    let before = report.results.len();
90    let mut ended = false;
91    let mut record = |line: &str| match parse_line(line) {
92        Some(Line::Pass(name)) => push(cx, &mut report.results, result(runner, name, true, String::new())),
93        Some(Line::Fail(name, why)) => {
94            push(cx, &mut report.results, result(runner, name, false, why));
95        }
96        Some(Line::End(..)) => ended = true,
97        None => {}
98    };
99    let ran = run_argv(cx, "checks.runners", &runner.step, &mut record).await;
100    if let Err(e) = &ran {
101        cx.events.warning(WarningCode::CheckRunnerFailed, &format!("{id}: {e}"));
102        if matches!(e, Error::Timeout { .. }) {
103            // A runner that hung outlives its CLI: remove what it left (`on_timeout`). Its steps
104            // report their own failures; the runner's is the one the report holds.
105            if let Err(cleanup) = run_steps(cx, "checks.on_timeout", &runner.on_timeout).await {
106                cx.events.warning(WarningCode::CheckCleanupFailed, &format!("{id}: {cleanup}"));
107            }
108        }
109    }
110    let reported = report.results.len() - before;
111    for check in runner.checks.iter().skip(reported) {
112        push(cx, &mut report.results, unreported(runner, check));
113    }
114    if ran.is_err() || !ended {
115        report.unfinished.push(id.to_owned());
116    }
117}
118
119/// Keeps `r`, and emits it as it comes.
120fn push(cx: &Cx<'_>, results: &mut Vec<CheckResult>, r: CheckResult) {
121    cx.events.emit(EventKind::Check {
122        runner: r.runner.clone(),
123        from: r.from.clone(),
124        name: r.name.clone(),
125        ok: r.ok,
126        detail: r.detail.clone(),
127    });
128    results.push(r);
129}
130
131fn result(runner: &CheckRunner, name: String, ok: bool, detail: String) -> CheckResult {
132    CheckResult {
133        runner: runner.id.to_string(),
134        from: runner.from.clone(),
135        name,
136        ok,
137        detail,
138    }
139}
140
141/// A check its runner stopped before.
142fn unreported(runner: &CheckRunner, check: &Check) -> CheckResult {
143    result(runner, check.name.clone(), false, "the runner stopped before this check".into())
144}
145
146#[cfg(test)]
147mod tests {
148    use super::*;
149
150    #[test]
151    fn runner_lines_are_read_anywhere_in_a_line() {
152        assert_eq!(parse_line("isoloom-check: PASS login works"), Some(Line::Pass("login works".into())));
153        assert_eq!(
154            parse_line("web-1  | isoloom-check: FAIL flag readable: 404"),
155            Some(Line::Fail("flag readable".into(), "404".into()))
156        );
157        assert_eq!(parse_line("isoloom-check: FAIL no reason"), Some(Line::Fail("no reason".into(), String::new())));
158        assert_eq!(parse_line("isoloom-check: END 3 passed, 1 failed"), Some(Line::End(3, 1)));
159        assert_eq!(parse_line("isoloom-check: END soon"), None);
160        assert_eq!(parse_line("some other output"), None);
161    }
162}
163
164#[cfg(test)]
165mod proptests {
166    use proptest::prelude::*;
167
168    use super::*;
169
170    proptest! {
171        #[test]
172        fn any_line_is_read_without_panicking(line in "\\PC{0,120}") {
173            let _ = parse_line(&line);
174        }
175
176        #[test]
177        fn a_pass_reads_back_its_name_after_any_prefix(prefix in "[^i]{0,20}", name in "[a-zA-Z0-9][a-zA-Z0-9 /:._-]{0,40}[a-zA-Z0-9]") {
178            prop_assert_eq!(parse_line(&format!("{prefix}isoloom-check: PASS {name}")), Some(Line::Pass(name)));
179        }
180
181        #[test]
182        fn a_fail_reads_back_its_name_and_reason(name in "[a-z0-9 ._-]{1,30}", reason in "[a-zA-Z0-9 ]{0,30}[a-zA-Z0-9]") {
183            prop_assert_eq!(parse_line(&format!("isoloom-check: FAIL {name}: {reason}")), Some(Line::Fail(name, reason)));
184        }
185    }
186}
isothrum_runner/checks.rs

isothrum_runner/
checks.rs
