speccoder_core/
retry.rs

1//! Retries: the verifier's failure
2//! digest for the next attempt, and the fingerprint that stops retries
3//! which cannot help.
4//!
5//! Everything here comes from verifier output — never from the model's
6//! transcript or its own account of what went wrong.
7
8use serde::{Deserialize, Serialize};
9
10use crate::verify::{CheckStatus, VerifyReport};
11
12/// Lines of output kept per failing check.
13const EXCERPT_LINES: usize = 40;
14/// Bytes of output kept per failing check.
15const EXCERPT_BYTES: usize = 4000;
16/// Words that mark the first error line of a check's output.
17const MARKERS: [&str; 5] = ["error", "panicked", "assert", "failed", "fail"];
18
19/// One check that did not pass.
20#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
21pub struct FailedCheck {
22    /// Name from `speccoder.toml` (or `acceptance`).
23    pub name: String,
24    /// The command, as run.
25    pub argv: Vec<String>,
26    /// What happened.
27    pub status: CheckStatus,
28    /// Failing test IDs, when the check reports tests.
29    pub failing_tests: Vec<String>,
30    /// Output from the first error line on, capped.
31    pub excerpt: String,
32}
33
34/// What the next attempt is told about the previous one.
35#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
36pub struct Digest {
37    /// The attempt this digest describes (1-based).
38    pub attempt: u32,
39    /// The verdict's reasons.
40    pub reasons: Vec<String>,
41    /// Checks that did not pass.
42    pub checks: Vec<FailedCheck>,
43    /// Paths the previous attempt changed; its changes were discarded.
44    pub discarded: Vec<String>,
45    /// Test files it wrote that never ran: one line each — the path, the
46    /// runner's rule and the ways out.
47    #[serde(default, skip_serializing_if = "Vec::is_empty")]
48    pub uncollected: Vec<String>,
49}
50
51/// The retry reason for failed held-out tests: the scenario IDs and the
52/// spec's own words — which the implementer has already seen — and nothing
53/// from the held-out run itself (no path, test name, literal or output).
54#[must_use]
55pub fn held_out_reason(failed: &[crate::spec::ScenarioRef]) -> String {
56    let mut failed: Vec<&crate::spec::ScenarioRef> = failed.iter().collect();
57    failed.sort_by(|a, b| a.id.cmp(&b.id));
58    failed.dedup_by(|a, b| a.id == b.id);
59    let parts: Vec<String> = failed
60        .iter()
61        .map(|s| {
62            format!(
63                "{} (\"{}\")",
64                s.id,
65                s.text.split_whitespace().collect::<Vec<_>>().join(" ")
66            )
67        })
68        .collect();
69    format!("held-out acceptance failed: {}", parts.join(", "))
70}
71
72/// Why a failed attempt must not be retried, if it must not: a check that
73/// could not run at all is an infrastructure problem, not the model's.
74#[must_use]
75pub fn infrastructure(report: &VerifyReport) -> Option<String> {
76    report
77        .checks
78        .iter()
79        .find(|c| c.status == CheckStatus::Error)
80        .map(|c| {
81            format!(
82                "check {} could not run{}",
83                c.name,
84                c.note
85                    .as_deref()
86                    .map(|n| format!(": {n}"))
87                    .unwrap_or_default()
88            )
89        })
90}
91
92/// The digest of a failed attempt.
93#[must_use]
94pub fn digest(
95    attempt: u32,
96    reasons: &[String],
97    report: &VerifyReport,
98    discarded: &[String],
99) -> Digest {
100    let checks = report
101        .checks
102        .iter()
103        .filter(|c| !matches!(c.status, CheckStatus::Passed | CheckStatus::Skipped))
104        .map(|c| FailedCheck {
105            name: c.name.clone(),
106            argv: c.argv.clone(),
107            status: c.status,
108            failing_tests: c
109                .tests
110                .iter()
111                .flatten()
112                .filter(|t| t.outcome.is_failing())
113                .map(|t| t.id.clone())
114                .collect(),
115            excerpt: excerpt(&output(&c.stderr_tail, &c.stdout_tail)),
116        })
117        .collect();
118    Digest {
119        attempt,
120        reasons: reasons.to_vec(),
121        checks,
122        discarded: discarded.to_vec(),
123        uncollected: Vec::new(),
124    }
125}
126
127/// What a failed check says first, for a person: its failing tests by
128/// name, else the first error line of its output (rules of `=` or `_`
129/// skipped), cut at 160 characters. `None`: it said nothing.
130#[must_use]
131pub fn first_cause(c: &crate::verify::CheckResult) -> Option<String> {
132    let failing: Vec<&str> = c
133        .tests
134        .iter()
135        .flatten()
136        .filter(|t| t.outcome.is_failing())
137        .map(|t| t.id.as_str())
138        .collect();
139    let said = match failing.as_slice() {
140        [] => {
141            let text = output(&c.stderr_tail, &c.stdout_tail);
142            let lines = || {
143                text.lines()
144                    .map(str::trim)
145                    .filter(|l| !l.is_empty() && !l.starts_with("===") && !l.starts_with("___"))
146            };
147            lines()
148                .find(|l| is_error_line(l))
149                .or_else(|| lines().next())?
150                .to_owned()
151        }
152        [one] => format!("{one} fails"),
153        [first, rest @ ..] => format!("{first} and {} more fail", rest.len()),
154    };
155    let cut: String = said.chars().take(160).collect();
156    Some(if cut.len() < said.len() {
157        format!("{cut}…")
158    } else {
159        cut
160    })
161}
162
163/// The failure's identity: failing checks by name, each with its first
164/// error line, numbers normalised. Equal fingerprints mean the same failure.
165#[must_use]
166pub fn fingerprint(digest: &Digest) -> String {
167    let mut parts: Vec<String> = digest
168        .checks
169        .iter()
170        .map(|c| format!("{}: {}", c.name, normalise(first_error(&c.excerpt))))
171        .collect();
172    parts.sort();
173    let joined = if parts.is_empty() {
174        let mut reasons: Vec<String> = digest.reasons.iter().map(|r| normalise(r)).collect();
175        reasons.sort();
176        reasons.join(" | ")
177    } else {
178        parts.join(" | ")
179    };
180    // The same file never running twice is the same failure.
181    let mut paths: Vec<&str> = digest
182        .uncollected
183        .iter()
184        .filter_map(|l| l.split('`').nth(1))
185        .collect();
186    paths.sort_unstable();
187    paths.dedup();
188    if paths.is_empty() {
189        joined
190    } else {
191        format!("{joined} | uncollected: {}", paths.join(", "))
192    }
193}
194
195fn output(stderr: &str, stdout: &str) -> String {
196    match (stderr.trim().is_empty(), stdout.trim().is_empty()) {
197        (false, false) => format!("{stderr}\n{stdout}"),
198        (false, true) => stderr.to_owned(),
199        _ => stdout.to_owned(),
200    }
201}
202
203fn is_error_line(line: &str) -> bool {
204    let lower = line.to_lowercase();
205    MARKERS.iter().any(|m| lower.contains(m))
206}
207
208fn first_error(text: &str) -> &str {
209    text.lines()
210        .find(|l| is_error_line(l))
211        .or_else(|| text.lines().find(|l| !l.trim().is_empty()))
212        .unwrap_or_default()
213}
214
215/// From the first error line on (or the last lines, if there is none),
216/// at most [`EXCERPT_LINES`] lines and [`EXCERPT_BYTES`] bytes.
217fn excerpt(text: &str) -> String {
218    let lines: Vec<&str> = text.lines().collect();
219    let start = lines
220        .iter()
221        .position(|l| is_error_line(l))
222        .unwrap_or_else(|| lines.len().saturating_sub(EXCERPT_LINES));
223    let mut out = lines
224        .iter()
225        .skip(start)
226        .take(EXCERPT_LINES)
227        .copied()
228        .collect::<Vec<_>>()
229        .join("\n");
230    if out.len() > EXCERPT_BYTES {
231        let mut end = EXCERPT_BYTES;
232        while !out.is_char_boundary(end) {
233            end -= 1;
234        }
235        out.truncate(end);
236        out.push_str("\n…");
237    }
238    out
239}
240
241/// Digits become `#`, whitespace runs one space; at most 200 characters.
242fn normalise(line: &str) -> String {
243    let mapped: String = line
244        .chars()
245        .map(|c| if c.is_ascii_digit() { '#' } else { c })
246        .collect();
247    mapped
248        .split_whitespace()
249        .collect::<Vec<_>>()
250        .join(" ")
251        .chars()
252        .take(200)
253        .collect()
254}
255
256#[cfg(test)]
257mod tests {
258    use super::*;
259
260    #[test]
261    fn a_held_out_failure_names_the_requirement_only() {
262        use crate::spec::ScenarioRef;
263        let failed = [
264            ScenarioRef {
265                id: "US1-2".to_owned(),
266                text: "**Given** a name,\n **When** greeted".to_owned(),
267            },
268            ScenarioRef {
269                id: "US1-1".to_owned(),
270                text: "x".to_owned(),
271            },
272        ];
273        let reason = held_out_reason(&failed);
274        assert_eq!(
275            reason,
276            "held-out acceptance failed: US1-1 (\"x\"), US1-2 (\"**Given** a name, **When** greeted\")"
277        );
278        let report = VerifyReport {
279            root: String::new(),
280            tier: "fast".to_owned(),
281            checks: Vec::new(),
282            passed: true,
283            comparison: None,
284        };
285        let d = digest(2, std::slice::from_ref(&reason), &report, &[]);
286        assert!(d.checks.is_empty(), "no check entry for the held-out run");
287        assert_eq!(fingerprint(&d), normalise(&reason));
288    }
289    use crate::verify::{CheckResult, TestOutcome, TestResult};
290
291    fn check(name: &str, status: CheckStatus, stderr: &str) -> CheckResult {
292        CheckResult {
293            name: name.to_owned(),
294            argv: vec!["cargo".to_owned(), "test".to_owned()],
295            status,
296            exit_code: Some(101),
297            duration_ms: 10,
298            junit: false,
299            tests: None,
300            stdout_tail: String::new(),
301            stderr_tail: stderr.to_owned(),
302            truncated: false,
303            note: None,
304            flaky: false,
305        }
306    }
307
308    fn report(checks: Vec<CheckResult>) -> VerifyReport {
309        VerifyReport {
310            root: ".".to_owned(),
311            tier: "fast".to_owned(),
312            checks,
313            comparison: None,
314            passed: false,
315        }
316    }
317
318    #[test]
319    fn the_digest_keeps_failing_checks_from_the_first_error() {
320        let mut failing = check(
321            "test",
322            CheckStatus::Failed,
323            "   Compiling x v0.1.0\nerror[E0425]: cannot find function `add`\n --> src/lib.rs:3:5\n",
324        );
325        failing.tests = Some(vec![
326            TestResult {
327                id: "adds".to_owned(),
328                outcome: TestOutcome::Failed,
329            },
330            TestResult {
331                id: "ok".to_owned(),
332                outcome: TestOutcome::Passed,
333            },
334        ]);
335        let r = report(vec![check("fmt", CheckStatus::Passed, ""), failing]);
336        let d = digest(
337            1,
338            &["check test failed".to_owned()],
339            &r,
340            &["src/lib.rs".to_owned()],
341        );
342        assert_eq!(d.checks.len(), 1);
343        assert_eq!(d.checks[0].failing_tests, ["adds"]);
344        assert!(
345            d.checks[0].excerpt.starts_with("error[E0425]"),
346            "{}",
347            d.checks[0].excerpt
348        );
349        assert!(!d.checks[0].excerpt.contains("Compiling"));
350        assert_eq!(d.discarded, ["src/lib.rs"]);
351    }
352
353    #[test]
354    fn fingerprints_ignore_numbers_and_order() {
355        let a = report(vec![
356            check("b", CheckStatus::Failed, "error at line 12"),
357            check("a", CheckStatus::Failed, "assertion failed: 4 == 5"),
358        ]);
359        let b = report(vec![
360            check("a", CheckStatus::Failed, "assertion failed: 3 == 5"),
361            check("b", CheckStatus::Failed, "error at line 40"),
362        ]);
363        let c = report(vec![check(
364            "a",
365            CheckStatus::Failed,
366            "error: something else",
367        )]);
368        let fp = |r: &VerifyReport| fingerprint(&digest(1, &[], r, &[]));
369        assert_eq!(fp(&a), fp(&b));
370        assert_ne!(fp(&a), fp(&c));
371        // Without failing checks (e.g. a regression), the reasons decide.
372        let none = report(vec![]);
373        let x = fingerprint(&digest(1, &["regressed: t1".to_owned()], &none, &[]));
374        assert_eq!(x, "regressed: t#");
375    }
376
377    #[test]
378    fn a_check_that_could_not_run_is_infrastructure() {
379        let mut broken = check("lint", CheckStatus::Error, "");
380        broken.note = Some("command not found".to_owned());
381        assert_eq!(
382            infrastructure(&report(vec![broken])).as_deref(),
383            Some("check lint could not run: command not found")
384        );
385        assert!(infrastructure(&report(vec![check("t", CheckStatus::Failed, "x")])).is_none());
386    }
387
388    #[test]
389    fn long_output_is_capped_on_a_char_boundary() {
390        let long = format!("error: {}", "é".repeat(5000));
391        let e = excerpt(&long);
392        assert!(e.len() <= EXCERPT_BYTES + "\n…".len());
393        assert!(e.ends_with('…'));
394    }
395}
speccoder_core/retry.rs

speccoder_core/
retry.rs
