speccoder_core/
retry.rs

1//! Retries: the verifier's failure
2//! digest for the next attempt, and the fingerprint that stops retries
3//! which cannot help.
4//!
5//! Everything here comes from verifier output — never from the model's
6//! transcript or its own account of what went wrong.
7
8use serde::{Deserialize, Serialize};
9
10use crate::verify::{CheckStatus, VerifyReport};
11
12/// Lines of output kept per failing check.
13const EXCERPT_LINES: usize = 40;
14/// Bytes of output kept per failing check.
15const EXCERPT_BYTES: usize = 4000;
16/// Words that mark the first error line of a check's output.
17const MARKERS: [&str; 5] = ["error", "panicked", "assert", "failed", "fail"];
18
19/// One check that did not pass.
20#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
21pub struct FailedCheck {
22    /// Name from `speccoder.toml` (or `acceptance`).
23    pub name: String,
24    /// The command, as run.
25    pub argv: Vec<String>,
26    /// What happened.
27    pub status: CheckStatus,
28    /// Failing test IDs, when the check reports tests.
29    pub failing_tests: Vec<String>,
30    /// Output from the first error line on, capped.
31    pub excerpt: String,
32}
33
34/// What the next attempt is told about the previous one.
35#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
36pub struct Digest {
37    /// The attempt this digest describes (1-based).
38    pub attempt: u32,
39    /// The verdict's reasons.
40    pub reasons: Vec<String>,
41    /// Checks that did not pass.
42    pub checks: Vec<FailedCheck>,
43    /// Paths the previous attempt changed; its changes were discarded.
44    pub discarded: Vec<String>,
45    /// Test files it wrote that never ran: one line each — the path, the
46    /// runner's rule and the ways out.
47    #[serde(default, skip_serializing_if = "Vec::is_empty")]
48    pub uncollected: Vec<String>,
49}
50
51/// The retry reason for failed held-out tests: the scenario IDs and the
52/// spec's own words — which the implementer has already seen — and nothing
53/// from the held-out run itself (no path, test name, literal or output).
54#[must_use]
55pub fn held_out_reason(failed: &[crate::spec::ScenarioRef]) -> String {
56    let mut failed: Vec<&crate::spec::ScenarioRef> = failed.iter().collect();
57    failed.sort_by(|a, b| a.id.cmp(&b.id));
58    failed.dedup_by(|a, b| a.id == b.id);
59    let parts: Vec<String> = failed
60        .iter()
61        .map(|s| {
62            format!(
63                "{} (\"{}\")",
64                s.id,
65                s.text.split_whitespace().collect::<Vec<_>>().join(" ")
66            )
67        })
68        .collect();
69    format!("held-out acceptance failed: {}", parts.join(", "))
70}
71
72/// Why a failed attempt must not be retried, if it must not: a check that
73/// could not run at all is an infrastructure problem, not the model's.
74#[must_use]
75pub fn infrastructure(report: &VerifyReport) -> Option<String> {
76    report
77        .checks
78        .iter()
79        .find(|c| c.status == CheckStatus::Error)
80        .map(|c| {
81            format!(
82                "check {} could not run{}",
83                c.name,
84                c.note
85                    .as_deref()
86                    .map(|n| format!(": {n}"))
87                    .unwrap_or_default()
88            )
89        })
90}
91
92/// The digest of a failed attempt.
93#[must_use]
94pub fn digest(
95    attempt: u32,
96    reasons: &[String],
97    report: &VerifyReport,
98    discarded: &[String],
99) -> Digest {
100    let checks = report
101        .checks
102        .iter()
103        .filter(|c| !matches!(c.status, CheckStatus::Passed | CheckStatus::Skipped))
104        .map(|c| FailedCheck {
105            name: c.name.clone(),
106            argv: c.argv.clone(),
107            status: c.status,
108            failing_tests: c
109                .tests
110                .iter()
111                .flatten()
112                .filter(|t| t.outcome.is_failing())
113                .map(|t| t.id.clone())
114                .collect(),
115            excerpt: excerpt(&output(&c.stderr_tail, &c.stdout_tail)),
116        })
117        .collect();
118    Digest {
119        attempt,
120        reasons: reasons.to_vec(),
121        checks,
122        discarded: discarded.to_vec(),
123        uncollected: Vec::new(),
124    }
125}
126
127/// The failure's identity: failing checks by name, each with its first
128/// error line, numbers normalised. Equal fingerprints mean the same failure.
129#[must_use]
130pub fn fingerprint(digest: &Digest) -> String {
131    let mut parts: Vec<String> = digest
132        .checks
133        .iter()
134        .map(|c| format!("{}: {}", c.name, normalise(first_error(&c.excerpt))))
135        .collect();
136    parts.sort();
137    let joined = if parts.is_empty() {
138        let mut reasons: Vec<String> = digest.reasons.iter().map(|r| normalise(r)).collect();
139        reasons.sort();
140        reasons.join(" | ")
141    } else {
142        parts.join(" | ")
143    };
144    // The same file never running twice is the same failure.
145    let mut paths: Vec<&str> = digest
146        .uncollected
147        .iter()
148        .filter_map(|l| l.split('`').nth(1))
149        .collect();
150    paths.sort_unstable();
151    paths.dedup();
152    if paths.is_empty() {
153        joined
154    } else {
155        format!("{joined} | uncollected: {}", paths.join(", "))
156    }
157}
158
159fn output(stderr: &str, stdout: &str) -> String {
160    match (stderr.trim().is_empty(), stdout.trim().is_empty()) {
161        (false, false) => format!("{stderr}\n{stdout}"),
162        (false, true) => stderr.to_owned(),
163        _ => stdout.to_owned(),
164    }
165}
166
167fn is_error_line(line: &str) -> bool {
168    let lower = line.to_lowercase();
169    MARKERS.iter().any(|m| lower.contains(m))
170}
171
172fn first_error(text: &str) -> &str {
173    text.lines()
174        .find(|l| is_error_line(l))
175        .or_else(|| text.lines().find(|l| !l.trim().is_empty()))
176        .unwrap_or_default()
177}
178
179/// From the first error line on (or the last lines, if there is none),
180/// at most [`EXCERPT_LINES`] lines and [`EXCERPT_BYTES`] bytes.
181fn excerpt(text: &str) -> String {
182    let lines: Vec<&str> = text.lines().collect();
183    let start = lines
184        .iter()
185        .position(|l| is_error_line(l))
186        .unwrap_or_else(|| lines.len().saturating_sub(EXCERPT_LINES));
187    let mut out = lines
188        .iter()
189        .skip(start)
190        .take(EXCERPT_LINES)
191        .copied()
192        .collect::<Vec<_>>()
193        .join("\n");
194    if out.len() > EXCERPT_BYTES {
195        let mut end = EXCERPT_BYTES;
196        while !out.is_char_boundary(end) {
197            end -= 1;
198        }
199        out.truncate(end);
200        out.push_str("\n…");
201    }
202    out
203}
204
205/// Digits become `#`, whitespace runs one space; at most 200 characters.
206fn normalise(line: &str) -> String {
207    let mapped: String = line
208        .chars()
209        .map(|c| if c.is_ascii_digit() { '#' } else { c })
210        .collect();
211    mapped
212        .split_whitespace()
213        .collect::<Vec<_>>()
214        .join(" ")
215        .chars()
216        .take(200)
217        .collect()
218}
219
220#[cfg(test)]
221mod tests {
222    use super::*;
223
224    #[test]
225    fn a_held_out_failure_names_the_requirement_only() {
226        use crate::spec::ScenarioRef;
227        let failed = [
228            ScenarioRef {
229                id: "US1-2".to_owned(),
230                text: "**Given** a name,\n **When** greeted".to_owned(),
231            },
232            ScenarioRef {
233                id: "US1-1".to_owned(),
234                text: "x".to_owned(),
235            },
236        ];
237        let reason = held_out_reason(&failed);
238        assert_eq!(
239            reason,
240            "held-out acceptance failed: US1-1 (\"x\"), US1-2 (\"**Given** a name, **When** greeted\")"
241        );
242        let report = VerifyReport {
243            root: String::new(),
244            tier: "fast".to_owned(),
245            checks: Vec::new(),
246            passed: true,
247            comparison: None,
248        };
249        let d = digest(2, std::slice::from_ref(&reason), &report, &[]);
250        assert!(d.checks.is_empty(), "no check entry for the held-out run");
251        assert_eq!(fingerprint(&d), normalise(&reason));
252    }
253    use crate::verify::{CheckResult, TestOutcome, TestResult};
254
255    fn check(name: &str, status: CheckStatus, stderr: &str) -> CheckResult {
256        CheckResult {
257            name: name.to_owned(),
258            argv: vec!["cargo".to_owned(), "test".to_owned()],
259            status,
260            exit_code: Some(101),
261            duration_ms: 10,
262            junit: false,
263            tests: None,
264            stdout_tail: String::new(),
265            stderr_tail: stderr.to_owned(),
266            truncated: false,
267            note: None,
268            flaky: false,
269        }
270    }
271
272    fn report(checks: Vec<CheckResult>) -> VerifyReport {
273        VerifyReport {
274            root: ".".to_owned(),
275            tier: "fast".to_owned(),
276            checks,
277            comparison: None,
278            passed: false,
279        }
280    }
281
282    #[test]
283    fn the_digest_keeps_failing_checks_from_the_first_error() {
284        let mut failing = check(
285            "test",
286            CheckStatus::Failed,
287            "   Compiling x v0.1.0\nerror[E0425]: cannot find function `add`\n --> src/lib.rs:3:5\n",
288        );
289        failing.tests = Some(vec![
290            TestResult {
291                id: "adds".to_owned(),
292                outcome: TestOutcome::Failed,
293            },
294            TestResult {
295                id: "ok".to_owned(),
296                outcome: TestOutcome::Passed,
297            },
298        ]);
299        let r = report(vec![check("fmt", CheckStatus::Passed, ""), failing]);
300        let d = digest(
301            1,
302            &["check test failed".to_owned()],
303            &r,
304            &["src/lib.rs".to_owned()],
305        );
306        assert_eq!(d.checks.len(), 1);
307        assert_eq!(d.checks[0].failing_tests, ["adds"]);
308        assert!(
309            d.checks[0].excerpt.starts_with("error[E0425]"),
310            "{}",
311            d.checks[0].excerpt
312        );
313        assert!(!d.checks[0].excerpt.contains("Compiling"));
314        assert_eq!(d.discarded, ["src/lib.rs"]);
315    }
316
317    #[test]
318    fn fingerprints_ignore_numbers_and_order() {
319        let a = report(vec![
320            check("b", CheckStatus::Failed, "error at line 12"),
321            check("a", CheckStatus::Failed, "assertion failed: 4 == 5"),
322        ]);
323        let b = report(vec![
324            check("a", CheckStatus::Failed, "assertion failed: 3 == 5"),
325            check("b", CheckStatus::Failed, "error at line 40"),
326        ]);
327        let c = report(vec![check(
328            "a",
329            CheckStatus::Failed,
330            "error: something else",
331        )]);
332        let fp = |r: &VerifyReport| fingerprint(&digest(1, &[], r, &[]));
333        assert_eq!(fp(&a), fp(&b));
334        assert_ne!(fp(&a), fp(&c));
335        // Without failing checks (e.g. a regression), the reasons decide.
336        let none = report(vec![]);
337        let x = fingerprint(&digest(1, &["regressed: t1".to_owned()], &none, &[]));
338        assert_eq!(x, "regressed: t#");
339    }
340
341    #[test]
342    fn a_check_that_could_not_run_is_infrastructure() {
343        let mut broken = check("lint", CheckStatus::Error, "");
344        broken.note = Some("command not found".to_owned());
345        assert_eq!(
346            infrastructure(&report(vec![broken])).as_deref(),
347            Some("check lint could not run: command not found")
348        );
349        assert!(infrastructure(&report(vec![check("t", CheckStatus::Failed, "x")])).is_none());
350    }
351
352    #[test]
353    fn long_output_is_capped_on_a_char_boundary() {
354        let long = format!("error: {}", "é".repeat(5000));
355        let e = excerpt(&long);
356        assert!(e.len() <= EXCERPT_BYTES + "\n…".len());
357        assert!(e.ends_with('…'));
358    }
359}
speccoder_core/retry.rs

speccoder_core/
retry.rs
