speccoder_core/
retry.rs

1//! Retries: the verifier's failure
2//! digest for the next attempt, and the fingerprint that stops retries
3//! which cannot help.
4//!
5//! Everything here comes from verifier output — never from the model's
6//! transcript or its own account of what went wrong.
7
8use serde::{Deserialize, Serialize};
9
10use crate::verify::{CheckStatus, VerifyReport};
11
12/// Lines of output kept per failing check.
13const EXCERPT_LINES: usize = 40;
14/// Bytes of output kept per failing check.
15const EXCERPT_BYTES: usize = 4000;
16/// Words that mark the first error line of a check's output.
17const MARKERS: [&str; 5] = ["error", "panicked", "assert", "failed", "fail"];
18
19/// One check that did not pass.
20#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
21pub struct FailedCheck {
22    /// Name from `speccoder.toml` (or `acceptance`).
23    pub name: String,
24    /// The command, as run.
25    pub argv: Vec<String>,
26    /// What happened.
27    pub status: CheckStatus,
28    /// Failing test IDs, when the check reports tests.
29    pub failing_tests: Vec<String>,
30    /// Output from the first error line on, capped.
31    pub excerpt: String,
32}
33
34/// What the next attempt is told about the previous one.
35#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
36pub struct Digest {
37    /// The attempt this digest describes (1-based).
38    pub attempt: u32,
39    /// The verdict's reasons.
40    pub reasons: Vec<String>,
41    /// Checks that did not pass.
42    pub checks: Vec<FailedCheck>,
43    /// Paths the previous attempt changed; its changes were discarded.
44    pub discarded: Vec<String>,
45}
46
47/// The retry reason for failed held-out tests: the scenario IDs and the
48/// spec's own words — which the implementer has already seen — and nothing
49/// from the held-out run itself (no path, test name, literal or output).
50#[must_use]
51pub fn held_out_reason(failed: &[crate::spec::ScenarioRef]) -> String {
52    let mut failed: Vec<&crate::spec::ScenarioRef> = failed.iter().collect();
53    failed.sort_by(|a, b| a.id.cmp(&b.id));
54    failed.dedup_by(|a, b| a.id == b.id);
55    let parts: Vec<String> = failed
56        .iter()
57        .map(|s| {
58            format!(
59                "{} (\"{}\")",
60                s.id,
61                s.text.split_whitespace().collect::<Vec<_>>().join(" ")
62            )
63        })
64        .collect();
65    format!("held-out acceptance failed: {}", parts.join(", "))
66}
67
68/// Why a failed attempt must not be retried, if it must not: a check that
69/// could not run at all is an infrastructure problem, not the model's.
70#[must_use]
71pub fn infrastructure(report: &VerifyReport) -> Option<String> {
72    report
73        .checks
74        .iter()
75        .find(|c| c.status == CheckStatus::Error)
76        .map(|c| {
77            format!(
78                "check {} could not run{}",
79                c.name,
80                c.note
81                    .as_deref()
82                    .map(|n| format!(": {n}"))
83                    .unwrap_or_default()
84            )
85        })
86}
87
88/// The digest of a failed attempt.
89#[must_use]
90pub fn digest(
91    attempt: u32,
92    reasons: &[String],
93    report: &VerifyReport,
94    discarded: &[String],
95) -> Digest {
96    let checks = report
97        .checks
98        .iter()
99        .filter(|c| !matches!(c.status, CheckStatus::Passed | CheckStatus::Skipped))
100        .map(|c| FailedCheck {
101            name: c.name.clone(),
102            argv: c.argv.clone(),
103            status: c.status,
104            failing_tests: c
105                .tests
106                .iter()
107                .flatten()
108                .filter(|t| t.outcome.is_failing())
109                .map(|t| t.id.clone())
110                .collect(),
111            excerpt: excerpt(&output(&c.stderr_tail, &c.stdout_tail)),
112        })
113        .collect();
114    Digest {
115        attempt,
116        reasons: reasons.to_vec(),
117        checks,
118        discarded: discarded.to_vec(),
119    }
120}
121
122/// The failure's identity: failing checks by name, each with its first
123/// error line, numbers normalised. Equal fingerprints mean the same failure.
124#[must_use]
125pub fn fingerprint(digest: &Digest) -> String {
126    let mut parts: Vec<String> = digest
127        .checks
128        .iter()
129        .map(|c| format!("{}: {}", c.name, normalise(first_error(&c.excerpt))))
130        .collect();
131    parts.sort();
132    if parts.is_empty() {
133        let mut reasons: Vec<String> = digest.reasons.iter().map(|r| normalise(r)).collect();
134        reasons.sort();
135        return reasons.join(" | ");
136    }
137    parts.join(" | ")
138}
139
140fn output(stderr: &str, stdout: &str) -> String {
141    match (stderr.trim().is_empty(), stdout.trim().is_empty()) {
142        (false, false) => format!("{stderr}\n{stdout}"),
143        (false, true) => stderr.to_owned(),
144        _ => stdout.to_owned(),
145    }
146}
147
148fn is_error_line(line: &str) -> bool {
149    let lower = line.to_lowercase();
150    MARKERS.iter().any(|m| lower.contains(m))
151}
152
153fn first_error(text: &str) -> &str {
154    text.lines()
155        .find(|l| is_error_line(l))
156        .or_else(|| text.lines().find(|l| !l.trim().is_empty()))
157        .unwrap_or_default()
158}
159
160/// From the first error line on (or the last lines, if there is none),
161/// at most [`EXCERPT_LINES`] lines and [`EXCERPT_BYTES`] bytes.
162fn excerpt(text: &str) -> String {
163    let lines: Vec<&str> = text.lines().collect();
164    let start = lines
165        .iter()
166        .position(|l| is_error_line(l))
167        .unwrap_or_else(|| lines.len().saturating_sub(EXCERPT_LINES));
168    let mut out = lines
169        .iter()
170        .skip(start)
171        .take(EXCERPT_LINES)
172        .copied()
173        .collect::<Vec<_>>()
174        .join("\n");
175    if out.len() > EXCERPT_BYTES {
176        let mut end = EXCERPT_BYTES;
177        while !out.is_char_boundary(end) {
178            end -= 1;
179        }
180        out.truncate(end);
181        out.push_str("\n…");
182    }
183    out
184}
185
186/// Digits become `#`, whitespace runs one space; at most 200 characters.
187fn normalise(line: &str) -> String {
188    let mapped: String = line
189        .chars()
190        .map(|c| if c.is_ascii_digit() { '#' } else { c })
191        .collect();
192    mapped
193        .split_whitespace()
194        .collect::<Vec<_>>()
195        .join(" ")
196        .chars()
197        .take(200)
198        .collect()
199}
200
201#[cfg(test)]
202mod tests {
203    use super::*;
204
205    #[test]
206    fn a_held_out_failure_names_the_requirement_only() {
207        use crate::spec::ScenarioRef;
208        let failed = [
209            ScenarioRef {
210                id: "US1-2".to_owned(),
211                text: "**Given** a name,\n **When** greeted".to_owned(),
212            },
213            ScenarioRef {
214                id: "US1-1".to_owned(),
215                text: "x".to_owned(),
216            },
217        ];
218        let reason = held_out_reason(&failed);
219        assert_eq!(
220            reason,
221            "held-out acceptance failed: US1-1 (\"x\"), US1-2 (\"**Given** a name, **When** greeted\")"
222        );
223        let report = VerifyReport {
224            root: String::new(),
225            tier: "fast".to_owned(),
226            checks: Vec::new(),
227            passed: true,
228            comparison: None,
229        };
230        let d = digest(2, std::slice::from_ref(&reason), &report, &[]);
231        assert!(d.checks.is_empty(), "no check entry for the held-out run");
232        assert_eq!(fingerprint(&d), normalise(&reason));
233    }
234    use crate::verify::{CheckResult, TestOutcome, TestResult};
235
236    fn check(name: &str, status: CheckStatus, stderr: &str) -> CheckResult {
237        CheckResult {
238            name: name.to_owned(),
239            argv: vec!["cargo".to_owned(), "test".to_owned()],
240            status,
241            exit_code: Some(101),
242            duration_ms: 10,
243            junit: false,
244            tests: None,
245            stdout_tail: String::new(),
246            stderr_tail: stderr.to_owned(),
247            truncated: false,
248            note: None,
249            flaky: false,
250        }
251    }
252
253    fn report(checks: Vec<CheckResult>) -> VerifyReport {
254        VerifyReport {
255            root: ".".to_owned(),
256            tier: "fast".to_owned(),
257            checks,
258            comparison: None,
259            passed: false,
260        }
261    }
262
263    #[test]
264    fn the_digest_keeps_failing_checks_from_the_first_error() {
265        let mut failing = check(
266            "test",
267            CheckStatus::Failed,
268            "   Compiling x v0.1.0\nerror[E0425]: cannot find function `add`\n --> src/lib.rs:3:5\n",
269        );
270        failing.tests = Some(vec![
271            TestResult {
272                id: "adds".to_owned(),
273                outcome: TestOutcome::Failed,
274            },
275            TestResult {
276                id: "ok".to_owned(),
277                outcome: TestOutcome::Passed,
278            },
279        ]);
280        let r = report(vec![check("fmt", CheckStatus::Passed, ""), failing]);
281        let d = digest(
282            1,
283            &["check test failed".to_owned()],
284            &r,
285            &["src/lib.rs".to_owned()],
286        );
287        assert_eq!(d.checks.len(), 1);
288        assert_eq!(d.checks[0].failing_tests, ["adds"]);
289        assert!(
290            d.checks[0].excerpt.starts_with("error[E0425]"),
291            "{}",
292            d.checks[0].excerpt
293        );
294        assert!(!d.checks[0].excerpt.contains("Compiling"));
295        assert_eq!(d.discarded, ["src/lib.rs"]);
296    }
297
298    #[test]
299    fn fingerprints_ignore_numbers_and_order() {
300        let a = report(vec![
301            check("b", CheckStatus::Failed, "error at line 12"),
302            check("a", CheckStatus::Failed, "assertion failed: 4 == 5"),
303        ]);
304        let b = report(vec![
305            check("a", CheckStatus::Failed, "assertion failed: 3 == 5"),
306            check("b", CheckStatus::Failed, "error at line 40"),
307        ]);
308        let c = report(vec![check(
309            "a",
310            CheckStatus::Failed,
311            "error: something else",
312        )]);
313        let fp = |r: &VerifyReport| fingerprint(&digest(1, &[], r, &[]));
314        assert_eq!(fp(&a), fp(&b));
315        assert_ne!(fp(&a), fp(&c));
316        // Without failing checks (e.g. a regression), the reasons decide.
317        let none = report(vec![]);
318        let x = fingerprint(&digest(1, &["regressed: t1".to_owned()], &none, &[]));
319        assert_eq!(x, "regressed: t#");
320    }
321
322    #[test]
323    fn a_check_that_could_not_run_is_infrastructure() {
324        let mut broken = check("lint", CheckStatus::Error, "");
325        broken.note = Some("command not found".to_owned());
326        assert_eq!(
327            infrastructure(&report(vec![broken])).as_deref(),
328            Some("check lint could not run: command not found")
329        );
330        assert!(infrastructure(&report(vec![check("t", CheckStatus::Failed, "x")])).is_none());
331    }
332
333    #[test]
334    fn long_output_is_capped_on_a_char_boundary() {
335        let long = format!("error: {}", "é".repeat(5000));
336        let e = excerpt(&long);
337        assert!(e.len() <= EXCERPT_BYTES + "\n…".len());
338        assert!(e.ends_with('…'));
339    }
340}
speccoder_core/retry.rs

speccoder_core/
retry.rs
