speccoder_runtime/pipeline/
judge.rs

1//! The judge's steps within one attempt: which writes a person approved,
2//! what the integrity audit finds, the verdict from the checks, and whether
3//! a failed attempt is worth retrying.
4
5use std::collections::BTreeSet;
6use std::path::Path;
7
8use serde_json::Value;
9use speccoder_core::RunReport;
10use speccoder_core::verify::{CheckStatus, Verdict, VerifyReport, compare, passes, verdict};
11
12use crate::tools::policy::{Object, Policy};
13use crate::{integrity, sandbox, verify, worktree};
14
15use super::{PreparedAcceptance, run_acceptance};
16
17/// Paths a person granted writing, for the task or inline in this attempt.
18/// A frozen test so approved is not a violation; the audit still flags
19/// removed tests and skip markers in it — approval permits editing a test,
20/// not weakening it.
21pub(crate) fn approved_writes(granted: &[Object], tool_calls: &Value) -> BTreeSet<String> {
22    granted
23        .iter()
24        .filter_map(|g| g.path().map(str::to_owned))
25        .chain(
26            tool_calls
27                .as_array()
28                .into_iter()
29                .flatten()
30                .filter(|c| c["approved_by"] == "person")
31                .filter_map(|c| {
32                    serde_json::from_value::<Object>(c["object"].clone())
33                        .ok()?
34                        .path()
35                        .map(str::to_owned)
36                }),
37        )
38        .collect()
39}
40
41/// What the integrity audit of one attempt reads.
42pub(crate) struct Audited<'a> {
43    pub(crate) wt_root: &'a Path,
44    pub(crate) policy: &'a Policy,
45    /// Judge inputs as the attempt found them.
46    pub(crate) judged_before: &'a integrity::JudgeSnapshot,
47    pub(crate) raw_changes: &'a [worktree::Change],
48    pub(crate) patch: &'a str,
49    pub(crate) frozen: &'a [String],
50    pub(crate) frozen_additions: bool,
51    pub(crate) approved_writes: &'a BTreeSet<String>,
52    /// Judge drift a command caused, reported by the tool (it ended the attempt).
53    pub(crate) command_drift: &'a Value,
54}
55
56/// Every integrity finding of an attempt, each `integrity: …`: the diff
57/// audit, judge inputs the diff cannot show (git-ignored files, whoever
58/// changed them — any executor), and drift a command caused.
59pub(crate) fn attempt_findings(a: &Audited<'_>) -> Vec<String> {
60    let unseen = a
61        .judged_before
62        .drift(
63            &integrity::JudgeSnapshot::take(a.wt_root, a.policy),
64            a.policy,
65            a.approved_writes,
66        )
67        .into_iter()
68        .filter(|d| !a.raw_changes.iter().any(|c| c.path == d.path))
69        .map(|d| format!("integrity: {} input {}: {}", d.class, d.change, d.path));
70    let frozen: BTreeSet<String> = a
71        .frozen
72        .iter()
73        .filter(|p| !a.approved_writes.contains(*p))
74        .cloned()
75        .collect();
76    let mut findings: Vec<String> = integrity::audit_in(
77        a.policy.protected(),
78        a.raw_changes,
79        a.patch,
80        &frozen,
81        a.frozen_additions,
82        a.approved_writes,
83    )
84    .into_iter()
85    .map(|f| format!("integrity: {f}"))
86    .chain(unseen)
87    .collect();
88    for d in a.command_drift.as_array().into_iter().flatten() {
89        let finding = format!(
90            "integrity: judge input changed by a command: {} ({})",
91            d["path"].as_str().unwrap_or_default(),
92            d["change"].as_str().unwrap_or_default()
93        );
94        if !findings.contains(&finding) {
95            findings.push(finding);
96        }
97    }
98    findings
99}
100
101/// `(path, added|modified|deleted)` for the report.
102pub(crate) fn changes_of(raw: Vec<worktree::Change>) -> Vec<(String, String)> {
103    raw.into_iter()
104        .map(|c| {
105            let kind = match c.kind {
106                worktree::ChangeKind::Added => "added",
107                worktree::ChangeKind::Modified => "modified",
108                worktree::ChangeKind::Deleted => "deleted",
109            };
110            (c.path, kind.to_owned())
111        })
112        .collect()
113}
114
115/// The verdict: the fast checks against the baseline, and the acceptance
116/// tests green — task evidence only when they were red first. Any integrity
117/// finding fails it, whatever the checks said.
118pub(crate) fn judge_checks(
119    project: &Path,
120    runner: &sandbox::Runner,
121    baseline: &VerifyReport,
122    acceptance: Option<&PreparedAcceptance>,
123    findings: &[String],
124    report: &mut RunReport,
125) -> Result<(), String> {
126    let mut current = verify::run_checks(project, Some(verify::config::Tier::Fast), runner)?;
127    let mut task_checks: Vec<&str> = Vec::new();
128    if let Some(a) = acceptance.filter(|a| a.argv.is_some()) {
129        let argv = a.argv.as_deref().unwrap_or_default();
130        let all: Vec<&[String]> = std::iter::once(argv)
131            .chain(a.more_argv.iter().map(Vec::as_slice))
132            .collect();
133        let green = verify::rerun_once(run_all_acceptance(project, &all, runner), || {
134            run_all_acceptance(project, &all, runner)
135        });
136        if let Some(r) = report.acceptance.as_mut() {
137            r["green"] = serde_json::json!(green.status == CheckStatus::Passed);
138            if green.flaky {
139                r["green_flaky"] = serde_json::json!(true);
140                r["note"] = serde_json::json!("acceptance passed only on re-run");
141            }
142        }
143        if a.red {
144            task_checks.push("acceptance");
145        }
146        current.checks.push(green);
147    }
148    let cmp = compare(baseline, &current);
149    current.passed = passes(&current.checks, Some(&cmp));
150    let judged = verdict(&task_checks, &current.checks, Some(&cmp));
151    report.verdict = Some(if findings.is_empty() {
152        judged
153    } else {
154        let mut reasons = findings.to_vec();
155        if let Verdict::Failed { reasons: more } = judged {
156            reasons.extend(more);
157        }
158        Verdict::Failed { reasons }
159    });
160    current.comparison = Some(cmp);
161    report.verification = Some(current);
162    Ok(())
163}
164
165/// Every command of an acceptance set, as one result: passed only when
166/// each passed (the first failure's output is kept).
167fn run_all_acceptance(
168    project: &Path,
169    all: &[&[String]],
170    runner: &sandbox::Runner,
171) -> speccoder_core::verify::CheckResult {
172    let mut first = None;
173    for argv in all {
174        let r = run_acceptance(project, argv, runner);
175        if r.status != CheckStatus::Passed {
176            return r;
177        }
178        first.get_or_insert(r);
179    }
180    first.unwrap_or_else(|| run_acceptance(project, &[], runner))
181}
182
183/// Why a failed, completed attempt is not retried, if it is not.
184pub(crate) fn retry_stop(
185    findings: &[String],
186    verification: &VerifyReport,
187    last_fingerprint: Option<&str>,
188    fingerprint: &str,
189    attempt_no: u32,
190    max_attempts: u32,
191) -> Option<String> {
192    if !findings.is_empty() {
193        Some("integrity findings are not retried".to_owned())
194    } else if let Some(infra) = speccoder_core::retry::infrastructure(verification) {
195        Some(format!("not retried: {infra}"))
196    } else if last_fingerprint == Some(fingerprint) {
197        Some(format!("same failure twice: {fingerprint}"))
198    } else if attempt_no >= max_attempts {
199        Some(format!("{attempt_no} of {max_attempts} attempts used"))
200    } else {
201        None
202    }
203}
204
205/// The gaming monitor's view of a passing attempt, and whether it suspects
206/// gaming. A monitor that fails blocks nothing: it can only add doubt.
207pub(crate) fn monitor_attempt(
208    rt: &tokio::runtime::Runtime,
209    plane: &crate::substrate::Plane,
210    feature: &str,
211    task: &str,
212    input: agentplane::core::Tainted<Value>,
213    secret: Option<&str>,
214) -> (Value, bool) {
215    match rt.block_on(plane.monitor(feature, task, input)) {
216        Ok(o) => {
217            let out = o
218                .output
219                .as_ref()
220                .map(|v| v.peek().clone())
221                .unwrap_or(Value::Null);
222            let suspected = out["answer"]["suspected"] == true;
223            (
224                serde_json::json!({
225                    "suspected": suspected,
226                    "findings": out["answer"]["findings"],
227                    "usage": out["usage"],
228                    "run_id": o.run_id.to_string(),
229                }),
230                suspected,
231            )
232        }
233        Err(e) => (
234            serde_json::json!({ "error": super::redact(&e.0, secret) }),
235            false,
236        ),
237    }
238}
speccoder_runtime/pipeline/judge.rs

speccoder_runtime/pipeline/
judge.rs
