speccoder_runtime/
evidence.rs

1//! A task's evidence as rows: its verdict and what
2//! established it, read from the signed statements on its commit
3//! and the report — never re-derived, so what an interface shows is what a
4//! reviewer can verify with `attest verify`. Every interface renders these
5//! rows; none computes its own.
6
7use serde_json::Value;
8use speccoder_core::{RunReport, TaskStatus};
9
10pub use speccoder_core::evidence::Row;
11
12/// The banner: the verdict's mark and its words, which never blur.
13#[must_use]
14pub fn banner(state: &str) -> (&'static str, &'static str) {
15    match state {
16        "verified" => ("verified", "VERIFIED"),
17        "regression_only" => (
18            "regression_only",
19            "REGRESSION ONLY — nothing task-specific was shown",
20        ),
21        "attested" => ("attested", "ATTESTED — a person's word, not evidence"),
22        "ticked_without_evidence" => (
23            "ticked_without_evidence",
24            "TICKED WITHOUT SPECCODER EVIDENCE",
25        ),
26        "evidence_without_tick" => (
27            "evidence_without_tick",
28            "COMMITTED — its local tick is missing",
29        ),
30        "skipped" => ("skipped", "SKIPPED"),
31        "ready" => ("ready", "READY"),
32        _ => ("blocked", "BLOCKED"),
33    }
34}
35
36/// The statements of a commit's note, decoded.
37#[must_use]
38pub fn statements(repo: &std::path::Path, commit: &str) -> Vec<Value> {
39    let Some(note) = crate::attest::notes::read(repo, commit) else {
40        return Vec::new();
41    };
42    crate::attest::dsse::decode(&note)
43        .0
44        .iter()
45        .filter_map(crate::attest::dsse::Envelope::statement)
46        .collect()
47}
48
49fn phase<'a>(statements: &'a [Value], name: &str) -> Option<&'a Value> {
50    statements.iter().find_map(|s| {
51        let p = &s["predicate"];
52        (p["configuration"][0]["name"] == format!("speccoder:phase/{name}")).then_some(p)
53    })
54}
55
56fn names(v: &Value) -> usize {
57    v.as_array().map_or(0, Vec::len)
58}
59
60/// The rows for `task`, from its commit's `statements` and, in a hosted
61/// run, the run's own report.
62#[must_use]
63pub fn rows(task: &TaskStatus, statements: &[Value], run: Option<&RunReport>) -> Vec<Row> {
64    let mut out = Vec::new();
65    let row = |label: &str, mark: &str, text: String| Row {
66        label: label.to_owned(),
67        mark: mark.to_owned(),
68        text,
69    };
70    let tv = statements
71        .iter()
72        .find(|s| s["predicateType"] == "https://speccoder.dev/task-verification/v0")
73        .map(|s| &s["predicate"]);
74    // Acceptance: red, then green.
75    match (phase(statements, "red"), phase(statements, "green")) {
76        (Some(red), Some(green)) => out.push(row(
77            "Acceptance",
78            "verified",
79            format!(
80                "red → green: {} failing before, {} passing after (spec-derived, frozen)",
81                names(&red["failedTests"]),
82                names(&green["passedTests"])
83            ),
84        )),
85        (None, Some(green)) => out.push(row(
86            "Acceptance",
87            "regression_only",
88            format!(
89                "none red first — the checks alone ({} passing)",
90                names(&green["passedTests"])
91            ),
92        )),
93        _ => out.push(row("Acceptance", "ready", "not run".to_owned())),
94    }
95    out.push(match phase(statements, "held_out") {
96        Some(h) if h["result"] == "PASSED" => row(
97            "Held-out",
98            "verified",
99            format!(
100                "{} passed (judge only, never shown to the implementer)",
101                names(&h["passedTests"])
102            ),
103        ),
104        Some(h) => row(
105            "Held-out",
106            "failed",
107            format!("{} failed", names(&h["failedTests"])),
108        ),
109        None => row("Held-out", "ready", "not run".to_owned()),
110    });
111    out.push(match phase(statements, "baseline") {
112        Some(b) => {
113            let warned = names(&b["warnedTests"]);
114            row(
115                "Checks",
116                if warned == 0 {
117                    "verified"
118                } else {
119                    "regression_only"
120                },
121                if warned == 0 {
122                    "baseline green, nothing regressed".to_owned()
123                } else {
124                    format!(
125                        "{warned} failing before the attempt (not blamed on it), nothing regressed"
126                    )
127                },
128            )
129        }
130        None => row("Checks", "ready", "no record".to_owned()),
131    });
132    if let Some(tv) = tv {
133        let findings = names(&tv["integrity"]["findings"]);
134        out.push(row(
135            "Diff audit",
136            if findings == 0 { "verified" } else { "failed" },
137            if findings == 0 {
138                "no tests weakened, no skips, no config changes".to_owned()
139            } else {
140                format!("{findings} finding(s)")
141            },
142        ));
143        let monitor = tv["monitor"]["state"]
144            .as_str()
145            .or_else(|| tv["monitor"].as_str())
146            .unwrap_or("off");
147        out.push(row(
148            "Gaming monitor",
149            match monitor {
150                "clean" => "verified",
151                "suspected" => "needs_approval",
152                _ => "ready",
153            },
154            monitor.to_owned(),
155        ));
156        let mutation = &tv["mutation"];
157        out.push(if mutation.is_null() {
158            row("Mutation", "ready", "not run".to_owned())
159        } else {
160            let caught = mutation["caught"].as_u64().unwrap_or(0);
161            let missed = mutation["missed"].as_u64().unwrap_or(0);
162            row(
163                "Mutation",
164                if missed == 0 {
165                    "verified"
166                } else {
167                    "regression_only"
168                },
169                format!("{caught} caught, {missed} survived"),
170            )
171        });
172        let decisions: Vec<String> = tv["decisions"]
173            .as_array()
174            .into_iter()
175            .flatten()
176            .map(|d| {
177                format!(
178                    "{} {} — {}{}",
179                    d["verb"].as_str().unwrap_or("decided"),
180                    d["grant"]
181                        .as_str()
182                        .or_else(|| d["tool"].as_str())
183                        .unwrap_or_default(),
184                    d["by"].as_str().unwrap_or("?"),
185                    d["via"]
186                        .as_str()
187                        .map(|v| format!(" via {v}"))
188                        .unwrap_or_default()
189                )
190            })
191            .collect();
192        out.push(row(
193            "Approvals",
194            if decisions.is_empty() {
195                "ready"
196            } else {
197                "verified"
198            },
199            if decisions.is_empty() {
200                "none needed".to_owned()
201            } else {
202                decisions.join("; ")
203            },
204        ));
205        let instructions: Vec<&str> = tv["instructions"]
206            .as_array()
207            .into_iter()
208            .flatten()
209            .filter_map(|i| i["path"].as_str())
210            .collect();
211        out.push(row(
212            "Instructions",
213            "ready",
214            if instructions.is_empty() {
215                "none".to_owned()
216            } else {
217                instructions.join(", ")
218            },
219        ));
220        if let Some(by) = tv["decider"]["by"].as_str() {
221            out.push(row(
222                "Decided by",
223                "attested",
224                format!(
225                    "{by} — {}",
226                    tv["decider"]["reason"].as_str().unwrap_or_default()
227                ),
228            ));
229        }
230    }
231    // Acceptance strength: how well the set tells right from wrong.
232    if let Some(st) = tv
233        .and_then(|tv| tv.get("strength"))
234        .filter(|v| !v.is_null())
235    {
236        let share = st["combined"]["share"].as_f64();
237        let rejected = st["permitted"]
238            .as_array()
239            .into_iter()
240            .flatten()
241            .any(|p| p["outcome"] == "rejected");
242        out.push(row(
243            "Strength",
244            match share {
245                _ if rejected => "needs_approval",
246                Some(x) if x >= 0.999 => "verified",
247                Some(_) => "regression_only",
248                None => "ready",
249            },
250            task.strength.clone().unwrap_or_else(|| {
251                share.map_or_else(|| "not measured".to_owned(), |x| format!("{x:.2}"))
252            }),
253        ));
254    } else if let Some(st) = &task.strength {
255        out.push(row("Strength", "ready", st.clone()));
256    }
257    out.push(match task.attestation.as_deref() {
258        Some(a) if a.starts_with("signed") => row("Attestation", "verified", a.to_owned()),
259        Some(a) => row("Attestation", "regression_only", a.to_owned()),
260        None => row(
261            "Attestation",
262            "ready",
263            "none (not ticked by speccoder)".to_owned(),
264        ),
265    });
266    if let Some(a) = &task.last_attempt {
267        out.push(row(
268            "Last attempt",
269            a.outcome.as_str(),
270            format!(
271                "{} — {} ({})",
272                a.outcome.replace('_', " "),
273                a.reason,
274                a.reference
275            ),
276        ));
277    }
278    if let Some(r) = run {
279        for e in &r.earlier {
280            out.push(row(
281                "Earlier attempt",
282                "failed",
283                format!("{} — run {}", e.attempt_status.replace('_', " "), e.run_id),
284            ));
285        }
286        if let Some(claim) = r.model_report.as_ref().and_then(|m| m["summary"].as_str()) {
287            out.push(row(
288                "Model says",
289                "ready",
290                format!("\"{claim}\" (a claim, not evidence)"),
291            ));
292        }
293    }
294    out
295}
296
297/// The model's account of `run`, turn by turn, read from the project's
298/// journal: its reasoning and its words — a claim, shown beside the
299/// evidence, never as it.
300///
301/// # Errors
302///
303/// No project or state directory, or the journal cannot be read.
304#[cfg(feature = "engine")]
305pub fn account_of(
306    path: &std::path::Path,
307    run: &str,
308) -> Result<Vec<crate::substrate::AccountTurn>, String> {
309    let root = crate::format::at(path)
310        .detect(path)
311        .ok_or_else(|| "not a spec project".to_owned())?;
312    let state = crate::worktree::state_dir().ok_or_else(|| "no state directory".to_owned())?;
313    let plane = crate::worktree::project_state_dir(&state, &root);
314    crate::pipeline::tokio_runtime()?
315        .block_on(crate::substrate::account(&plane, run))
316        .map_err(|e| e.0)
317}
318
319#[cfg(test)]
320mod tests {
321    use super::*;
322    use serde_json::json;
323
324    fn task(state: &str) -> TaskStatus {
325        TaskStatus {
326            id: "T002".to_owned(),
327            title: "t".to_owned(),
328            phase: "P".to_owned(),
329            state: state.to_owned(),
330            drift: None,
331            blocked_by: Vec::new(),
332            commit: Some("abc".to_owned()),
333            run_id: None,
334            last_attempt: None,
335            set_aside: None,
336            bug: None,
337            note: None,
338            attestation: Some("signed ssh SHA256:x".to_owned()),
339            strength: None,
340        }
341    }
342
343    fn phase_stmt(name: &str, result: &str, passed: &[&str], failed: &[&str]) -> Value {
344        json!({ "predicateType": "https://in-toto.io/attestation/test-result/v0.1", "predicate": {
345            "result": result, "configuration": [{ "name": format!("speccoder:phase/{name}") }],
346            "passedTests": passed, "failedTests": failed, "warnedTests": [] } })
347    }
348
349    #[test]
350    fn rows_come_from_the_statements() {
351        let statements = vec![
352            phase_stmt("baseline", "PASSED", &["a"], &[]),
353            phase_stmt("red", "FAILED", &[], &["accept::login"]),
354            phase_stmt("green", "PASSED", &["accept::login", "a"], &[]),
355            json!({ "predicateType": "https://speccoder.dev/task-verification/v0", "predicate": {
356                "integrity": { "findings": [] }, "monitor": { "state": "clean" },
357                "mutation": { "caught": 26, "missed": 3 },
358                "decisions": [{ "verb": "approve", "grant": "run `cargo add x`", "by": "Ada", "via": "dashboard" }],
359                "instructions": [{ "path": "AGENTS.md" }] } }),
360        ];
361        let rows = rows(&task("verified"), &statements, None);
362        let text = |label: &str| {
363            rows.iter()
364                .find(|r| r.label == label)
365                .map(|r| r.text.clone())
366                .unwrap_or_default()
367        };
368        assert!(
369            text("Acceptance").contains("1 failing before, 2 passing after"),
370            "{rows:?}"
371        );
372        assert_eq!(text("Held-out"), "not run");
373        assert_eq!(text("Mutation"), "26 caught, 3 survived");
374        assert!(text("Approvals").contains("Ada via dashboard"));
375        assert_eq!(text("Instructions"), "AGENTS.md");
376        assert!(text("Attestation").starts_with("signed"));
377    }
378
379    #[test]
380    fn banners_never_blur() {
381        assert_eq!(banner("verified").1, "VERIFIED");
382        assert!(
383            banner("regression_only")
384                .1
385                .contains("nothing task-specific")
386        );
387        assert!(banner("attested").1.contains("not evidence"));
388        assert_ne!(banner("attested").0, banner("verified").0);
389    }
390}
speccoder_runtime/evidence.rs

speccoder_runtime/
evidence.rs
