speccoder_runtime/pipeline/
outcome.rs

1//! How a task run ends: its scope finding, what a failed attempt is saved
2//! as, the command's status and message, and the progress event.
3
4use serde_json::Value;
5use speccoder_core::scope::ScopeFinding;
6use speccoder_core::verify::Verdict;
7use speccoder_core::{CommandStatus, RunReport};
8
9use crate::progress;
10
11use super::{add_usage, monitor_summary};
12
13/// What the attempt changed beyond the paths the task line `title` names
14/// (speccoder's own acceptance tests exempt).
15pub(crate) fn scope_of(report: &RunReport, title: &str) -> ScopeFinding {
16    let exempt: Vec<String> = report
17        .acceptance
18        .as_ref()
19        .and_then(|a| a["files"].as_array())
20        .into_iter()
21        .flatten()
22        .filter_map(|f| f.as_str().map(str::to_owned))
23        .collect();
24    let changed: Vec<String> = report.changes.iter().map(|(p, _)| p.clone()).collect();
25    let named = speccoder_core::scope::named_paths(title);
26    speccoder_core::scope::finding(&named, &changed, &exempt)
27}
28
29/// The outcome and reason an attempt that did not pass is saved with.
30pub(crate) fn saved_outcome(
31    report: &RunReport,
32    attempt_status: &str,
33    suspected: bool,
34) -> (String, String) {
35    let outcome = match attempt_status {
36        _ if suspected => "suspected_gaming".to_owned(),
37        "completed" | "integrity" => "verification_failed".to_owned(),
38        other => other.to_owned(),
39    };
40    let reason = match &report.verdict {
41        _ if suspected => monitor_summary(report.monitor.as_ref()),
42        Some(Verdict::Failed { reasons }) => reasons.join("; "),
43        _ => report
44            .reason
45            .clone()
46            .unwrap_or_else(|| attempt_status.to_owned()),
47    };
48    (outcome, reason)
49}
50
51/// The command's status and message for a finished task run.
52pub(crate) fn status_and_message(
53    report: &RunReport,
54    attempt_status: &str,
55    suspected: bool,
56) -> (CommandStatus, String) {
57    match (&report.verdict, attempt_status) {
58        (_, "over_strict") => (
59            CommandStatus::NeedsHuman,
60            format!(
61                "the checks passed, but {}; the attempt is held, not ticked — rule on it (`speccoder task rule {task} --tests-right --reason …`), regenerate the tests (`speccoder run --task {task} --fresh-acceptance`) or accept it (`speccoder task accept {task}`)",
62                report.reason.clone().unwrap_or_default(),
63                task = report.task
64            ),
65        ),
66        (_, "acceptance_weak") => (
67            CommandStatus::NeedsHuman,
68            format!(
69                "the checks passed, but the {}; the attempt is saved, not ticked",
70                report.reason.clone().unwrap_or_default()
71            ),
72        ),
73        _ if suspected => (
74            CommandStatus::NeedsHuman,
75            format!(
76                "the checks passed, but the gaming monitor suspects: {}; the attempt is saved, not ticked — review it, then `speccoder task accept` or run again",
77                monitor_summary(report.monitor.as_ref())
78            ),
79        ),
80        (_, "incomplete") => (
81            CommandStatus::NeedsHuman,
82            format!(
83                "attempt incomplete by the model's own account{}",
84                if report.changes.is_empty() {
85                    "; no files changed"
86                } else {
87                    ""
88                }
89            ),
90        ),
91        (None, "collection_unknown") => (
92            CommandStatus::NeedsHuman,
93            format!(
94                "{}; the attempt is saved, not ticked",
95                report.reason.clone().unwrap_or_default()
96            ),
97        ),
98        (Some(Verdict::Failed { reasons }), _) => (
99            CommandStatus::VerificationFailed,
100            format!("verdict: failed — {}", reasons.join("; ")),
101        ),
102        (Some(v), _) => (CommandStatus::Ok, format!("verdict: {}", verdict_word(v))),
103        // A call nobody could ask about: named, with how to let it run.
104        (None, "needs_approval") if report.pending.as_ref().is_some_and(Value::is_object) => (
105            CommandStatus::NeedsHuman,
106            format!(
107                "attempt needs_approval: {}",
108                crate::tools::policy::unasked(
109                    report.pending.as_ref().unwrap_or(&Value::Null),
110                    &format!(
111                        "approve it (`speccoder task approve {}`), or run with prompts (without `--no-prompt`) to be asked",
112                        report.task
113                    ),
114                )
115            ),
116        ),
117        (
118            None,
119            "blocked" | "needs_approval" | "budget_exhausted" | "denied" | "interrupted"
120            | "held_out" | "intent_changed" | "judge_changed" | "checkout_changed" | "git_changed",
121        ) => (
122            CommandStatus::NeedsHuman,
123            format!(
124                "attempt {attempt_status}: {}",
125                report
126                    .reason
127                    .clone()
128                    .or_else(|| report
129                        .pending
130                        .as_ref()
131                        .map(|p| p["reason"].as_str().unwrap_or_default().to_owned()))
132                    .unwrap_or_default()
133            ),
134        ),
135        (None, _) => (
136            CommandStatus::RunFailed,
137            format!(
138                "attempt failed: {}",
139                report.reason.clone().unwrap_or_default()
140            ),
141        ),
142    }
143}
144
145/// `verified`, `regression_only` or `failed`.
146pub(crate) fn verdict_word(v: &Verdict) -> String {
147    serde_json::to_value(v)
148        .ok()
149        .and_then(|v| v["verdict"].as_str().map(str::to_owned))
150        .unwrap_or_default()
151}
152
153/// The progress event that ends a task run; `message` is already redacted.
154pub(crate) fn finished_event(
155    task: &str,
156    report: &RunReport,
157    attempt_status: &str,
158    suspected: bool,
159    message: String,
160) -> progress::Event {
161    let total = add_usage(
162        report.usage.as_ref(),
163        report.acceptance.as_ref().and_then(|a| a.get("usage")),
164    );
165    let tokens = |k: &str| total.as_ref().and_then(|u| u[k].as_u64()).unwrap_or(0);
166    progress::Event::TaskFinished {
167        task: task.to_owned(),
168        // The verdict for a completed attempt; otherwise how the attempt ended
169        // (an incomplete attempt with green checks is not a pass).
170        outcome: if suspected {
171            "suspected_gaming".to_owned()
172        } else if attempt_status == "completed" {
173            report
174                .verdict
175                .as_ref()
176                .map_or_else(|| attempt_status.to_owned(), verdict_word)
177        } else {
178            attempt_status.to_owned()
179        },
180        message,
181        run: (!report.run_id.is_empty()).then(|| report.run_id.clone()),
182        input_tokens: tokens("input_tokens"),
183        output_tokens: tokens("output_tokens"),
184        cache_read_tokens: tokens("cache_read_tokens"),
185        cache_write_tokens: tokens("cache_write_tokens"),
186        usd: None,
187        regression: report.regression.clone(),
188    }
189}
190
191/// A run report with nothing established yet.
192pub(crate) fn bare_report(
193    feature: &str,
194    task: &str,
195    worktree: &str,
196    attempt_status: &str,
197) -> RunReport {
198    RunReport {
199        run_id: String::new(),
200        feature: feature.to_owned(),
201        task: task.to_owned(),
202        worktree: worktree.to_owned(),
203        attempt_status: attempt_status.to_owned(),
204        model_report: None,
205        reason: None,
206        pending: None,
207        tool_calls: Vec::new(),
208        changes: Vec::new(),
209        verification: None,
210        verdict: None,
211        usage: None,
212        integrity: Vec::new(),
213        ticked: false,
214        commit: None,
215        acceptance: None,
216        earlier: Vec::new(),
217        retry_stop: None,
218        saved_attempt: None,
219        set_aside: None,
220        notices: Vec::new(),
221        monitor: None,
222        mutation: None,
223        samples: Vec::new(),
224        scope: None,
225        attestation: None,
226        instructions: Vec::new(),
227        strength: None,
228        bug: None,
229        collection: None,
230        regression: None,
231        context: None,
232    }
233}
234
235/// Why a convergence task's gap is not demonstrated: no examples, no tests
236/// written, or tests that do not fail on an assertion before any change.
237pub(crate) fn undemonstrated_why(acceptance: Option<&super::PreparedAcceptance>) -> String {
238    match acceptance {
239        None => "the spec has no acceptance examples to test it against".to_owned(),
240        Some(a) => match (a.report["skipped"].as_str(), a.report["note"].as_str()) {
241            (Some(s), _) => format!("no acceptance tests ({s})"),
242            (None, Some(note)) => match a.report["excerpt"].as_str().and_then(|e| e.lines().next())
243            {
244                Some(first) => format!("{note}: {first}"),
245                None => note.to_owned(),
246            },
247            (None, None) => "its acceptance tests already pass before any change".to_owned(),
248        },
249    }
250}
speccoder_runtime/pipeline/outcome.rs

speccoder_runtime/pipeline/
outcome.rs
