warble_eval_runner/
capture.rs

1//! Capture-confirmed — golden growth (roadmap Phase 1.4 step 5).
2//!
3//! Turns one *confirmed* run into a candidate golden case: given the question, the captured
4//! `{columns, rows}` result, and the context version it was confirmed against, it renders a
5//! golden-shaped YAML case for a human to accept into the golden set. This is the OSS **local, basic**
6//! hook only — scale generation and an annotation UI are SaaS. Nothing here
7//! auto-accepts: a captured result becomes a *candidate*, never a golden, until a person moves it in.
8//!
9//! Soft-depends on the 1.3 conversation runtime for the "confirmed" signal; until that surfaces one,
10//! this is driven by hand — pipe a captured result in, get a candidate out.
11
12use serde::Serialize;
13
14/// The recognized result-comparison modes (mirrors `warble_eval_compare::MatchMode`).
15const MATCH_MODES: [&str; 3] = ["scalar", "set", "ordered"];
16
17/// Everything needed to mint one candidate golden case.
18pub struct CaptureInput<'a> {
19    pub id: &'a str,
20    pub question: &'a str,
21    /// `scalar` | `set` | `ordered`.
22    pub match_mode: &'a str,
23    pub numeric_tolerance: f64,
24    pub tags: Vec<String>,
25    /// The agent's final text (raw, or a `claude -p --output-format json` envelope already unwrapped
26    /// to its `.result`) — the `{columns, rows}` object is extracted from it.
27    pub result_text: &'a str,
28}
29
30#[derive(Serialize)]
31struct Tol {
32    numeric: f64,
33}
34
35#[derive(Serialize)]
36struct CandidateCase {
37    id: String,
38    question: String,
39    #[serde(skip_serializing_if = "Vec::is_empty")]
40    tags: Vec<String>,
41    #[serde(rename = "match")]
42    match_mode: String,
43    tolerance: Tol,
44    expected: serde_json::Value,
45}
46
47/// The candidates staging file header — a golden-shaped document so accepted candidates can be moved
48/// straight into a golden set. Written once when the file is first created.
49pub fn candidates_header(dataset: Option<&str>, context_version: Option<&str>) -> String {
50    let mut out = String::from(
51        "# Candidate goldens captured from confirmed runs (`warble eval capture`).\n\
52# Review each case, then move accepted ones into the golden set. NOT auto-accepted.\n",
53    );
54    if let Some(d) = dataset {
55        out.push_str(&format!("dataset: {d}\n"));
56    }
57    if let Some(cv) = context_version {
58        out.push_str(&format!("context_version: {cv}\n"));
59    }
60    out.push_str("cases:\n");
61    out
62}
63
64/// Render one confirmed run as a golden-shaped case block (a single `- …` list item), ready to
65/// append under a candidates file's `cases:`. Fails if the result carries no `{columns, rows}` or the
66/// match mode is unrecognized — a candidate that wouldn't parse as a golden is not worth writing.
67pub fn build_candidate_yaml(input: &CaptureInput) -> Result<String, String> {
68    if !MATCH_MODES.contains(&input.match_mode) {
69        return Err(format!(
70            "unknown match mode '{}' (expected one of: {})",
71            input.match_mode,
72            MATCH_MODES.join(", ")
73        ));
74    }
75    let expected = crate::extract_result_json(input.result_text).ok_or_else(|| {
76        "no parseable {columns, rows} object in the captured result — nothing to capture"
77            .to_string()
78    })?;
79
80    let case = CandidateCase {
81        id: input.id.to_string(),
82        question: input.question.to_string(),
83        tags: input.tags.clone(),
84        match_mode: input.match_mode.to_string(),
85        tolerance: Tol {
86            numeric: input.numeric_tolerance,
87        },
88        expected,
89    };
90    // Serializing a single-element Vec yields exactly a `cases:` list item (`- id: …`).
91    serde_yaml::to_string(&vec![case]).map_err(|e| format!("serialize candidate: {e}"))
92}
93
94#[cfg(test)]
95mod tests {
96    use super::*;
97    use crate::{Golden, GoldenCase};
98
99    fn input<'a>(result: &'a str, match_mode: &'a str) -> CaptureInput<'a> {
100        CaptureInput {
101            id: "q_new_case",
102            question: "How many orders shipped in Q3?",
103            match_mode,
104            numeric_tolerance: 0.01,
105            tags: vec!["agg".into(), "time-filter".into()],
106            result_text: result,
107        }
108    }
109
110    #[test]
111    fn candidate_from_fenced_claude_result_parses_back_as_a_golden() {
112        // A confirmed run's final text, with the result in a fenced block (as agents emit).
113        let result = "Here you go:\n```json\n{\"columns\":[\"n\"],\"rows\":[[42]]}\n```\n";
114        let block = build_candidate_yaml(&input(result, "scalar")).unwrap();
115
116        // Assemble a full candidates file and prove it round-trips through the golden parser.
117        let doc = format!(
118            "{}{}",
119            candidates_header(Some("jaffle_shop"), Some("jaffle_shop@abc1234")),
120            block
121        );
122        let golden: Golden = serde_yaml::from_str(&doc).expect("candidate file parses as a golden");
123        assert_eq!(golden.dataset.as_deref(), Some("jaffle_shop"));
124        assert_eq!(
125            golden.context_version.as_deref(),
126            Some("jaffle_shop@abc1234")
127        );
128        assert_eq!(golden.cases.len(), 1);
129        let case: &GoldenCase = &golden.cases[0];
130        assert_eq!(case.id, "q_new_case");
131        assert_eq!(case.expected.columns, vec!["n"]);
132        assert_eq!(case.tags, vec!["agg", "time-filter"]);
133    }
134
135    #[test]
136    fn candidate_from_bare_object_works() {
137        let result = "{\"columns\":[\"seg\",\"rev\"],\"rows\":[[\"ent\",100],[\"smb\",50]]}";
138        let block = build_candidate_yaml(&input(result, "set")).unwrap();
139        let doc = format!("{}{}", candidates_header(None, None), block);
140        let golden: Golden = serde_yaml::from_str(&doc).unwrap();
141        assert_eq!(golden.cases[0].expected.rows.len(), 2);
142    }
143
144    #[test]
145    fn rejects_result_without_a_table() {
146        let err =
147            build_candidate_yaml(&input("sorry, I couldn't answer that", "scalar")).unwrap_err();
148        assert!(err.contains("no parseable"));
149    }
150
151    #[test]
152    fn rejects_unknown_match_mode() {
153        let result = "{\"columns\":[\"n\"],\"rows\":[[1]]}";
154        let err = build_candidate_yaml(&input(result, "fuzzy")).unwrap_err();
155        assert!(err.contains("unknown match mode"));
156    }
157}
warble_eval_runner/capture.rs

warble_eval_runner/
capture.rs
