warble_eval_runner/
context.rs

1//! MDL-version reverify — golden lifecycle (roadmap Phase 1.4 step 6).
2//!
3//! A golden's `context_version` pins the Wren fixture's schema+knowledge context it was confirmed
4//! against. When the bound MDL changes, the golden's ground truth may have silently rotted. This module
5//! computes the **git SHA of the bound MDL files** (host-side, content-addressed via
6//! `git hash-object` — no ContextLoader, Phase-2-independent), compares it to the
7//! golden's pin, and flags a mismatch as `stale`. Stale goldens can be re-stamped (accept the new
8//! MDL) or re-verified (re-run to see which cases the MDL change actually moved).
9//!
10//! Only the MDL semantics travel into the SHA (`*.yml`/`*.yaml`/`*.md` under the project), so a pure
11//! connection/credential edit that leaves the models untouched does not spuriously mark goldens stale.
12
13use serde::Serialize;
14use std::path::{Path, PathBuf};
15use std::process::Command;
16
17/// Whether a golden's pinned `context_version` still matches the bound MDL.
18#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
19#[serde(rename_all = "lowercase")]
20pub enum Freshness {
21    /// Pin is a SHA and matches the current MDL SHA (exact or abbreviated prefix).
22    Fresh,
23    /// Pin is a SHA but the MDL has since changed — ground truth may have rotted.
24    Stale,
25    /// Pin is symbolic (not a SHA), so it can't be checked; stamp it to start tracking.
26    Unpinned,
27}
28
29#[derive(Debug, Clone, Serialize)]
30pub struct VerifyResult {
31    pub freshness: Freshness,
32    /// The pin recorded in the golden's `context_version` (the part after `@`, or the whole value).
33    pub pinned: String,
34    /// The dataset prefix (the part before `@`), when present.
35    pub dataset: Option<String>,
36    /// The freshly computed MDL SHA.
37    pub current_sha: String,
38    pub detail: String,
39}
40
41/// Split a `context_version` (`<dataset>@<pin>`) into its dataset prefix and pin. With no `@`, the
42/// whole value is the pin and there is no dataset.
43pub fn parse_context_version(cv: &str) -> (Option<String>, String) {
44    match cv.split_once('@') {
45        Some((dataset, pin)) => (Some(dataset.to_string()), pin.to_string()),
46        None => (None, cv.to_string()),
47    }
48}
49
50/// A pin looks like a git SHA when it is 7–64 hex chars (git's abbreviated..full range).
51pub fn is_sha_like(pin: &str) -> bool {
52    let len = pin.len();
53    (7..=64).contains(&len) && pin.chars().all(|c| c.is_ascii_hexdigit())
54}
55
56/// Classify a pin against a freshly computed MDL SHA.
57pub fn classify(pin: &str, current_sha: &str) -> Freshness {
58    if !is_sha_like(pin) {
59        Freshness::Unpinned
60    } else if current_sha == pin || current_sha.starts_with(pin) {
61        Freshness::Fresh
62    } else {
63        Freshness::Stale
64    }
65}
66
67/// Collect the MDL files under `project` (`*.yml`/`*.yaml`/`*.md`), sorted by relative path, skipping
68/// runtime dirs (`.git`, `.claude`, `.venv`) that are not part of the semantic definition.
69fn mdl_files(project: &Path) -> Result<Vec<PathBuf>, String> {
70    let mut out = Vec::new();
71    collect(project, &mut out)?;
72    out.sort();
73    Ok(out)
74}
75
76fn collect(dir: &Path, out: &mut Vec<PathBuf>) -> Result<(), String> {
77    for entry in std::fs::read_dir(dir).map_err(|e| format!("read {}: {e}", dir.display()))? {
78        let entry = entry.map_err(|e| e.to_string())?;
79        let path = entry.path();
80        let name = entry.file_name();
81        let name = name.to_string_lossy();
82        if path.is_dir() {
83            if matches!(name.as_ref(), ".git" | ".claude" | ".venv" | "node_modules") {
84                continue;
85            }
86            collect(&path, out)?;
87        } else if matches!(
88            path.extension().and_then(|e| e.to_str()),
89            Some("yml") | Some("yaml") | Some("md")
90        ) {
91            out.push(path);
92        }
93    }
94    Ok(())
95}
96
97/// One `git hash-object <file>` blob SHA (content-addressed; works outside a repo, stable across
98/// machines — the property we want for a pin).
99fn blob_sha(file: &Path) -> Result<String, String> {
100    let output = Command::new("git")
101        .arg("hash-object")
102        .arg(file)
103        .output()
104        .map_err(|e| format!("git hash-object {}: {e}", file.display()))?;
105    if !output.status.success() {
106        return Err(format!(
107            "git hash-object {} failed: {}",
108            file.display(),
109            String::from_utf8_lossy(&output.stderr).trim()
110        ));
111    }
112    Ok(String::from_utf8_lossy(&output.stdout).trim().to_string())
113}
114
115/// Hash an arbitrary string to a stable 40-hex git blob SHA (content-addressed; no repo needed).
116/// Written via a temp file rather than a `--stdin` pipe, matching [`blob_sha`]. Shared by the MDL
117/// SHA, the agent-artifact SHA, and the trace-cache key hash — same content → same hash on any
118/// machine, which is exactly the property a cache key wants.
119pub(crate) fn hash_str(s: &str) -> Result<String, String> {
120    let tmp = tempfile::NamedTempFile::new().map_err(|e| format!("tempfile: {e}"))?;
121    std::fs::write(tmp.path(), s).map_err(|e| format!("write hash input: {e}"))?;
122    blob_sha(tmp.path())
123}
124
125/// Build a `<relpath> <blob-sha>\n` manifest over `files` (already sorted), relative to `root`.
126fn file_manifest(root: &Path, files: &[PathBuf]) -> Result<String, String> {
127    let mut manifest = String::new();
128    for file in files {
129        let rel = file.strip_prefix(root).unwrap_or(file);
130        manifest.push_str(&format!("{} {}\n", rel.display(), blob_sha(file)?));
131    }
132    Ok(manifest)
133}
134
135/// Compute a single stable SHA over all MDL files: hash each file's content (git blob SHA), build a
136/// sorted `<relpath> <sha>` manifest, and hash the manifest. Any content or path change moves it.
137pub fn compute_mdl_sha(project: &Path) -> Result<String, String> {
138    let files = mdl_files(project)?;
139    if files.is_empty() {
140        return Err(format!(
141            "no MDL files (*.yml/*.yaml/*.md) under {}",
142            project.display()
143        ));
144    }
145    hash_str(&file_manifest(project, &files)?)
146}
147
148/// Compute a stable SHA over every file under `dir` (all extensions, recursive), the same way
149/// [`compute_mdl_sha`] hashes MDL files. Used for the **agent-artifact SHA**: the dispatched agent
150/// dir (`.claude/agents/*.md`, `settings.json`, `.mcp.json`, …) fully determines the agent's
151/// behavior, so hashing it gives the `agent_sha` component of the trace-cache key — a change to any
152/// emitted file (prompt, tier binding, settings) moves the SHA and correctly misses the cache.
153pub(crate) fn compute_dir_sha(dir: &Path) -> Result<String, String> {
154    let mut files = Vec::new();
155    all_files(dir, &mut files)?;
156    if files.is_empty() {
157        return Err(format!("no files under {}", dir.display()));
158    }
159    files.sort();
160    hash_str(&file_manifest(dir, &files)?)
161}
162
163/// Compute a single file's content SHA (git blob SHA) — the single-artifact analogue of
164/// [`compute_dir_sha`], used for the **`claude-agent-sdk` back-end's agent-artifact SHA**: that
165/// back-end has no emitted agent dir to hash (it dispatches straight from a compiled `ir.json`), so
166/// the IR file itself stands in as the artifact whose content determines the run.
167pub(crate) fn compute_file_sha(file: &Path) -> Result<String, String> {
168    blob_sha(file)
169}
170
171/// Collect every regular file under `dir` recursively, skipping the same runtime dirs as
172/// [`collect`] (`.git`, `.venv`, `node_modules`) so build/VCS noise stays out of the SHA.
173fn all_files(dir: &Path, out: &mut Vec<PathBuf>) -> Result<(), String> {
174    for entry in std::fs::read_dir(dir).map_err(|e| format!("read {}: {e}", dir.display()))? {
175        let entry = entry.map_err(|e| e.to_string())?;
176        let path = entry.path();
177        let name = entry.file_name();
178        let name = name.to_string_lossy();
179        if path.is_dir() {
180            if matches!(name.as_ref(), ".git" | ".venv" | "node_modules" | "target") {
181                continue;
182            }
183            all_files(&path, out)?;
184        } else {
185            out.push(path);
186        }
187    }
188    Ok(())
189}
190
191/// Verify a golden's `context_version` against the bound project's current MDL SHA.
192pub fn verify_context(
193    golden_context_version: Option<&str>,
194    project: &Path,
195) -> Result<VerifyResult, String> {
196    let current_sha = compute_mdl_sha(project)?;
197    let cv = golden_context_version.unwrap_or("");
198    let (dataset, pinned) = parse_context_version(cv);
199    let freshness = if cv.is_empty() {
200        Freshness::Unpinned
201    } else {
202        classify(&pinned, &current_sha)
203    };
204    let detail = match freshness {
205        Freshness::Fresh => format!("pin '{pinned}' matches current MDL SHA"),
206        Freshness::Stale => format!(
207            "pin '{pinned}' != current MDL SHA {current_sha} — MDL changed; ground truth may be stale"
208        ),
209        Freshness::Unpinned => {
210            if cv.is_empty() {
211                format!("golden has no context_version; current MDL SHA is {current_sha}")
212            } else {
213                format!("pin '{pinned}' is symbolic (not a SHA); current MDL SHA is {current_sha}")
214            }
215        }
216    };
217    Ok(VerifyResult {
218        freshness,
219        pinned,
220        dataset,
221        current_sha,
222        detail,
223    })
224}
225
226/// Rewrite a golden YAML's `context_version:` line to `<dataset>@<sha>`, preserving the dataset
227/// prefix when the golden already had one (else falling back to `fallback_dataset`). Line-based so
228/// comments and formatting survive. Returns the new document text.
229pub fn stamp_context_version(
230    golden_text: &str,
231    sha: &str,
232    fallback_dataset: Option<&str>,
233) -> Result<String, String> {
234    let mut lines: Vec<String> = golden_text.lines().map(str::to_string).collect();
235    let idx = lines
236        .iter()
237        .position(|l| l.trim_start().starts_with("context_version:"));
238
239    // Derive the dataset prefix from the existing value, else the golden's `dataset:`, else none.
240    let existing_dataset = idx.and_then(|i| {
241        let after = lines[i].split_once("context_version:")?.1.trim();
242        // strip trailing inline comment
243        let value = after
244            .split_once('#')
245            .map(|(v, _)| v.trim())
246            .unwrap_or(after);
247        parse_context_version(value).0
248    });
249    let dataset = existing_dataset.or_else(|| fallback_dataset.map(str::to_string));
250    let new_value = match &dataset {
251        Some(d) => format!("{d}@{sha}"),
252        None => sha.to_string(),
253    };
254    let new_line = format!(
255        "context_version: {new_value}      # stamped by `warble eval verify-context --stamp`"
256    );
257
258    match idx {
259        Some(i) => lines[i] = new_line,
260        None => {
261            // Insert after `dataset:` if present, else at the top.
262            let insert_at = lines
263                .iter()
264                .position(|l| l.trim_start().starts_with("dataset:"))
265                .map(|i| i + 1)
266                .unwrap_or(0);
267            lines.insert(insert_at, new_line);
268        }
269    }
270    Ok(format!("{}\n", lines.join("\n")))
271}
272
273#[cfg(test)]
274mod tests {
275    use super::*;
276    use std::fs;
277
278    #[test]
279    fn parse_splits_dataset_and_pin() {
280        assert_eq!(
281            parse_context_version("jaffle_shop@a1b2c3d"),
282            (Some("jaffle_shop".into()), "a1b2c3d".into())
283        );
284        assert_eq!(parse_context_version("bareword"), (None, "bareword".into()));
285    }
286
287    #[test]
288    fn sha_like_detection() {
289        assert!(is_sha_like("a1b2c3d")); // 7 hex
290        assert!(is_sha_like("ce013625030ba8dba906f756967f9e9ca394464a")); // 40 hex
291        assert!(!is_sha_like("frozen-poc")); // has '-'
292        assert!(!is_sha_like("abc")); // too short
293    }
294
295    #[test]
296    fn classify_fresh_stale_unpinned() {
297        let sha = "ce013625030ba8dba906f756967f9e9ca394464a";
298        assert_eq!(classify(sha, sha), Freshness::Fresh);
299        assert_eq!(
300            classify("ce01362", sha),
301            Freshness::Fresh,
302            "abbrev prefix matches"
303        );
304        assert_eq!(classify("deadbeef", sha), Freshness::Stale);
305        assert_eq!(classify("frozen-poc", sha), Freshness::Unpinned);
306    }
307
308    #[test]
309    fn mdl_change_moves_the_sha_and_marks_stale() {
310        // The deliverable: an MDL edit must move the SHA → stale detection fires.
311        let dir = tempfile::tempdir().unwrap();
312        fs::create_dir_all(dir.path().join("models")).unwrap();
313        fs::write(
314            dir.path().join("wren_project.yml"),
315            "catalog: c\nschema: s\n",
316        )
317        .unwrap();
318        fs::write(
319            dir.path().join("models/orders.yml"),
320            "name: orders\ncolumns: [id, amount]\n",
321        )
322        .unwrap();
323
324        let sha_before = compute_mdl_sha(dir.path()).unwrap();
325        assert_eq!(sha_before.len(), 40, "40-hex git sha");
326
327        // A pin equal to the pre-change SHA is Fresh…
328        assert_eq!(classify(&sha_before, &sha_before), Freshness::Fresh);
329
330        // …edit an MDL file…
331        fs::write(
332            dir.path().join("models/orders.yml"),
333            "name: orders\ncolumns: [id, amount, discount]\n",
334        )
335        .unwrap();
336        let sha_after = compute_mdl_sha(dir.path()).unwrap();
337        assert_ne!(sha_before, sha_after, "MDL change must move the SHA");
338
339        // …and the old pin is now Stale against the new MDL.
340        let verify = verify_context(Some(&format!("jaffle@{sha_before}")), dir.path()).unwrap();
341        assert_eq!(verify.freshness, Freshness::Stale);
342        assert_eq!(verify.current_sha, sha_after);
343    }
344
345    #[test]
346    fn adding_a_model_file_moves_the_sha() {
347        let dir = tempfile::tempdir().unwrap();
348        fs::write(dir.path().join("a.yml"), "x: 1\n").unwrap();
349        let before = compute_mdl_sha(dir.path()).unwrap();
350        fs::write(dir.path().join("b.yml"), "y: 2\n").unwrap();
351        let after = compute_mdl_sha(dir.path()).unwrap();
352        assert_ne!(before, after, "adding an MDL file changes the manifest SHA");
353    }
354
355    #[test]
356    fn context_report_or_prompt_change_moves_the_agent_sha() {
357        let dir = tempfile::tempdir().unwrap();
358        fs::create_dir_all(dir.path().join(".claude/agents")).unwrap();
359        fs::write(
360            dir.path().join(".claude/agents/answer.md"),
361            "Context injection mode: schema-only\n",
362        )
363        .unwrap();
364        fs::write(
365            dir.path().join("context-report.json"),
366            r#"{"mode":"schema-only"}"#,
367        )
368        .unwrap();
369        let schema_only = compute_dir_sha(dir.path()).unwrap();
370
371        fs::write(
372            dir.path().join(".claude/agents/answer.md"),
373            "Context injection mode: schema+knowledge\nRULE\n",
374        )
375        .unwrap();
376        fs::write(
377            dir.path().join("context-report.json"),
378            r#"{"mode":"schema+knowledge"}"#,
379        )
380        .unwrap();
381        let with_knowledge = compute_dir_sha(dir.path()).unwrap();
382
383        assert_ne!(
384            schema_only, with_knowledge,
385            "context identity must miss cache"
386        );
387    }
388
389    #[test]
390    fn stamp_rewrites_pin_preserving_dataset_and_comments() {
391        let golden = "# header\ndataset: jaffle_shop\ncontext_version: jaffle_shop@frozen-poc   # pin\ncases: []\n";
392        let stamped = stamp_context_version(golden, "abc1234def", None).unwrap();
393        assert!(stamped.contains("context_version: jaffle_shop@abc1234def"));
394        assert!(stamped.contains("# header"), "comments preserved");
395        assert!(stamped.contains("cases: []"));
396        assert!(!stamped.contains("frozen-poc"), "old pin replaced");
397    }
398
399    #[test]
400    fn stamp_inserts_when_absent_using_fallback_dataset() {
401        let golden = "dataset: shop\ncases: []\n";
402        let stamped = stamp_context_version(golden, "abc1234", Some("shop")).unwrap();
403        assert!(stamped.contains("context_version: shop@abc1234"));
404    }
405}
warble_eval_runner/context.rs

warble_eval_runner/
context.rs
