warble_eval_runner/cache.rs
1//! Trace cache — content-addressed per-case results + 0-LLM re-scoring.
2//!
3//! Every replayed golden case emits a **trace**: the agent's captured `{columns, rows}` result plus
4//! the run's cost/latency/turns and the inputs that determined it — `(case, agent_sha, model,
5//! context_sha, sample)`. The trace is written under a content-addressed key so a later run can
6//! reuse it:
7//!
8//! - **Re-score without re-run** (the killer): a golden's `expected`/`tolerance`/`match` can change
9//! while the agent, model, MDL, and sample index do not. The key deliberately excludes the
10//! expectation, so such a change still *hits* — and [`rescore`] re-compares the cached result
11//! against the **current** expectation with **zero LLM calls**. The v1→v2 golden-calibration
12//! rerun goes from ~90 min to sub-second.
13//! - **Content-addressed skip**: identical `(case, agent_sha, model, context_sha, sample)` → reuse
14//! the cached result on a plain re-run too (no re-sampling). `--no-cache` is the escape hatch that
15//! forces every case to re-run and refreshes the cache.
16//!
17//! The key material reuses the same `git hash-object` content addressing as `eval verify-context`
18//! (the MDL SHA): `agent_sha` = SHA of the dispatched agent dir, `context_sha` = MDL SHA of the
19//! bound project, and the cache filename is the SHA of the canonical key string. No new deps, no
20//! network, single machine — the OSS boundary is preserved.
21//!
22//! No silent caps: the caller surfaces hit/miss counts in the report and per-case `[cache]` markers,
23//! and a re-scored run is never mistaken for a fresh one.
24
25use std::path::{Path, PathBuf};
26
27use serde::{Deserialize, Serialize};
28use warble_eval_compare::CompareResult;
29
30use crate::context::hash_str;
31use crate::{score_value, Backend, GoldenCase};
32
33/// Sentinel `model` component for the ablation / frontmatter path, where there is no whole-run
34/// `--model` override (the per-step tier binding is baked into the agent, so `agent_sha` already
35/// distinguishes the points). Keeps the key total for every run.
36pub(crate) const FRONTMATTER_MODEL: &str = "frontmatter";
37
38/// One captured case result. The `result` is stored as the raw `{columns, rows}` JSON value (the
39/// same object `capture` embeds) so no `Serialize` is needed on the compare crate's `Table`; it is
40/// deserialized back into a `Table` at re-score time.
41///
42/// `sql_executed` and `tool_calls` are schema slots the P4 trace shape reserves, but the single-shot
43/// `claude -p --output-format json` envelope does not expose either (only `num_turns`), so they are
44/// captured as `None` today. `turns` (from `num_turns`) is the delivered per-case diagnostic; full
45/// SQL / tool-call granularity would need `--output-format stream-json` (a later change).
46#[derive(Debug, Clone, Serialize, Deserialize)]
47pub struct Trace {
48 pub case_id: String,
49 pub agent_sha: String,
50 pub model: String,
51 /// The golden's symbolic `context_version` pin (informational), e.g. `driftwood@a1b2c3d`.
52 pub context_version: Option<String>,
53 /// The computed MDL SHA of the bound project — the `context_sha` key component.
54 pub context_sha: String,
55 /// The question the result answers. Part of the key: if a case's question changes under the
56 /// same id, the cached result is for a different prompt and must be re-run (a miss).
57 pub question: String,
58 /// The SQL the agent executed. Not exposed by the `--output-format json` result envelope; `None`.
59 pub sql_executed: Option<String>,
60 /// The captured `{columns, rows}` result object.
61 pub result: serde_json::Value,
62 /// `None` when the backend that produced this trace couldn't report cost — absent is not zero.
63 #[serde(default)]
64 pub cost: Option<f64>,
65 pub latency_ms: u64,
66 /// Conversation turns (`num_turns`) — the round-trip count ("4–8 round-trips" per case, typically).
67 /// `None` when the backend couldn't report a turn count.
68 #[serde(default)]
69 pub turns: Option<u64>,
70 /// Tool-call count. Not exposed by the JSON result envelope; `None` (see struct docs).
71 pub tool_calls: Option<u64>,
72 /// Which back-end/runtime produced this trace — part of the cache key (see [`CaseKey::backend`])
73 /// so a trace from one back-end is never replayed as another's result. `#[serde(default)]` so a
74 /// pre-existing on-disk trace (no field at all) deserializes to [`Backend::default`] — the
75 /// back-end that behavior always meant before this dimension existed.
76 #[serde(default)]
77 pub backend: Backend,
78}
79
80/// The inputs that determine a case's result. Everything here is in the cache key; the golden's
81/// expectation (expected/tolerance/match) deliberately is **not**, which is what makes re-scoring a
82/// changed expectation a cache hit.
83pub struct CaseKey<'a> {
84 pub case_id: &'a str,
85 pub question: &'a str,
86 pub agent_sha: &'a str,
87 /// The whole-run `--model` binding, or `FRONTMATTER_MODEL` on the ablation path.
88 pub model: &'a str,
89 pub context_sha: &'a str,
90 /// Which repeated-sample run this is (0-indexed). Distinct samples of the same case are
91 /// distinct cache entries — the pass-rate methodology depends on each sample actually invoking
92 /// the agent (or replaying its OWN prior trace), not silently collapsing onto sample 0.
93 pub sample: u32,
94 /// Which back-end/runtime this run is measuring. Part of the key so runs on different
95 /// back-ends can never hit each other's cache.
96 pub backend: Backend,
97 /// The `--max-turns` cap this run was invoked with, or `None` when the back-end's own default
98 /// applied. Part of the key for the same reason `backend` is: a run capped to 1 turn and a run
99 /// left at the default turn budget are different experiments and must never share a cache
100 /// entry. `None` keeps the pre-existing (no-such-knob) key shape untouched — see `canonical()`
101 /// below, which appends this field to the key array only when it is `Some`.
102 pub max_turns: Option<u32>,
103 /// A canonical, self-describing rendering of a differentiated `--strong`/`--cheap`/
104 /// `--orchestrator` (or `--models-config`) binding (e.g.
105 /// `"strong=sonnet,cheap=haiku,orchestrator=sonnet"`), or `None` on every pre-existing path —
106 /// the single-model `--models` sweep and the ablation/frontmatter path both leave this unset,
107 /// since `model` alone already determines what ran there. Part of the key — any parameter
108 /// that shapes how a run executes belongs in the cache key — for the same reason
109 /// `backend`/`max_turns` are: a differentiated binding is a different experiment from any
110 /// single-model run and must never share a cache entry with one, even when `model` happens to
111 /// equal one of the three tier values. `None` keeps the pre-existing key shape untouched — see
112 /// `canonical()` below, which appends this field to the key array only when it is `Some`.
113 pub tier_binding: Option<&'a str>,
114}
115
116impl CaseKey<'_> {
117 /// The canonical, order-fixed key string that gets hashed. Encoded as a fixed-order JSON array
118 /// so a field value can never spoof the delimiter structure — a `question` may legitimately
119 /// contain newlines or `=`, which a plain `k=v\n` join could use to collide with a different
120 /// logical key (relevant once goldens are machine-generated, not just hand-authored). `sample`
121 /// is stringified and placed among the first five so they stay stable for anyone comparing
122 /// keys across a `samples`-unaware era.
123 ///
124 /// `backend` is appended as a **sixth** element only when it isn't [`Backend::default`],
125 /// `max_turns` is appended after that only when it is `Some`, and `tier_binding` is appended
126 /// after `max_turns` only when it is `Some` — so a run that names none of the three still
127 /// produces the exact legacy 6-element array (byte-identical, same hash), keeping pre-existing
128 /// cache entries for the default back-end at the default turn budget and no tier binding valid,
129 /// while setting any of the three dimensions grows the array and is guaranteed not to collide
130 /// with a key that left it unset for the same case.
131 fn canonical(&self) -> String {
132 let sample = self.sample.to_string();
133 let max_turns = self.max_turns.map(|n| n.to_string());
134 let mut parts: Vec<&str> = vec![
135 self.case_id,
136 self.question,
137 self.agent_sha,
138 self.model,
139 self.context_sha,
140 &sample,
141 ];
142 if self.backend != Backend::default() {
143 parts.push(self.backend.as_str());
144 }
145 if let Some(ref max_turns) = max_turns {
146 parts.push(max_turns);
147 }
148 if let Some(tier_binding) = self.tier_binding {
149 parts.push(tier_binding);
150 }
151 serde_json::to_string(&parts).expect("array of strings serializes")
152 }
153
154 /// Content-addressed hash of the key → the cache entry's filename stem.
155 pub fn hash(&self) -> Result<String, String> {
156 hash_str(&self.canonical())
157 }
158}
159
160/// A content-addressed store of [`Trace`]s under one directory (`<dir>/<key-hash>.json`).
161///
162/// When `enabled` is false (the `--no-cache` escape hatch) [`load`](Self::load) always misses and
163/// [`store`](Self::store) is a no-op, so a run behaves exactly as it did before P4.
164pub struct TraceStore {
165 dir: PathBuf,
166 enabled: bool,
167}
168
169impl TraceStore {
170 /// A store rooted at `dir`. `enabled = false` disables both read and write (`--no-cache`).
171 pub fn new(dir: PathBuf, enabled: bool) -> Self {
172 TraceStore { dir, enabled }
173 }
174
175 /// An always-missing store (used when the cache key material can't be computed, so the run
176 /// still proceeds — every case just re-runs and nothing is cached).
177 pub fn disabled() -> Self {
178 TraceStore {
179 dir: PathBuf::new(),
180 enabled: false,
181 }
182 }
183
184 pub fn is_enabled(&self) -> bool {
185 self.enabled
186 }
187
188 pub fn dir(&self) -> &Path {
189 &self.dir
190 }
191
192 fn entry_path(&self, key_hash: &str) -> PathBuf {
193 self.dir.join(format!("{key_hash}.json"))
194 }
195
196 /// Load a cached trace for `key`, or `None` on a disabled store, a missing entry, or an
197 /// unreadable/corrupt one (a corrupt entry is treated as a miss — the case just re-runs).
198 pub fn load(&self, key: &CaseKey) -> Option<Trace> {
199 if !self.enabled {
200 return None;
201 }
202 let hash = key.hash().ok()?;
203 let raw = std::fs::read_to_string(self.entry_path(&hash)).ok()?;
204 serde_json::from_str(&raw).ok()
205 }
206
207 /// Write `trace` under `key`. A no-op on a disabled store. Creates the cache dir on first write.
208 pub fn store(&self, key: &CaseKey, trace: &Trace) -> Result<(), String> {
209 if !self.enabled {
210 return Ok(());
211 }
212 let hash = key.hash()?;
213 std::fs::create_dir_all(&self.dir)
214 .map_err(|e| format!("create cache dir {}: {e}", self.dir.display()))?;
215 let json = serde_json::to_string_pretty(trace).map_err(|e| e.to_string())?;
216 std::fs::write(self.entry_path(&hash), json)
217 .map_err(|e| format!("write trace {}: {e}", self.entry_path(&hash).display()))
218 }
219}
220
221/// Re-score a cached `trace` against a golden case's **current** expectation — the 0-LLM path.
222/// Dispatches on the case's [`crate::ResultKind`] via [`score_value`] just like a fresh run does, so
223/// a Table case's cached `{columns,rows}` and a Verdict case's cached envelope both re-project the
224/// same way. Returns `None` if the cached result doesn't have the shape the case's kind expects (a
225/// corrupt entry, or a case whose `result_kind` changed since the trace was captured; the caller then
226/// re-runs the case). This is what makes "only the golden's expected changed" cost zero LLM calls:
227/// the result is reused, only the comparison is redone.
228pub fn rescore(trace: &Trace, case: &GoldenCase) -> Option<CompareResult> {
229 score_value(&trace.result, case)
230}
231
232#[cfg(test)]
233mod tests {
234 use super::*;
235 use warble_eval_compare::{MatchMode, Tolerance};
236
237 fn trace_with(result: serde_json::Value) -> Trace {
238 Trace {
239 case_id: "q1".into(),
240 agent_sha: "AAAA".into(),
241 model: "opus".into(),
242 context_version: Some("driftwood@c0ffee".into()),
243 context_sha: "CCCC".into(),
244 question: "how many orders?".into(),
245 sql_executed: None,
246 result,
247 cost: Some(0.12),
248 latency_ms: 20_000,
249 turns: Some(5),
250 tool_calls: None,
251 backend: Backend::default(),
252 }
253 }
254
255 fn case_expecting(rows: serde_json::Value) -> GoldenCase {
256 // Build a GoldenCase via YAML so we don't depend on field visibility.
257 let yaml = format!(
258 "id: q1\nquestion: \"how many orders?\"\nmatch: scalar\nexpected: {{ columns: [n], rows: {rows} }}\n"
259 );
260 serde_yaml::from_str(&yaml).expect("golden case parses")
261 }
262
263 #[test]
264 fn key_hash_is_stable_and_expectation_independent() {
265 let k1 = CaseKey {
266 case_id: "q1",
267 question: "how many orders?",
268 agent_sha: "AAAA",
269 model: "opus",
270 context_sha: "CCCC",
271 sample: 0,
272 backend: Backend::default(),
273 max_turns: None,
274 tier_binding: None,
275 };
276 // Same inputs → same hash (content-addressed, deterministic).
277 assert_eq!(k1.hash().unwrap(), k1.hash().unwrap());
278 assert_eq!(k1.hash().unwrap().len(), 40, "git blob sha is 40 hex");
279 }
280
281 #[test]
282 fn changing_a_key_component_changes_the_hash() {
283 let base = CaseKey {
284 case_id: "q1",
285 question: "how many orders?",
286 agent_sha: "AAAA",
287 model: "opus",
288 context_sha: "CCCC",
289 sample: 0,
290 backend: Backend::default(),
291 max_turns: None,
292 tier_binding: None,
293 };
294 let base_hash = base.hash().unwrap();
295 // Each of the six key components moves the hash (→ a miss → a re-run): a non-default
296 // backend must never hit a default-backend key's cache entry.
297 for changed in [
298 CaseKey {
299 question: "how many customers?",
300 ..key_copy(&base)
301 },
302 CaseKey {
303 agent_sha: "BBBB",
304 ..key_copy(&base)
305 },
306 CaseKey {
307 model: "haiku",
308 ..key_copy(&base)
309 },
310 CaseKey {
311 context_sha: "DDDD",
312 ..key_copy(&base)
313 },
314 CaseKey {
315 sample: 1,
316 ..key_copy(&base)
317 },
318 CaseKey {
319 backend: Backend::ClaudeAgentSdk,
320 ..key_copy(&base)
321 },
322 CaseKey {
323 max_turns: Some(1),
324 ..key_copy(&base)
325 },
326 ] {
327 assert_ne!(base_hash, changed.hash().unwrap());
328 }
329 }
330
331 // Helper to copy a CaseKey's fields (CaseKey holds &str, so this is cheap).
332 fn key_copy<'a>(k: &CaseKey<'a>) -> CaseKey<'a> {
333 CaseKey {
334 case_id: k.case_id,
335 question: k.question,
336 agent_sha: k.agent_sha,
337 model: k.model,
338 context_sha: k.context_sha,
339 sample: k.sample,
340 backend: k.backend,
341 max_turns: k.max_turns,
342 tier_binding: k.tier_binding,
343 }
344 }
345
346 #[test]
347 fn store_round_trips_and_no_cache_never_hits() {
348 let dir = tempfile::tempdir().unwrap();
349 let key = CaseKey {
350 case_id: "q1",
351 question: "how many orders?",
352 agent_sha: "AAAA",
353 model: "opus",
354 context_sha: "CCCC",
355 sample: 0,
356 backend: Backend::default(),
357 max_turns: None,
358 tier_binding: None,
359 };
360 let trace = trace_with(serde_json::json!({"columns":["n"],"rows":[[42]]}));
361
362 let enabled = TraceStore::new(dir.path().to_path_buf(), true);
363 enabled.store(&key, &trace).unwrap();
364 let loaded = enabled.load(&key).expect("hit");
365 assert_eq!(loaded.result, trace.result);
366 assert_eq!(loaded.turns, Some(5));
367
368 // A disabled store (the --no-cache path) never hits, even with the entry on disk.
369 let disabled = TraceStore::new(dir.path().to_path_buf(), false);
370 assert!(disabled.load(&key).is_none());
371 // …and writing is a silent no-op.
372 disabled.store(&key, &trace).unwrap();
373 }
374
375 #[test]
376 fn rescore_reuses_result_against_the_current_expectation() {
377 let trace = trace_with(serde_json::json!({"columns":["n"],"rows":[[42]]}));
378
379 // Golden still expects 42 → pass, using only the cached result (no LLM).
380 let pass = rescore(&trace, &case_expecting(serde_json::json!([[42]]))).unwrap();
381 assert!(pass.pass, "cached 42 matches expected 42");
382
383 // The golden's expected changes to 99 → the SAME cached result now fails the re-score.
384 // This is the "only the expectation changed" path: 0 LLM, correct new verdict.
385 let fail = rescore(&trace, &case_expecting(serde_json::json!([[99]]))).unwrap();
386 assert!(!fail.pass, "cached 42 no longer matches expected 99");
387
388 let _ = Tolerance::default();
389 let _ = MatchMode::Scalar;
390 }
391
392 #[test]
393 fn canonical_key_is_a_six_element_array_for_the_default_backend() {
394 // The default backend's canonical key stays byte-identical to the pre-Backend-dimension
395 // shape — this is what keeps legacy cache entries valid for the default target.
396 let key = CaseKey {
397 case_id: "q1",
398 question: "how many orders?",
399 agent_sha: "AAAA",
400 model: "opus",
401 context_sha: "CCCC",
402 sample: 2,
403 backend: Backend::default(),
404 max_turns: None,
405 tier_binding: None,
406 };
407 let parsed: Vec<String> = serde_json::from_str(&key.canonical()).unwrap();
408 assert_eq!(
409 parsed,
410 vec!["q1", "how many orders?", "AAAA", "opus", "CCCC", "2"]
411 );
412 }
413
414 #[test]
415 fn canonical_key_is_a_seven_element_array_for_a_non_default_backend() {
416 // A non-default backend gets an extra element — guaranteed not to collide with any
417 // default-backend (legacy) key for the same case, so no cross-backend cache hit is even
418 // representable at the key-encoding level.
419 let key = CaseKey {
420 case_id: "q1",
421 question: "how many orders?",
422 agent_sha: "AAAA",
423 model: "opus",
424 context_sha: "CCCC",
425 sample: 2,
426 backend: Backend::ClaudeAgentSdk,
427 max_turns: None,
428 tier_binding: None,
429 };
430 let parsed: Vec<String> = serde_json::from_str(&key.canonical()).unwrap();
431 assert_eq!(
432 parsed,
433 vec![
434 "q1",
435 "how many orders?",
436 "AAAA",
437 "opus",
438 "CCCC",
439 "2",
440 "claude-agent-sdk"
441 ]
442 );
443 }
444
445 #[test]
446 fn max_turns_none_key_is_byte_identical_to_the_pre_max_turns_key() {
447 // The exact guard the 24 already-paid driftwood traces depend on: a run that never
448 // mentions `--max-turns` (max_turns: None) must still hash to the same key it always did,
449 // for both the default backend (6-element legacy array) and a non-default one (7-element
450 // array) — adding the max_turns dimension must not move either.
451 let default_backend = CaseKey {
452 case_id: "q1",
453 question: "how many orders?",
454 agent_sha: "AAAA",
455 model: "opus",
456 context_sha: "CCCC",
457 sample: 2,
458 backend: Backend::default(),
459 max_turns: None,
460 tier_binding: None,
461 };
462 assert_eq!(
463 default_backend.canonical(),
464 r#"["q1","how many orders?","AAAA","opus","CCCC","2"]"#
465 );
466
467 let sdk_backend = CaseKey {
468 backend: Backend::ClaudeAgentSdk,
469 ..key_copy(&default_backend)
470 };
471 assert_eq!(
472 sdk_backend.canonical(),
473 r#"["q1","how many orders?","AAAA","opus","CCCC","2","claude-agent-sdk"]"#
474 );
475 }
476
477 #[test]
478 fn max_turns_some_key_differs_from_the_none_key() {
479 // The guard against silent-replay: a 1-turn sdk run must never hash to the same key as
480 // the existing (max_turns-less) sdk traces, or the experiment would just replay old data.
481 let none_key = CaseKey {
482 case_id: "q1",
483 question: "how many orders?",
484 agent_sha: "AAAA",
485 model: "opus",
486 context_sha: "CCCC",
487 sample: 0,
488 backend: Backend::ClaudeAgentSdk,
489 max_turns: None,
490 tier_binding: None,
491 };
492 let some_key = CaseKey {
493 max_turns: Some(1),
494 ..key_copy(&none_key)
495 };
496 assert_ne!(none_key.hash().unwrap(), some_key.hash().unwrap());
497 assert_ne!(none_key.canonical(), some_key.canonical());
498 assert_eq!(
499 some_key.canonical(),
500 r#"["q1","how many orders?","AAAA","opus","CCCC","0","claude-agent-sdk","1"]"#
501 );
502 }
503
504 #[test]
505 fn tier_binding_none_key_is_byte_identical_to_the_pre_tier_binding_key() {
506 // The exact guard every already-paid trace (single-model or max-turns-capped) depends on:
507 // a run that carries no differentiated tier binding (tier_binding: None) must still hash to
508 // the same key it always did, at every prior array length (6, 7, and 8 elements) — adding
509 // the tier_binding dimension must not move any of them.
510 let default_backend = CaseKey {
511 case_id: "q1",
512 question: "how many orders?",
513 agent_sha: "AAAA",
514 model: "opus",
515 context_sha: "CCCC",
516 sample: 2,
517 backend: Backend::default(),
518 max_turns: None,
519 tier_binding: None,
520 };
521 assert_eq!(
522 default_backend.canonical(),
523 r#"["q1","how many orders?","AAAA","opus","CCCC","2"]"#
524 );
525
526 let sdk_backend = CaseKey {
527 backend: Backend::ClaudeAgentSdk,
528 ..key_copy(&default_backend)
529 };
530 assert_eq!(
531 sdk_backend.canonical(),
532 r#"["q1","how many orders?","AAAA","opus","CCCC","2","claude-agent-sdk"]"#
533 );
534
535 let sdk_backend_capped = CaseKey {
536 max_turns: Some(1),
537 ..key_copy(&sdk_backend)
538 };
539 assert_eq!(
540 sdk_backend_capped.canonical(),
541 r#"["q1","how many orders?","AAAA","opus","CCCC","2","claude-agent-sdk","1"]"#
542 );
543 }
544
545 #[test]
546 fn tier_binding_some_key_differs_and_cannot_collide_with_any_single_model_key() {
547 // A differentiated --strong/--cheap/--orchestrator binding is a different
548 // experiment from any single-model run and must never share a cache entry with one, even
549 // when `model` happens to equal one of the three tier values (here "sonnet" is also the
550 // synthesized binding's `strong` value). The extra element makes collision unrepresentable
551 // at the key-encoding level, exactly like the backend/max_turns dimensions before it.
552 let flat_key = CaseKey {
553 case_id: "q1",
554 question: "how many orders?",
555 agent_sha: "AAAA",
556 model: "sonnet",
557 context_sha: "CCCC",
558 sample: 0,
559 backend: Backend::ClaudeAgentSdk,
560 max_turns: None,
561 tier_binding: None,
562 };
563 let tiered_key = CaseKey {
564 tier_binding: Some("strong=sonnet,cheap=haiku,orchestrator=sonnet"),
565 ..key_copy(&flat_key)
566 };
567 assert_ne!(flat_key.hash().unwrap(), tiered_key.hash().unwrap());
568 assert_ne!(flat_key.canonical(), tiered_key.canonical());
569 assert_eq!(
570 tiered_key.canonical(),
571 r#"["q1","how many orders?","AAAA","sonnet","CCCC","0","claude-agent-sdk","strong=sonnet,cheap=haiku,orchestrator=sonnet"]"#
572 );
573 }
574}