use std::collections::HashMap;
use std::sync::Mutex;
use std::time::{Duration, Instant};
use firstpass_core::{ServedFrom, Trace, Verdict};
use crate::provider::ModelResponse;
#[derive(Debug, Clone)]
pub struct Entry {
pub response: ModelResponse,
pub source: uuid::Uuid,
pub original_cost_usd: f64,
stored: Instant,
}
fn key(tenant: &str, prompt_hash: &str, ladder: &[String]) -> String {
format!("{tenant}|{prompt_hash}|{}", ladder.join(">"))
}
#[derive(Debug)]
pub struct VerifiedCache {
entries: Mutex<HashMap<String, Entry>>,
ttl: Duration,
max_entries: usize,
}
impl VerifiedCache {
#[must_use]
pub fn new(ttl: Duration, max_entries: usize) -> Self {
Self {
entries: Mutex::new(HashMap::new()),
ttl,
max_entries,
}
}
#[must_use]
pub fn get(
&self,
tenant: &str,
prompt_hash: &str,
ladder: &[String],
now: Instant,
) -> Option<Entry> {
let Ok(map) = self.entries.lock() else {
return None;
};
map.get(&key(tenant, prompt_hash, ladder))
.filter(|e| now.duration_since(e.stored) < self.ttl)
.cloned()
}
pub fn offer(
&self,
trace: &Trace,
response: &ModelResponse,
ladder: &[String],
now: Instant,
) -> bool {
if trace.final_.served_from != ServedFrom::Attempt {
return false;
}
if trace.final_.cache_source.is_some() {
return false;
}
let Some(served) = trace.final_.served_rung else {
return false;
};
let passed = trace
.attempts
.iter()
.any(|a| a.rung == served && a.verdict == Verdict::Pass);
if !passed {
return false;
}
if trace.deferred.iter().any(|d| d.verdict == Verdict::Fail) {
return false;
}
let Ok(mut map) = self.entries.lock() else {
return false;
};
if map.len() >= self.max_entries {
map.retain(|_, e| now.duration_since(e.stored) < self.ttl);
if map.len() >= self.max_entries {
let mut by_age: Vec<(String, Instant)> =
map.iter().map(|(k, e)| (k.clone(), e.stored)).collect();
by_age.sort_by_key(|(_, t)| *t);
for (k, _) in by_age.into_iter().take(self.max_entries.max(2) / 2) {
map.remove(&k);
}
}
}
map.insert(
key(&trace.tenant_id, &trace.request.prompt_hash, ladder),
Entry {
response: response.clone(),
source: trace.trace_id,
original_cost_usd: trace.final_.total_cost_usd,
stored: now,
},
);
true
}
pub fn retract(&self, source: uuid::Uuid) -> usize {
let Ok(mut map) = self.entries.lock() else {
return 0;
};
let before = map.len();
map.retain(|_, e| e.source != source);
before - map.len()
}
#[must_use]
pub fn len(&self) -> usize {
self.entries.lock().map(|m| m.len()).unwrap_or(0)
}
#[must_use]
pub fn is_empty(&self) -> bool {
self.len() == 0
}
}
#[cfg(test)]
mod tests {
use super::*;
use firstpass_core::{
Attempt, DeferredVerdict, Features, FinalOutcome, GENESIS_HASH, GateResult, Mode,
PolicyRef, RequestInfo, TaskKind,
};
fn resp(text: &str) -> ModelResponse {
ModelResponse {
model: "anthropic/claude-haiku-4-5".to_owned(),
text: text.to_owned(),
in_tokens: 10,
cache_write_tokens: 0,
cache_read_tokens: 0,
out_tokens: 5,
raw: serde_json::Value::Null,
}
}
fn ladder() -> Vec<String> {
vec!["anthropic/claude-haiku-4-5".to_owned()]
}
fn trace(served_from: ServedFrom, verdict: Verdict) -> Trace {
Trace {
trace_id: uuid::Uuid::now_v7(),
prev_hash: GENESIS_HASH.to_owned(),
tenant_id: "t".to_owned(),
session_id: "s".to_owned(),
ts: jiff::Timestamp::UNIX_EPOCH,
mode: Mode::Enforce,
policy: PolicyRef {
id: "static-ladder@v0".to_owned(),
explore: false,
propensity: None,
mode_profile: None,
},
request: RequestInfo {
api: "anthropic.messages".to_owned(),
prompt_hash: "hash-abc".to_owned(),
features: Features::new(TaskKind::CodeEdit),
},
attempts: vec![Attempt {
rung: 0,
model: "anthropic/claude-haiku-4-5".to_owned(),
provider: "anthropic".to_owned(),
in_tokens: 10,
cache_write_tokens: 0,
cache_read_tokens: 0,
out_tokens: 5,
cost_usd: 0.01,
latency_ms: 100,
gates: vec![GateResult::deterministic("non-empty", verdict, 1)],
verdict,
}],
final_: FinalOutcome {
served_rung: Some(0),
served_from,
total_cost_usd: 0.01,
gate_cost_usd: 0.0,
total_latency_ms: 120,
escalations: 0,
counterfactual_baseline_usd: 0.05,
savings_usd: 0.04,
cache_source: None,
},
deferred: vec![],
predicted_pass: None,
probe: None,
elastic: None,
rollout: None,
shadow: None,
route_ix: None,
}
}
#[test]
fn a_passing_answer_is_stored_and_replayed_with_its_proof() {
let c = VerifiedCache::new(Duration::from_secs(60), 100);
let now = Instant::now();
let t = trace(ServedFrom::Attempt, Verdict::Pass);
assert!(c.offer(&t, &resp("the answer"), &ladder(), now));
let hit = c.get("t", "hash-abc", &ladder(), now).expect("hit");
assert_eq!(hit.response.text, "the answer");
assert_eq!(hit.source, t.trace_id);
assert!((hit.original_cost_usd - 0.01).abs() < f64::EPSILON);
}
#[test]
fn a_failed_answer_is_never_cached() {
let c = VerifiedCache::new(Duration::from_secs(60), 100);
let now = Instant::now();
assert!(!c.offer(
&trace(ServedFrom::Attempt, Verdict::Fail),
&resp("bad"),
&ladder(),
now
));
assert!(
c.get("t", "hash-abc", &ladder(), now).is_none(),
"must not be replayable"
);
}
#[test]
fn an_abstain_is_never_cached() {
let c = VerifiedCache::new(Duration::from_secs(60), 100);
let now = Instant::now();
assert!(!c.offer(
&trace(ServedFrom::Attempt, Verdict::Abstain),
&resp("?"),
&ladder(),
now
));
assert!(c.get("t", "hash-abc", &ladder(), now).is_none());
}
#[test]
fn a_budget_exhausted_best_effort_serve_is_never_cached() {
let c = VerifiedCache::new(Duration::from_secs(60), 100);
let now = Instant::now();
assert!(!c.offer(
&trace(ServedFrom::BestAttempt, Verdict::Pass),
&resp("x"),
&ladder(),
now
));
assert!(c.get("t", "hash-abc", &ladder(), now).is_none());
}
#[test]
fn a_later_failing_verdict_retracts_the_proof() {
let c = VerifiedCache::new(Duration::from_secs(60), 100);
let now = Instant::now();
let mut t = trace(ServedFrom::Attempt, Verdict::Pass);
t.deferred.push(DeferredVerdict {
gate_id: "tests".to_owned(),
verdict: Verdict::Fail,
score: None,
reported_at: jiff::Timestamp::UNIX_EPOCH,
reporter: "ci".to_owned(),
});
assert!(!c.offer(&t, &resp("looked fine"), &ladder(), now));
}
#[test]
fn a_retraction_evicts_the_answer_it_disproved() {
let c = VerifiedCache::new(Duration::from_secs(600), 100);
let now = Instant::now();
let t = trace(ServedFrom::Attempt, Verdict::Pass);
assert!(c.offer(&t, &resp("looked fine"), &ladder(), now));
assert!(c.get("t", "hash-abc", &ladder(), now).is_some());
assert_eq!(c.retract(t.trace_id), 1);
assert!(
c.get("t", "hash-abc", &ladder(), now).is_none(),
"an answer whose proof was retracted must stop being replayed immediately"
);
}
#[test]
fn retracting_an_unrelated_decision_evicts_nothing() {
let c = VerifiedCache::new(Duration::from_secs(600), 100);
let now = Instant::now();
c.offer(
&trace(ServedFrom::Attempt, Verdict::Pass),
&resp("a"),
&ladder(),
now,
);
assert_eq!(c.retract(uuid::Uuid::now_v7()), 0);
assert!(c.get("t", "hash-abc", &ladder(), now).is_some());
}
#[test]
fn a_cache_hit_is_not_itself_re_cached() {
let c = VerifiedCache::new(Duration::from_secs(60), 100);
let now = Instant::now();
let mut t = trace(ServedFrom::Attempt, Verdict::Pass);
t.final_.cache_source = Some(uuid::Uuid::now_v7());
assert!(!c.offer(&t, &resp("replayed"), &ladder(), now));
}
#[test]
fn one_tenant_can_never_replay_anothers_answer() {
let c = VerifiedCache::new(Duration::from_secs(60), 100);
let now = Instant::now();
let mut t = trace(ServedFrom::Attempt, Verdict::Pass);
t.tenant_id = "tenant-a".to_owned();
assert!(c.offer(&t, &resp("a's private answer"), &ladder(), now));
assert!(
c.get("tenant-b", "hash-abc", &ladder(), now).is_none(),
"cross-tenant replay — this is a data leak, not a cache miss"
);
assert!(
c.get("tenant-a", "hash-abc", &ladder(), now).is_some(),
"the owning tenant must still hit"
);
}
#[test]
fn the_cached_tenant_comes_from_the_receipt_not_the_caller() {
let c = VerifiedCache::new(Duration::from_secs(60), 100);
let now = Instant::now();
let mut t = trace(ServedFrom::Attempt, Verdict::Pass);
t.tenant_id = "owner".to_owned();
c.offer(&t, &resp("x"), &ladder(), now);
assert!(c.get("owner", "hash-abc", &ladder(), now).is_some());
assert!(c.get("t", "hash-abc", &ladder(), now).is_none());
}
#[test]
fn a_different_ladder_is_a_different_question() {
let c = VerifiedCache::new(Duration::from_secs(60), 100);
let now = Instant::now();
c.offer(
&trace(ServedFrom::Attempt, Verdict::Pass),
&resp("a"),
&ladder(),
now,
);
let other = vec!["openai/gpt-5.5".to_owned()];
assert!(c.get("t", "hash-abc", &other, now).is_none());
assert!(c.get("t", "hash-abc", &ladder(), now).is_some());
}
#[test]
fn an_entry_expires() {
let c = VerifiedCache::new(Duration::from_secs(1), 100);
let t0 = Instant::now();
c.offer(
&trace(ServedFrom::Attempt, Verdict::Pass),
&resp("a"),
&ladder(),
t0,
);
assert!(c.get("t", "hash-abc", &ladder(), t0).is_some());
assert!(
c.get("t", "hash-abc", &ladder(), t0 + Duration::from_secs(2))
.is_none(),
"a stale proof is not a proof"
);
}
#[test]
fn the_cache_cannot_grow_without_bound() {
let c = VerifiedCache::new(Duration::from_secs(600), 10);
let now = Instant::now();
for i in 0..100 {
let mut t = trace(ServedFrom::Attempt, Verdict::Pass);
t.request.prompt_hash = format!("h{i}");
c.offer(&t, &resp("a"), &ladder(), now);
}
assert!(c.len() <= 10, "expected <= 10 entries, got {}", c.len());
}
}