Skip to main content

firstpass_proxy/
verified_cache.rs

1//! A response cache that can only ever return an answer which **passed a gate**.
2//!
3//! ## Why this is not the same feature everyone else ships
4//!
5//! A conventional semantic or exact cache stores whatever the model said and replays it. It saves
6//! money by skipping the model — and skipping the model is exactly how a gateway stops knowing
7//! whether the thing it just served was any good. On a cache hit rate of 40%, a product whose claim
8//! is "nothing ships until a real check passes" would be shipping 40% of its traffic unchecked.
9//!
10//! So insertion is the gate here, not lookup: an entry is written **only** when the request it came
11//! from was served on a passing verdict. A failed answer, a best-effort serve after budget
12//! exhaustion, an abstain — none of them are cacheable, because replaying one would launder an
13//! unverified answer into a served one and no later reader could tell.
14//!
15//! The receipt records `served_from: cache` and `cache_source: <trace_id>`, so a hit is not the
16//! word "cached" but a link to the decision whose gate passed. That is the difference between
17//! asserting a saving and being able to prove what was served.
18//!
19//! ## What it is keyed on
20//!
21//! The salted prompt hash the router already computes, plus the ladder identity. Exact-match, not
22//! semantic: two prompts that merely *look* similar can want different answers, and a verification
23//! product guessing that they don't is the one shortcut it cannot take. Semantic matching would
24//! need its own gate over the similarity decision, which is a bigger design than a cache.
25//!
26//! The ladder is part of the key because the same prompt under a different ladder is a different
27//! question — an answer proven against one gate configuration says nothing about another.
28
29use std::collections::HashMap;
30use std::sync::Mutex;
31use std::time::{Duration, Instant};
32
33use firstpass_core::{ServedFrom, Trace, Verdict};
34
35use crate::provider::ModelResponse;
36
37/// An answer that passed, kept for replay.
38#[derive(Debug, Clone)]
39pub struct Entry {
40    /// The response that was served.
41    ///
42    /// Stored as the normalized [`ModelResponse`] rather than raw wire bytes, so a replay flows
43    /// back out through the same dialect-shaping the live path uses. Caching bytes would mean an
44    /// answer cached from an Anthropic caller could not be replayed to an OpenAI one, and the two
45    /// paths would drift.
46    pub response: ModelResponse,
47    /// The decision that proved it — recorded on every replay's receipt.
48    pub source: uuid::Uuid,
49    /// What that decision cost, so a replay can report the spend it avoided rather than guess.
50    pub original_cost_usd: f64,
51    /// Insert time, for TTL.
52    stored: Instant,
53}
54
55/// Key: tenant, then the salted prompt hash, then the ladder that answered it.
56///
57/// **The tenant is not optional.** The prompt salt is per-deployment, not per-tenant, so two
58/// tenants sending an identical prompt produce an identical hash — a key without the tenant would
59/// serve one customer's paid-for, verified answer to another. That is a cross-tenant data leak in
60/// a multi-tenant proxy, not merely a wrong cache hit, and it is silent: the receipt would look
61/// entirely normal.
62fn key(tenant: &str, prompt_hash: &str, ladder: &[String]) -> String {
63    format!("{tenant}|{prompt_hash}|{}", ladder.join(">"))
64}
65
66/// A bounded, in-process store of verified answers.
67#[derive(Debug)]
68pub struct VerifiedCache {
69    entries: Mutex<HashMap<String, Entry>>,
70    ttl: Duration,
71    max_entries: usize,
72}
73
74impl VerifiedCache {
75    /// Build a cache holding at most `max_entries` for `ttl`.
76    #[must_use]
77    pub fn new(ttl: Duration, max_entries: usize) -> Self {
78        Self {
79            entries: Mutex::new(HashMap::new()),
80            ttl,
81            max_entries,
82        }
83    }
84
85    /// Look up a proven answer for this prompt under this ladder.
86    #[must_use]
87    pub fn get(
88        &self,
89        tenant: &str,
90        prompt_hash: &str,
91        ladder: &[String],
92        now: Instant,
93    ) -> Option<Entry> {
94        let Ok(map) = self.entries.lock() else {
95            // A poisoned lock must not take serving down. Missing the cache is always safe: the
96            // request simply runs the ladder, which is the behaviour with caching off.
97            return None;
98        };
99        map.get(&key(tenant, prompt_hash, ladder))
100            .filter(|e| now.duration_since(e.stored) < self.ttl)
101            .cloned()
102    }
103
104    /// Offer a finished decision to the cache.
105    ///
106    /// Returns whether it was stored. Everything about the safety of this feature is in the
107    /// rejection rules below, so they are checked here — in one place, against the receipt — rather
108    /// than trusted to each call site to have got right.
109    pub fn offer(
110        &self,
111        trace: &Trace,
112        response: &ModelResponse,
113        ladder: &[String],
114        now: Instant,
115    ) -> bool {
116        // 1. It must have been served from a real attempt. `BestAttempt` is the budget-exhausted
117        //    fallback — served without a passing verdict, and caching it would replay an answer
118        //    that never passed anything.
119        if trace.final_.served_from != ServedFrom::Attempt {
120            return false;
121        }
122        // 2. Never re-cache a cache hit. Harmless in itself, but it would relocate `cache_source`
123        //    onto a replay rather than the decision that actually ran a gate, and the provenance
124        //    chain is the feature.
125        if trace.final_.cache_source.is_some() {
126            return false;
127        }
128        // 3. The served rung's verdict must be an unambiguous Pass. An Abstain is "the gate could
129        //    not tell", which is precisely the state that must not be frozen and replayed.
130        let Some(served) = trace.final_.served_rung else {
131            return false;
132        };
133        let passed = trace
134            .attempts
135            .iter()
136            .any(|a| a.rung == served && a.verdict == Verdict::Pass);
137        if !passed {
138            return false;
139        }
140        // 4. A deferred verdict already on the receipt that says Fail retracts the proof.
141        //
142        //    This is rarely the path that matters: deferred verdicts usually arrive minutes later
143        //    via `/v1/feedback`, long after insertion, so this check alone would be close to a
144        //    no-op — a safety rule that never fires, which is worse than none because the comment
145        //    claims protection it does not give. `retract()` below is the half that actually runs
146        //    in production, called from the feedback handler.
147        if trace.deferred.iter().any(|d| d.verdict == Verdict::Fail) {
148            return false;
149        }
150
151        let Ok(mut map) = self.entries.lock() else {
152            return false;
153        };
154        if map.len() >= self.max_entries {
155            map.retain(|_, e| now.duration_since(e.stored) < self.ttl);
156            if map.len() >= self.max_entries {
157                // ponytail: drop the oldest half on the rare over-cap write rather than maintain an
158                // LRU list on every read. A cache is an optimisation; its bookkeeping should not
159                // cost more than the thing it saves.
160                let mut by_age: Vec<(String, Instant)> =
161                    map.iter().map(|(k, e)| (k.clone(), e.stored)).collect();
162                by_age.sort_by_key(|(_, t)| *t);
163                for (k, _) in by_age.into_iter().take(self.max_entries.max(2) / 2) {
164                    map.remove(&k);
165                }
166            }
167        }
168        map.insert(
169            // The tenant comes off the RECEIPT, not from a parameter a caller could pass wrong —
170            // the same reason the pass-only rule is checked against the receipt.
171            key(&trace.tenant_id, &trace.request.prompt_hash, ladder),
172            Entry {
173                response: response.clone(),
174                source: trace.trace_id,
175                original_cost_usd: trace.final_.total_cost_usd,
176                stored: now,
177            },
178        );
179        true
180    }
181
182    /// Evict every entry proven by `source`, because that proof has been retracted.
183    ///
184    /// Called when a deferred gate or downstream outcome reports a Fail for a decision. Without
185    /// this, an answer the world has since disproven keeps being replayed until its TTL expires —
186    /// and each replay carries a `cache_source` pointing at a receipt that now records a failure,
187    /// so the audit trail would actively contradict what was served.
188    ///
189    /// Returns how many entries were dropped. A single decision can back several keys once the
190    /// same prompt is seen under more than one ladder, so this scans by source rather than
191    /// assuming one.
192    pub fn retract(&self, source: uuid::Uuid) -> usize {
193        let Ok(mut map) = self.entries.lock() else {
194            return 0;
195        };
196        let before = map.len();
197        map.retain(|_, e| e.source != source);
198        before - map.len()
199    }
200
201    /// Entries currently held (tests and `/metrics`).
202    #[must_use]
203    pub fn len(&self) -> usize {
204        self.entries.lock().map(|m| m.len()).unwrap_or(0)
205    }
206
207    /// Whether the cache holds nothing.
208    #[must_use]
209    pub fn is_empty(&self) -> bool {
210        self.len() == 0
211    }
212}
213
214#[cfg(test)]
215mod tests {
216    use super::*;
217    use firstpass_core::{
218        Attempt, DeferredVerdict, Features, FinalOutcome, GENESIS_HASH, GateResult, Mode,
219        PolicyRef, RequestInfo, TaskKind,
220    };
221
222    fn resp(text: &str) -> ModelResponse {
223        ModelResponse {
224            model: "anthropic/claude-haiku-4-5".to_owned(),
225            text: text.to_owned(),
226            in_tokens: 10,
227            cache_write_tokens: 0,
228            cache_read_tokens: 0,
229            out_tokens: 5,
230            raw: serde_json::Value::Null,
231        }
232    }
233
234    fn ladder() -> Vec<String> {
235        vec!["anthropic/claude-haiku-4-5".to_owned()]
236    }
237
238    fn trace(served_from: ServedFrom, verdict: Verdict) -> Trace {
239        Trace {
240            trace_id: uuid::Uuid::now_v7(),
241            prev_hash: GENESIS_HASH.to_owned(),
242            tenant_id: "t".to_owned(),
243            session_id: "s".to_owned(),
244            ts: jiff::Timestamp::UNIX_EPOCH,
245            mode: Mode::Enforce,
246            policy: PolicyRef {
247                id: "static-ladder@v0".to_owned(),
248                explore: false,
249                propensity: None,
250                mode_profile: None,
251            },
252            request: RequestInfo {
253                api: "anthropic.messages".to_owned(),
254                prompt_hash: "hash-abc".to_owned(),
255                features: Features::new(TaskKind::CodeEdit),
256            },
257            attempts: vec![Attempt {
258                rung: 0,
259                model: "anthropic/claude-haiku-4-5".to_owned(),
260                provider: "anthropic".to_owned(),
261                in_tokens: 10,
262                cache_write_tokens: 0,
263                cache_read_tokens: 0,
264                out_tokens: 5,
265                cost_usd: 0.01,
266                latency_ms: 100,
267                gates: vec![GateResult::deterministic("non-empty", verdict, 1)],
268                verdict,
269            }],
270            final_: FinalOutcome {
271                served_rung: Some(0),
272                served_from,
273                total_cost_usd: 0.01,
274                gate_cost_usd: 0.0,
275                total_latency_ms: 120,
276                escalations: 0,
277                counterfactual_baseline_usd: 0.05,
278                savings_usd: 0.04,
279                cache_source: None,
280            },
281            deferred: vec![],
282            predicted_pass: None,
283            probe: None,
284            elastic: None,
285            rollout: None,
286            shadow: None,
287            route_ix: None,
288        }
289    }
290
291    #[test]
292    fn a_passing_answer_is_stored_and_replayed_with_its_proof() {
293        let c = VerifiedCache::new(Duration::from_secs(60), 100);
294        let now = Instant::now();
295        let t = trace(ServedFrom::Attempt, Verdict::Pass);
296
297        assert!(c.offer(&t, &resp("the answer"), &ladder(), now));
298
299        let hit = c.get("t", "hash-abc", &ladder(), now).expect("hit");
300        assert_eq!(hit.response.text, "the answer");
301        // Not merely "cached" — the decision whose gate passed, so the proof is one lookup away.
302        assert_eq!(hit.source, t.trace_id);
303        assert!((hit.original_cost_usd - 0.01).abs() < f64::EPSILON);
304    }
305
306    #[test]
307    fn a_failed_answer_is_never_cached() {
308        let c = VerifiedCache::new(Duration::from_secs(60), 100);
309        let now = Instant::now();
310
311        assert!(!c.offer(
312            &trace(ServedFrom::Attempt, Verdict::Fail),
313            &resp("bad"),
314            &ladder(),
315            now
316        ));
317
318        assert!(
319            c.get("t", "hash-abc", &ladder(), now).is_none(),
320            "must not be replayable"
321        );
322    }
323
324    #[test]
325    fn an_abstain_is_never_cached() {
326        // "The gate could not tell" is exactly the state that must not be frozen and replayed —
327        // every later hit would inherit an uncertainty nobody re-examines.
328        let c = VerifiedCache::new(Duration::from_secs(60), 100);
329        let now = Instant::now();
330        assert!(!c.offer(
331            &trace(ServedFrom::Attempt, Verdict::Abstain),
332            &resp("?"),
333            &ladder(),
334            now
335        ));
336        assert!(c.get("t", "hash-abc", &ladder(), now).is_none());
337    }
338
339    #[test]
340    fn a_budget_exhausted_best_effort_serve_is_never_cached() {
341        // BestAttempt is served WITHOUT a passing verdict. Caching it would launder an unverified
342        // answer into a served one, and no later reader could tell the difference.
343        let c = VerifiedCache::new(Duration::from_secs(60), 100);
344        let now = Instant::now();
345        assert!(!c.offer(
346            &trace(ServedFrom::BestAttempt, Verdict::Pass),
347            &resp("x"),
348            &ladder(),
349            now
350        ));
351        assert!(c.get("t", "hash-abc", &ladder(), now).is_none());
352    }
353
354    #[test]
355    fn a_later_failing_verdict_retracts_the_proof() {
356        // A deferred gate (tests in CI, downstream feedback) that comes back Fail means the answer
357        // was not good after all. It must not be sitting in the cache waiting to be served again.
358        let c = VerifiedCache::new(Duration::from_secs(60), 100);
359        let now = Instant::now();
360        let mut t = trace(ServedFrom::Attempt, Verdict::Pass);
361        t.deferred.push(DeferredVerdict {
362            gate_id: "tests".to_owned(),
363            verdict: Verdict::Fail,
364            score: None,
365            reported_at: jiff::Timestamp::UNIX_EPOCH,
366            reporter: "ci".to_owned(),
367        });
368
369        assert!(!c.offer(&t, &resp("looked fine"), &ladder(), now));
370    }
371
372    #[test]
373    fn a_retraction_evicts_the_answer_it_disproved() {
374        // Found in review: rule 4 only inspects the receipt at INSERT time, and deferred verdicts
375        // arrive later via /v1/feedback — so on its own it is a no-op in production and a
376        // disproven answer keeps being served until TTL. This is the half that actually runs.
377        let c = VerifiedCache::new(Duration::from_secs(600), 100);
378        let now = Instant::now();
379        let t = trace(ServedFrom::Attempt, Verdict::Pass);
380        assert!(c.offer(&t, &resp("looked fine"), &ladder(), now));
381        assert!(c.get("t", "hash-abc", &ladder(), now).is_some());
382
383        assert_eq!(c.retract(t.trace_id), 1);
384
385        assert!(
386            c.get("t", "hash-abc", &ladder(), now).is_none(),
387            "an answer whose proof was retracted must stop being replayed immediately"
388        );
389    }
390
391    #[test]
392    fn retracting_an_unrelated_decision_evicts_nothing() {
393        let c = VerifiedCache::new(Duration::from_secs(600), 100);
394        let now = Instant::now();
395        c.offer(
396            &trace(ServedFrom::Attempt, Verdict::Pass),
397            &resp("a"),
398            &ladder(),
399            now,
400        );
401
402        assert_eq!(c.retract(uuid::Uuid::now_v7()), 0);
403        assert!(c.get("t", "hash-abc", &ladder(), now).is_some());
404    }
405
406    #[test]
407    fn a_cache_hit_is_not_itself_re_cached() {
408        // Otherwise `cache_source` would point at a replay instead of the decision that actually
409        // ran a gate, and the provenance chain — the whole feature — would go one hop stale.
410        let c = VerifiedCache::new(Duration::from_secs(60), 100);
411        let now = Instant::now();
412        let mut t = trace(ServedFrom::Attempt, Verdict::Pass);
413        t.final_.cache_source = Some(uuid::Uuid::now_v7());
414        assert!(!c.offer(&t, &resp("replayed"), &ladder(), now));
415    }
416
417    #[test]
418    fn one_tenant_can_never_replay_anothers_answer() {
419        // Caught in review, and the worst bug in this feature: the key was
420        // `prompt_hash | ladder`, with no tenant. The prompt salt is per-DEPLOYMENT, so two
421        // tenants sending an identical prompt hash identically — tenant B would have been served
422        // tenant A's paid-for, verified answer, with a receipt that looked entirely normal.
423        let c = VerifiedCache::new(Duration::from_secs(60), 100);
424        let now = Instant::now();
425        let mut t = trace(ServedFrom::Attempt, Verdict::Pass);
426        t.tenant_id = "tenant-a".to_owned();
427        assert!(c.offer(&t, &resp("a's private answer"), &ladder(), now));
428
429        assert!(
430            c.get("tenant-b", "hash-abc", &ladder(), now).is_none(),
431            "cross-tenant replay — this is a data leak, not a cache miss"
432        );
433        assert!(
434            c.get("tenant-a", "hash-abc", &ladder(), now).is_some(),
435            "the owning tenant must still hit"
436        );
437    }
438
439    #[test]
440    fn the_cached_tenant_comes_from_the_receipt_not_the_caller() {
441        // Reading it off the receipt means a call site cannot pass the wrong tenant and quietly
442        // file an answer under someone else's key.
443        let c = VerifiedCache::new(Duration::from_secs(60), 100);
444        let now = Instant::now();
445        let mut t = trace(ServedFrom::Attempt, Verdict::Pass);
446        t.tenant_id = "owner".to_owned();
447        c.offer(&t, &resp("x"), &ladder(), now);
448
449        assert!(c.get("owner", "hash-abc", &ladder(), now).is_some());
450        assert!(c.get("t", "hash-abc", &ladder(), now).is_none());
451    }
452
453    #[test]
454    fn a_different_ladder_is_a_different_question() {
455        // An answer proven against one gate/ladder configuration says nothing about another.
456        let c = VerifiedCache::new(Duration::from_secs(60), 100);
457        let now = Instant::now();
458        c.offer(
459            &trace(ServedFrom::Attempt, Verdict::Pass),
460            &resp("a"),
461            &ladder(),
462            now,
463        );
464
465        let other = vec!["openai/gpt-5.5".to_owned()];
466        assert!(c.get("t", "hash-abc", &other, now).is_none());
467        assert!(c.get("t", "hash-abc", &ladder(), now).is_some());
468    }
469
470    #[test]
471    fn an_entry_expires() {
472        let c = VerifiedCache::new(Duration::from_secs(1), 100);
473        let t0 = Instant::now();
474        c.offer(
475            &trace(ServedFrom::Attempt, Verdict::Pass),
476            &resp("a"),
477            &ladder(),
478            t0,
479        );
480
481        assert!(c.get("t", "hash-abc", &ladder(), t0).is_some());
482        assert!(
483            c.get("t", "hash-abc", &ladder(), t0 + Duration::from_secs(2))
484                .is_none(),
485            "a stale proof is not a proof"
486        );
487    }
488
489    #[test]
490    fn the_cache_cannot_grow_without_bound() {
491        let c = VerifiedCache::new(Duration::from_secs(600), 10);
492        let now = Instant::now();
493        for i in 0..100 {
494            let mut t = trace(ServedFrom::Attempt, Verdict::Pass);
495            t.request.prompt_hash = format!("h{i}");
496            c.offer(&t, &resp("a"), &ladder(), now);
497        }
498        assert!(c.len() <= 10, "expected <= 10 entries, got {}", c.len());
499    }
500}