Skip to main content

firstpass_proxy/
verified_cache.rs

1//! A response cache that can only ever return an answer which **passed a gate**.
2//!
3//! ## Why this is not the same feature everyone else ships
4//!
5//! A conventional semantic or exact cache stores whatever the model said and replays it. It saves
6//! money by skipping the model — and skipping the model is exactly how a gateway stops knowing
7//! whether the thing it just served was any good. On a cache hit rate of 40%, a product whose claim
8//! is "nothing ships until a real check passes" would be shipping 40% of its traffic unchecked.
9//!
10//! So insertion is the gate here, not lookup: an entry is written **only** when the request it came
11//! from was served on a passing verdict. A failed answer, a best-effort serve after budget
12//! exhaustion, an abstain — none of them are cacheable, because replaying one would launder an
13//! unverified answer into a served one and no later reader could tell.
14//!
15//! The receipt records `served_from: cache` and `cache_source: <trace_id>`, so a hit is not the
16//! word "cached" but a link to the decision whose gate passed. That is the difference between
17//! asserting a saving and being able to prove what was served.
18//!
19//! ## What it is keyed on
20//!
21//! The salted prompt hash the router already computes, plus the ladder identity. Exact-match, not
22//! semantic: two prompts that merely *look* similar can want different answers, and a verification
23//! product guessing that they don't is the one shortcut it cannot take. Semantic matching would
24//! need its own gate over the similarity decision, which is a bigger design than a cache.
25//!
26//! The ladder is part of the key because the same prompt under a different ladder is a different
27//! question — an answer proven against one gate configuration says nothing about another.
28
29use std::collections::HashMap;
30use std::sync::Mutex;
31use std::time::{Duration, Instant};
32
33use firstpass_core::{ServedFrom, Trace, Verdict};
34
35use crate::provider::ModelResponse;
36
37/// An answer that passed, kept for replay.
38#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
39pub struct Entry {
40    /// The response that was served.
41    ///
42    /// Stored as the normalized [`ModelResponse`] rather than raw wire bytes, so a replay flows
43    /// back out through the same dialect-shaping the live path uses. Caching bytes would mean an
44    /// answer cached from an Anthropic caller could not be replayed to an OpenAI one, and the two
45    /// paths would drift.
46    pub response: ModelResponse,
47    /// The decision that proved it — recorded on every replay's receipt.
48    pub source: uuid::Uuid,
49    /// What that decision cost, so a replay can report the spend it avoided rather than guess.
50    pub original_cost_usd: f64,
51    /// Insert time, for the in-process store's TTL.
52    ///
53    /// Skipped on the wire and re-stamped on read: an `Instant` is meaningless in another process,
54    /// and expiry in the shared store is Redis's own TTL rather than a timestamp we compare. Two
55    /// replicas comparing monotonic clocks would disagree about what is stale.
56    #[serde(skip, default = "Instant::now")]
57    stored: Instant,
58}
59
60/// Whether a finished decision may be cached at all.
61///
62/// **The single place these rules live.** They were duplicated nowhere and must stay that way: a
63/// second backend re-deriving "was this proven?" from its own reading of the receipt is how one of
64/// them eventually accepts something the other would reject, and the failure is silent — an
65/// unverified answer served from cache looks exactly like a verified one.
66///
67/// Rejects, in order:
68/// 1. **Not served from a real attempt.** `BestAttempt` is the budget-exhausted fallback, served
69///    *without* a passing verdict; caching it would replay an answer that never passed anything.
70/// 2. **A cache hit.** Harmless in itself, but it would relocate `cache_source` onto a replay
71///    rather than the decision that ran a gate, letting the provenance chain go a hop stale.
72/// 3. **A verdict that is not an unambiguous Pass.** An `Abstain` is "the gate could not tell",
73///    which is precisely the state that must never be frozen and replayed.
74/// 4. **A deferred Fail already on the receipt.** Rarely the path that matters — deferred verdicts
75///    usually arrive later via `/v1/feedback`, which is what `retract` handles — but free to check.
76#[must_use]
77pub fn is_cacheable(trace: &Trace) -> bool {
78    if trace.final_.served_from != ServedFrom::Attempt {
79        return false;
80    }
81    if trace.final_.cache_source.is_some() {
82        return false;
83    }
84    let Some(served) = trace.final_.served_rung else {
85        return false;
86    };
87    if !trace
88        .attempts
89        .iter()
90        .any(|a| a.rung == served && a.verdict == Verdict::Pass)
91    {
92        return false;
93    }
94    !trace.deferred.iter().any(|d| d.verdict == Verdict::Fail)
95}
96
97impl Entry {
98    /// Build an entry from a decision and what it served.
99    ///
100    /// Takes the proof off the trace rather than as a parameter, for the same reason the tenant is
101    /// read off the receipt: a call site cannot then file an answer under the wrong decision.
102    #[must_use]
103    pub fn new(response: ModelResponse, trace: &Trace, now: Instant) -> Self {
104        Self {
105            response,
106            source: trace.trace_id,
107            original_cost_usd: trace.final_.total_cost_usd,
108            stored: now,
109        }
110    }
111}
112
113/// Key: tenant, then the salted prompt hash, then the ladder that answered it.
114///
115/// **The tenant is not optional.** The prompt salt is per-deployment, not per-tenant, so two
116/// tenants sending an identical prompt produce an identical hash — a key without the tenant would
117/// serve one customer's paid-for, verified answer to another. That is a cross-tenant data leak in
118/// a multi-tenant proxy, not merely a wrong cache hit, and it is silent: the receipt would look
119/// entirely normal.
120fn key(tenant: &str, prompt_hash: &str, ladder: &[String]) -> String {
121    format!("{tenant}|{prompt_hash}|{}", ladder.join(">"))
122}
123
124/// Where verified entries live.
125///
126/// Exists so the in-process store stays exactly what it was — the version four review rounds
127/// hardened — while a shared L2 can sit behind it. A single-instance deployment keeps byte-identical
128/// behaviour and no Redis dependency; a multi-replica one gets entries and, critically,
129/// **retractions** that cross process boundaries.
130///
131/// Async because an L2 is a network hop. Blocking one inside an async proxy would stall the runtime
132/// on a cache lookup, which is precisely the wrong place to lose latency.
133#[async_trait::async_trait]
134pub trait CacheStore: Send + Sync + std::fmt::Debug {
135    /// Fetch a live entry, if one exists.
136    async fn get(&self, key: &str, now: Instant) -> Option<Entry>;
137    /// Store an entry under `key`, proven by `source`.
138    ///
139    /// `source` is passed separately because a backend has to index by it: retraction arrives
140    /// naming a decision, not a key, and one decision can back several keys once the same prompt
141    /// is seen under more than one ladder.
142    async fn put(&self, key: &str, source: uuid::Uuid, entry: Entry, now: Instant);
143    /// Drop every entry proven by `source`. Returns how many went.
144    async fn retract(&self, source: uuid::Uuid) -> usize;
145    /// Entries currently held, for tests and `/metrics`.
146    async fn entry_count(&self) -> usize;
147}
148
149/// A bounded, in-process store of verified answers.
150#[derive(Debug)]
151pub struct VerifiedCache {
152    entries: Mutex<HashMap<String, Entry>>,
153    ttl: Duration,
154    max_entries: usize,
155}
156
157impl VerifiedCache {
158    /// Build a cache holding at most `max_entries` for `ttl`.
159    #[must_use]
160    pub fn new(ttl: Duration, max_entries: usize) -> Self {
161        Self {
162            entries: Mutex::new(HashMap::new()),
163            ttl,
164            max_entries,
165        }
166    }
167
168    /// Look up a proven answer for this prompt under this ladder.
169    #[must_use]
170    pub fn get(
171        &self,
172        tenant: &str,
173        prompt_hash: &str,
174        ladder: &[String],
175        now: Instant,
176    ) -> Option<Entry> {
177        let Ok(map) = self.entries.lock() else {
178            // A poisoned lock must not take serving down. Missing the cache is always safe: the
179            // request simply runs the ladder, which is the behaviour with caching off.
180            return None;
181        };
182        map.get(&key(tenant, prompt_hash, ladder))
183            .filter(|e| now.duration_since(e.stored) < self.ttl)
184            .cloned()
185    }
186
187    /// Offer a finished decision to the cache.
188    ///
189    /// Returns whether it was stored. Everything about the safety of this feature is in the
190    /// rejection rules below, so they are checked here — in one place, against the receipt — rather
191    /// than trusted to each call site to have got right.
192    pub fn offer(
193        &self,
194        trace: &Trace,
195        response: &ModelResponse,
196        ladder: &[String],
197        now: Instant,
198    ) -> bool {
199        if !is_cacheable(trace) {
200            return false;
201        }
202        let mut entry = Entry {
203            response: response.clone(),
204            source: trace.trace_id,
205            original_cost_usd: trace.final_.total_cost_usd,
206            stored: now,
207        };
208        entry.stored = now;
209        self.put_by_key(
210            &key(&trace.tenant_id, &trace.request.prompt_hash, ladder),
211            entry,
212        );
213        true
214    }
215
216    /// Fetch by a pre-built key. The key-level primitive behind [`CacheStore`]; the
217    /// tenant/prompt/ladder composition lives in [`key`] so both stores agree on it.
218    #[must_use]
219    pub fn get_by_key(&self, key: &str, now: Instant) -> Option<Entry> {
220        let Ok(map) = self.entries.lock() else {
221            return None;
222        };
223        map.get(key)
224            .filter(|e| now.duration_since(e.stored) < self.ttl)
225            .cloned()
226    }
227
228    /// Store by a pre-built key, applying the same bound as `offer`.
229    pub fn put_by_key(&self, key: &str, entry: Entry) {
230        let Ok(mut map) = self.entries.lock() else {
231            return;
232        };
233        let now = entry.stored;
234        if map.len() >= self.max_entries {
235            map.retain(|_, e| now.duration_since(e.stored) < self.ttl);
236            if map.len() >= self.max_entries {
237                let mut by_age: Vec<(String, Instant)> =
238                    map.iter().map(|(k, e)| (k.clone(), e.stored)).collect();
239                by_age.sort_by_key(|(_, t)| *t);
240                for (k, _) in by_age.into_iter().take(self.max_entries.max(2) / 2) {
241                    map.remove(&k);
242                }
243            }
244        }
245        map.insert(key.to_owned(), entry);
246    }
247
248    /// Compose the key both stores use, so an entry written by one is found by the other.
249    #[must_use]
250    pub fn compose_key(tenant: &str, prompt_hash: &str, ladder: &[String]) -> String {
251        key(tenant, prompt_hash, ladder)
252    }
253
254    /// Evict every entry proven by `source`, because that proof has been retracted.
255    ///
256    /// Called when a deferred gate or downstream outcome reports a Fail for a decision. Without
257    /// this, an answer the world has since disproven keeps being replayed until its TTL expires —
258    /// and each replay carries a `cache_source` pointing at a receipt that now records a failure,
259    /// so the audit trail would actively contradict what was served.
260    ///
261    /// Returns how many entries were dropped. A single decision can back several keys once the
262    /// same prompt is seen under more than one ladder, so this scans by source rather than
263    /// assuming one.
264    pub fn retract(&self, source: uuid::Uuid) -> usize {
265        let Ok(mut map) = self.entries.lock() else {
266            return 0;
267        };
268        let before = map.len();
269        map.retain(|_, e| e.source != source);
270        before - map.len()
271    }
272
273    /// Entries currently held (tests and `/metrics`).
274    #[must_use]
275    pub fn len(&self) -> usize {
276        self.entries.lock().map(|m| m.len()).unwrap_or(0)
277    }
278
279    /// Whether the cache holds nothing.
280    #[must_use]
281    pub fn is_empty(&self) -> bool {
282        self.len() == 0
283    }
284}
285
286#[async_trait::async_trait]
287impl CacheStore for VerifiedCache {
288    // Delegates to the existing synchronous methods rather than reimplementing them. The
289    // in-process path is the one four review rounds hardened — a parallel async copy would be free
290    // to drift on exactly the rules that keep an unproven answer out.
291    async fn get(&self, key: &str, now: Instant) -> Option<Entry> {
292        self.get_by_key(key, now)
293    }
294
295    async fn put(&self, key: &str, _source: uuid::Uuid, entry: Entry, _now: Instant) {
296        self.put_by_key(key, entry);
297    }
298
299    async fn retract(&self, source: uuid::Uuid) -> usize {
300        Self::retract(self, source)
301    }
302
303    async fn entry_count(&self) -> usize {
304        Self::len(self)
305    }
306}
307
308/// A Redis-backed L2, so entries and retractions cross process boundaries.
309///
310/// Enabled by the `redis-cache` feature and a `redis_url`. Without it the cache is per-replica:
311/// behind N instances a session's answer is cached N times over and the hit rate drops roughly by
312/// N. Worse, a retraction only reaches the replica that received the feedback — the other N-1 keep
313/// serving an answer the world has disproven, which is the multi-instance form of the bug this
314/// module already had once.
315///
316/// ## Keys
317///
318/// - `fp:vc:<key>` → the serialized entry, with a Redis TTL so expiry is the server's job rather
319///   than ours. A manual sweep across replicas would race; `SETEX` does not.
320/// - `fp:vc:src:<uuid>` → a SET of the entry keys that decision proved, so `retract` can find them.
321///   Given the same TTL, so the index cannot outlive what it points at and leak.
322///
323/// ## What it deliberately does not do
324///
325/// No local caching of L2 reads. A stale local copy would survive a retraction issued on another
326/// replica — exactly the failure this exists to prevent — and a cache that can serve a disproven
327/// answer is worse than no cache in a product whose claim is that it does not.
328#[cfg(feature = "redis-cache")]
329pub struct RedisStore {
330    client: redis::aio::ConnectionManager,
331    ttl_secs: u64,
332}
333
334// Hand-written: `ConnectionManager` is not `Debug`, and printing it would risk putting a
335// credential-bearing URL into a log line anyway.
336#[cfg(feature = "redis-cache")]
337impl std::fmt::Debug for RedisStore {
338    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
339        f.debug_struct("RedisStore")
340            .field("ttl_secs", &self.ttl_secs)
341            .finish_non_exhaustive()
342    }
343}
344
345/// How long to wait for Redis at startup before giving up.
346///
347/// Short on purpose: this runs before the listener binds, so every second here is a second the
348/// proxy is not serving. A reachable Redis answers in milliseconds; one that needs ten seconds is
349/// not one to put in the request path.
350#[cfg(feature = "redis-cache")]
351const CONNECT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
352
353/// Strip credentials from a Redis URL so it can appear in a log line.
354///
355/// `redis://user:hunter2@cache:6379/0` → `redis://cache:6379/0`. A connect failure has to name the
356/// server or the message is useless for fixing it, and the userinfo is the one part that must not
357/// be named. This module's own `Debug` impl avoids printing the URL for exactly this reason; the
358/// error paths were interpolating it raw, which made that care pointless.
359#[cfg(feature = "redis-cache")]
360#[must_use]
361pub fn redact_redis_url(url: &str) -> String {
362    // Split on the LAST `@` before any path: a password may itself contain `@`, and splitting on
363    // the first would leak the tail of it.
364    let Some((scheme, rest)) = url.split_once("://") else {
365        return "<malformed redis url>".to_owned();
366    };
367    let host_part = match rest.rfind('@') {
368        Some(i) => &rest[i + 1..],
369        None => rest,
370    };
371    format!("{scheme}://{host_part}")
372}
373
374#[cfg(feature = "redis-cache")]
375impl RedisStore {
376    /// Connect to `url`.
377    ///
378    /// # Errors
379    /// A malformed URL or an unreachable server. Surfaced at startup rather than swallowed: a
380    /// cache that silently degrades to "always miss" looks like the feature is off, and an operator
381    /// who configured Redis should be told it is not working.
382    pub async fn connect(url: &str, ttl_secs: u64) -> Result<Self, String> {
383        let safe = redact_redis_url(url);
384        let client = redis::Client::open(url).map_err(|e| format!("redis url {safe}: {e}"))?;
385        // Bounded, because `ConnectionManager::new` retries a dead server indefinitely by default
386        // — the proxy simply never finishes starting, with no error and no listening socket. A
387        // hang is the worst of the three outcomes here: an operator can act on a refusal and can
388        // live with a degraded cache, but "startup never returns" looks like a deploy that stalled
389        // for unrelated reasons.
390        let mgr = tokio::time::timeout(CONNECT_TIMEOUT, redis::aio::ConnectionManager::new(client))
391            .await
392            .map_err(|_| {
393                format!(
394                    "redis at {safe} did not answer within {}s. The proxy refuses to start rather \
395                 than run with a cache that silently stays per-replica — check the server is \
396                 reachable, or remove redis_url to use the in-process cache.",
397                    CONNECT_TIMEOUT.as_secs()
398                )
399            })?
400            .map_err(|e| format!("redis connect {safe}: {e}"))?;
401
402        // `ConnectionManager::new` connects LAZILY: it returns Ok against a server that is not
403        // running, the proxy logs "shared via redis", binds, and serves with a cache that never
404        // works. That is worse than degrading to per-replica — it is non-functional while
405        // announcing success, which is the exact failure this fail-fast exists to prevent.
406        //
407        // So prove the connection with a real round trip before claiming it.
408        let mut probe = mgr.clone();
409        let pong: Result<String, _> = tokio::time::timeout(
410            CONNECT_TIMEOUT,
411            redis::cmd("PING").query_async(&mut probe),
412        )
413        .await
414        .map_err(|_| {
415            format!(
416                "redis at {safe} did not answer PING within {}s. Refusing to start rather than \
417                 run with a cache that reports itself as shared and never works — check the \
418                 server is reachable, or remove redis_url to use the in-process cache.",
419                CONNECT_TIMEOUT.as_secs()
420            )
421        })?;
422        pong.map_err(|e| format!("redis at {safe} rejected PING: {e}"))?;
423
424        Ok(Self {
425            client: mgr,
426            ttl_secs,
427        })
428    }
429
430    fn entry_key(key: &str) -> String {
431        format!("fp:vc:{key}")
432    }
433
434    fn source_key(source: uuid::Uuid) -> String {
435        format!("fp:vc:src:{source}")
436    }
437}
438
439#[cfg(feature = "redis-cache")]
440#[async_trait::async_trait]
441impl CacheStore for RedisStore {
442    async fn get(&self, key: &str, _now: Instant) -> Option<Entry> {
443        let mut c = self.client.clone();
444        let raw: Option<String> = redis::cmd("GET")
445            .arg(Self::entry_key(key))
446            .query_async(&mut c)
447            .await
448            .ok()?;
449        // A malformed value is treated as a miss rather than an error: the request simply runs the
450        // ladder, which is always safe. Failing the request because a cache entry did not
451        // deserialize would turn an optimisation into an outage.
452        serde_json::from_str(&raw?).ok()
453    }
454
455    async fn put(&self, key: &str, source: uuid::Uuid, entry: Entry, _now: Instant) {
456        let Ok(body) = serde_json::to_string(&entry) else {
457            return;
458        };
459        let mut c = self.client.clone();
460        let ek = Self::entry_key(key);
461        // Pipelined, not transactional: if the index write is lost the entry becomes unretractable
462        // until TTL, so the TTL is the backstop that bounds that window. A MULTI would remove the
463        // race but not the failure mode, and the entry TTL already caps the exposure.
464        let _: Result<(), _> = redis::pipe()
465            .cmd("SETEX")
466            .arg(&ek)
467            .arg(self.ttl_secs)
468            .arg(body)
469            .ignore()
470            .cmd("SADD")
471            .arg(Self::source_key(source))
472            .arg(&ek)
473            .ignore()
474            .cmd("EXPIRE")
475            .arg(Self::source_key(source))
476            .arg(self.ttl_secs)
477            .ignore()
478            .query_async::<()>(&mut c)
479            .await;
480    }
481
482    async fn retract(&self, source: uuid::Uuid) -> usize {
483        let mut c = self.client.clone();
484        let sk = Self::source_key(source);
485        let keys: Vec<String> = redis::cmd("SMEMBERS")
486            .arg(&sk)
487            .query_async(&mut c)
488            .await
489            .unwrap_or_default();
490        if keys.is_empty() {
491            return 0;
492        }
493        let n: usize = redis::cmd("DEL")
494            .arg(&keys)
495            .query_async(&mut c)
496            .await
497            .unwrap_or(0);
498        let _: Result<(), _> = redis::cmd("DEL").arg(&sk).query_async(&mut c).await;
499        n
500    }
501
502    async fn entry_count(&self) -> usize {
503        let mut c = self.client.clone();
504        let keys: Vec<String> = redis::cmd("KEYS")
505            .arg("fp:vc:*")
506            .query_async(&mut c)
507            .await
508            .unwrap_or_default();
509        // ponytail: KEYS is O(n) and blocks Redis; acceptable because this is only read by
510        // /metrics and tests, never on the request path. Switch to SCAN if it is ever called
511        // per-request.
512        keys.iter().filter(|k| !k.starts_with("fp:vc:src:")).count()
513    }
514}
515
516#[cfg(test)]
517mod tests {
518    use super::*;
519    use firstpass_core::{
520        Attempt, DeferredVerdict, Features, FinalOutcome, GENESIS_HASH, GateResult, Mode,
521        PolicyRef, RequestInfo, TaskKind,
522    };
523
524    fn resp(text: &str) -> ModelResponse {
525        ModelResponse {
526            model: "anthropic/claude-haiku-4-5".to_owned(),
527            text: text.to_owned(),
528            in_tokens: 10,
529            cache_write_tokens: 0,
530            cache_read_tokens: 0,
531            out_tokens: 5,
532            raw: serde_json::Value::Null,
533        }
534    }
535
536    fn ladder() -> Vec<String> {
537        vec!["anthropic/claude-haiku-4-5".to_owned()]
538    }
539
540    fn trace(served_from: ServedFrom, verdict: Verdict) -> Trace {
541        Trace {
542            trace_id: uuid::Uuid::now_v7(),
543            prev_hash: GENESIS_HASH.to_owned(),
544            tenant_id: "t".to_owned(),
545            session_id: "s".to_owned(),
546            ts: jiff::Timestamp::UNIX_EPOCH,
547            mode: Mode::Enforce,
548            policy: PolicyRef {
549                id: "static-ladder@v0".to_owned(),
550                explore: false,
551                propensity: None,
552                mode_profile: None,
553            },
554            request: RequestInfo {
555                api: "anthropic.messages".to_owned(),
556                prompt_hash: "hash-abc".to_owned(),
557                features: Features::new(TaskKind::CodeEdit),
558            },
559            attempts: vec![Attempt {
560                rung: 0,
561                model: "anthropic/claude-haiku-4-5".to_owned(),
562                provider: "anthropic".to_owned(),
563                in_tokens: 10,
564                cache_write_tokens: 0,
565                cache_read_tokens: 0,
566                out_tokens: 5,
567                cost_usd: 0.01,
568                latency_ms: 100,
569                gates: vec![GateResult::deterministic("non-empty", verdict, 1)],
570                verdict,
571            }],
572            final_: FinalOutcome {
573                served_rung: Some(0),
574                served_from,
575                total_cost_usd: 0.01,
576                gate_cost_usd: 0.0,
577                total_latency_ms: 120,
578                escalations: 0,
579                counterfactual_baseline_usd: 0.05,
580                savings_usd: 0.04,
581                cache_source: None,
582            },
583            deferred: vec![],
584            predicted_pass: None,
585            probe: None,
586            elastic: None,
587            rollout: None,
588            shadow: None,
589            route_ix: None,
590        }
591    }
592
593    #[test]
594    fn a_passing_answer_is_stored_and_replayed_with_its_proof() {
595        let c = VerifiedCache::new(Duration::from_secs(60), 100);
596        let now = Instant::now();
597        let t = trace(ServedFrom::Attempt, Verdict::Pass);
598
599        assert!(c.offer(&t, &resp("the answer"), &ladder(), now));
600
601        let hit = c.get("t", "hash-abc", &ladder(), now).expect("hit");
602        assert_eq!(hit.response.text, "the answer");
603        // Not merely "cached" — the decision whose gate passed, so the proof is one lookup away.
604        assert_eq!(hit.source, t.trace_id);
605        assert!((hit.original_cost_usd - 0.01).abs() < f64::EPSILON);
606    }
607
608    #[test]
609    fn a_failed_answer_is_never_cached() {
610        let c = VerifiedCache::new(Duration::from_secs(60), 100);
611        let now = Instant::now();
612
613        assert!(!c.offer(
614            &trace(ServedFrom::Attempt, Verdict::Fail),
615            &resp("bad"),
616            &ladder(),
617            now
618        ));
619
620        assert!(
621            c.get("t", "hash-abc", &ladder(), now).is_none(),
622            "must not be replayable"
623        );
624    }
625
626    #[test]
627    fn an_abstain_is_never_cached() {
628        // "The gate could not tell" is exactly the state that must not be frozen and replayed —
629        // every later hit would inherit an uncertainty nobody re-examines.
630        let c = VerifiedCache::new(Duration::from_secs(60), 100);
631        let now = Instant::now();
632        assert!(!c.offer(
633            &trace(ServedFrom::Attempt, Verdict::Abstain),
634            &resp("?"),
635            &ladder(),
636            now
637        ));
638        assert!(c.get("t", "hash-abc", &ladder(), now).is_none());
639    }
640
641    #[test]
642    fn a_budget_exhausted_best_effort_serve_is_never_cached() {
643        // BestAttempt is served WITHOUT a passing verdict. Caching it would launder an unverified
644        // answer into a served one, and no later reader could tell the difference.
645        let c = VerifiedCache::new(Duration::from_secs(60), 100);
646        let now = Instant::now();
647        assert!(!c.offer(
648            &trace(ServedFrom::BestAttempt, Verdict::Pass),
649            &resp("x"),
650            &ladder(),
651            now
652        ));
653        assert!(c.get("t", "hash-abc", &ladder(), now).is_none());
654    }
655
656    #[test]
657    fn a_later_failing_verdict_retracts_the_proof() {
658        // A deferred gate (tests in CI, downstream feedback) that comes back Fail means the answer
659        // was not good after all. It must not be sitting in the cache waiting to be served again.
660        let c = VerifiedCache::new(Duration::from_secs(60), 100);
661        let now = Instant::now();
662        let mut t = trace(ServedFrom::Attempt, Verdict::Pass);
663        t.deferred.push(DeferredVerdict {
664            gate_id: "tests".to_owned(),
665            verdict: Verdict::Fail,
666            score: None,
667            reported_at: jiff::Timestamp::UNIX_EPOCH,
668            reporter: "ci".to_owned(),
669        });
670
671        assert!(!c.offer(&t, &resp("looked fine"), &ladder(), now));
672    }
673
674    #[cfg(feature = "redis-cache")]
675    #[test]
676    fn a_redis_url_never_reaches_a_log_line_with_its_password() {
677        // The module's own Debug impl avoids printing the URL for this reason; the error paths
678        // were interpolating it raw, which made that care pointless.
679        assert_eq!(
680            redact_redis_url("redis://user:hunter2@cache:6379/0"),
681            "redis://cache:6379/0"
682        );
683        // A password containing '@' must not leak its tail — split on the LAST '@', not the first.
684        assert_eq!(
685            redact_redis_url("redis://user:p@ss@cache:6379/0"),
686            "redis://cache:6379/0"
687        );
688        assert_eq!(
689            redact_redis_url("redis://cache:6379/0"),
690            "redis://cache:6379/0"
691        );
692        assert_eq!(redact_redis_url("rediss://u:p@h:6380"), "rediss://h:6380");
693        assert_eq!(redact_redis_url("nonsense"), "<malformed redis url>");
694
695        for u in [
696            "redis://user:hunter2@cache:6379/0",
697            "redis://user:p@ss@cache:6379/0",
698        ] {
699            let out = redact_redis_url(u);
700            assert!(!out.contains("hunter2"), "{out}");
701            assert!(!out.contains("p@ss"), "{out}");
702            assert!(!out.contains("user"), "{out}");
703        }
704    }
705
706    /// Exercise the real Redis backend.
707    ///
708    /// Requires a reachable server at `FIRSTPASS_TEST_REDIS`. **Absent, this fails rather than
709    /// skipping**: a suite that quietly passes when its subject is unavailable reports green while
710    /// testing nothing, which is how this repo's `provider-smoke` job once read as verified.
711    #[cfg(feature = "redis-cache")]
712    #[tokio::test]
713    async fn redis_store_round_trips_and_retracts() {
714        let url = std::env::var("FIRSTPASS_TEST_REDIS").unwrap_or_else(|_| {
715            panic!(
716                "set FIRSTPASS_TEST_REDIS (e.g. redis://127.0.0.1:6379/15) to run the redis-cache \
717                 tests. Not skipped on purpose: a green suite that tested nothing is worse than a \
718                 red one."
719            )
720        });
721        let store = RedisStore::connect(&url, 60).await.expect("connect");
722
723        let t = trace(ServedFrom::Attempt, Verdict::Pass);
724        let k = VerifiedCache::compose_key("t", &format!("h-{}", t.trace_id), &ladder());
725        let now = Instant::now();
726        let entry = Entry::new(resp("shared answer"), &t, now);
727
728        store.put(&k, t.trace_id, entry, now).await;
729        let got = store
730            .get(&k, now)
731            .await
732            .expect("entry crossed the boundary");
733        assert_eq!(got.response.text, "shared answer");
734        // The PROOF is what has to survive; a replay's receipt is built from it.
735        assert_eq!(got.source, t.trace_id);
736
737        // Retraction must work through the source index, since feedback names a decision, not a key.
738        assert_eq!(store.retract(t.trace_id).await, 1);
739        assert!(
740            store.get(&k, now).await.is_none(),
741            "a retracted answer must stop being replayable in the SHARED store too"
742        );
743    }
744
745    #[cfg(feature = "redis-cache")]
746    #[tokio::test]
747    async fn redis_store_isolates_tenants() {
748        // The cross-tenant leak found in review must not reappear through the shared surface.
749        let url = std::env::var("FIRSTPASS_TEST_REDIS")
750            .expect("set FIRSTPASS_TEST_REDIS to run the redis-cache tests");
751        let store = RedisStore::connect(&url, 60).await.expect("connect");
752
753        let mut t = trace(ServedFrom::Attempt, Verdict::Pass);
754        t.tenant_id = "tenant-a".to_owned();
755        let hash = format!("iso-{}", t.trace_id);
756        let ka = VerifiedCache::compose_key("tenant-a", &hash, &ladder());
757        let kb = VerifiedCache::compose_key("tenant-b", &hash, &ladder());
758        let now = Instant::now();
759
760        store
761            .put(&ka, t.trace_id, Entry::new(resp("a"), &t, now), now)
762            .await;
763
764        assert!(
765            store.get(&kb, now).await.is_none(),
766            "cross-tenant replay is a data leak"
767        );
768        assert!(store.get(&ka, now).await.is_some());
769        store.retract(t.trace_id).await;
770    }
771
772    #[tokio::test]
773    async fn the_trait_and_the_concrete_api_agree_on_keys() {
774        // The two stores must compose the key identically or a multi-replica deployment silently
775        // never hits: one writes under a key the other never looks up, and the only symptom is a
776        // hit rate that quietly stays at zero.
777        let c = VerifiedCache::new(Duration::from_secs(60), 100);
778        let now = Instant::now();
779        let t = trace(ServedFrom::Attempt, Verdict::Pass);
780        assert!(c.offer(&t, &resp("via offer"), &ladder(), now));
781
782        let k = VerifiedCache::compose_key("t", "hash-abc", &ladder());
783        let via_trait = CacheStore::get(&c, &k, now)
784            .await
785            .expect("trait sees offer's entry");
786        assert_eq!(via_trait.response.text, "via offer");
787    }
788
789    #[tokio::test]
790    async fn the_trait_path_still_isolates_tenants() {
791        // The leak found in review must not reappear through the new surface: `compose_key` is the
792        // single place the tenant enters, and both stores go through it.
793        let a = VerifiedCache::compose_key("tenant-a", "h", &ladder());
794        let b = VerifiedCache::compose_key("tenant-b", "h", &ladder());
795        assert_ne!(a, b, "tenant must be part of the key on the trait path too");
796        assert!(a.starts_with("tenant-a|"), "{a}");
797    }
798
799    #[tokio::test]
800    async fn an_entry_survives_a_serde_round_trip_without_its_local_clock() {
801        // Redis carries entries between processes, where an Instant is meaningless. The proof
802        // (source) and the answer must survive; `stored` is re-stamped, and expiry there is
803        // Redis's TTL rather than a comparison of two machines' monotonic clocks.
804        let e = Entry {
805            response: resp("answer"),
806            source: uuid::Uuid::now_v7(),
807            original_cost_usd: 0.01,
808            stored: Instant::now(),
809        };
810        let wire = serde_json::to_string(&e).expect("serializes");
811        assert!(
812            !wire.contains("stored"),
813            "a local Instant must not go on the wire: {wire}"
814        );
815
816        let back: Entry = serde_json::from_str(&wire).expect("deserializes");
817        assert_eq!(back.response.text, "answer");
818        assert_eq!(back.source, e.source, "the PROOF must survive the crossing");
819        assert!((back.original_cost_usd - 0.01).abs() < f64::EPSILON);
820    }
821
822    #[test]
823    fn a_retraction_evicts_the_answer_it_disproved() {
824        // Found in review: rule 4 only inspects the receipt at INSERT time, and deferred verdicts
825        // arrive later via /v1/feedback — so on its own it is a no-op in production and a
826        // disproven answer keeps being served until TTL. This is the half that actually runs.
827        let c = VerifiedCache::new(Duration::from_secs(600), 100);
828        let now = Instant::now();
829        let t = trace(ServedFrom::Attempt, Verdict::Pass);
830        assert!(c.offer(&t, &resp("looked fine"), &ladder(), now));
831        assert!(c.get("t", "hash-abc", &ladder(), now).is_some());
832
833        assert_eq!(c.retract(t.trace_id), 1);
834
835        assert!(
836            c.get("t", "hash-abc", &ladder(), now).is_none(),
837            "an answer whose proof was retracted must stop being replayed immediately"
838        );
839    }
840
841    #[test]
842    fn retracting_an_unrelated_decision_evicts_nothing() {
843        let c = VerifiedCache::new(Duration::from_secs(600), 100);
844        let now = Instant::now();
845        c.offer(
846            &trace(ServedFrom::Attempt, Verdict::Pass),
847            &resp("a"),
848            &ladder(),
849            now,
850        );
851
852        assert_eq!(c.retract(uuid::Uuid::now_v7()), 0);
853        assert!(c.get("t", "hash-abc", &ladder(), now).is_some());
854    }
855
856    #[test]
857    fn a_cache_hit_is_not_itself_re_cached() {
858        // Otherwise `cache_source` would point at a replay instead of the decision that actually
859        // ran a gate, and the provenance chain — the whole feature — would go one hop stale.
860        let c = VerifiedCache::new(Duration::from_secs(60), 100);
861        let now = Instant::now();
862        let mut t = trace(ServedFrom::Attempt, Verdict::Pass);
863        t.final_.cache_source = Some(uuid::Uuid::now_v7());
864        assert!(!c.offer(&t, &resp("replayed"), &ladder(), now));
865    }
866
867    #[test]
868    fn one_tenant_can_never_replay_anothers_answer() {
869        // Caught in review, and the worst bug in this feature: the key was
870        // `prompt_hash | ladder`, with no tenant. The prompt salt is per-DEPLOYMENT, so two
871        // tenants sending an identical prompt hash identically — tenant B would have been served
872        // tenant A's paid-for, verified answer, with a receipt that looked entirely normal.
873        let c = VerifiedCache::new(Duration::from_secs(60), 100);
874        let now = Instant::now();
875        let mut t = trace(ServedFrom::Attempt, Verdict::Pass);
876        t.tenant_id = "tenant-a".to_owned();
877        assert!(c.offer(&t, &resp("a's private answer"), &ladder(), now));
878
879        assert!(
880            c.get("tenant-b", "hash-abc", &ladder(), now).is_none(),
881            "cross-tenant replay — this is a data leak, not a cache miss"
882        );
883        assert!(
884            c.get("tenant-a", "hash-abc", &ladder(), now).is_some(),
885            "the owning tenant must still hit"
886        );
887    }
888
889    #[test]
890    fn the_cached_tenant_comes_from_the_receipt_not_the_caller() {
891        // Reading it off the receipt means a call site cannot pass the wrong tenant and quietly
892        // file an answer under someone else's key.
893        let c = VerifiedCache::new(Duration::from_secs(60), 100);
894        let now = Instant::now();
895        let mut t = trace(ServedFrom::Attempt, Verdict::Pass);
896        t.tenant_id = "owner".to_owned();
897        c.offer(&t, &resp("x"), &ladder(), now);
898
899        assert!(c.get("owner", "hash-abc", &ladder(), now).is_some());
900        assert!(c.get("t", "hash-abc", &ladder(), now).is_none());
901    }
902
903    #[test]
904    fn a_different_ladder_is_a_different_question() {
905        // An answer proven against one gate/ladder configuration says nothing about another.
906        let c = VerifiedCache::new(Duration::from_secs(60), 100);
907        let now = Instant::now();
908        c.offer(
909            &trace(ServedFrom::Attempt, Verdict::Pass),
910            &resp("a"),
911            &ladder(),
912            now,
913        );
914
915        let other = vec!["openai/gpt-5.5".to_owned()];
916        assert!(c.get("t", "hash-abc", &other, now).is_none());
917        assert!(c.get("t", "hash-abc", &ladder(), now).is_some());
918    }
919
920    #[test]
921    fn an_entry_expires() {
922        let c = VerifiedCache::new(Duration::from_secs(1), 100);
923        let t0 = Instant::now();
924        c.offer(
925            &trace(ServedFrom::Attempt, Verdict::Pass),
926            &resp("a"),
927            &ladder(),
928            t0,
929        );
930
931        assert!(c.get("t", "hash-abc", &ladder(), t0).is_some());
932        assert!(
933            c.get("t", "hash-abc", &ladder(), t0 + Duration::from_secs(2))
934                .is_none(),
935            "a stale proof is not a proof"
936        );
937    }
938
939    #[test]
940    fn the_cache_cannot_grow_without_bound() {
941        let c = VerifiedCache::new(Duration::from_secs(600), 10);
942        let now = Instant::now();
943        for i in 0..100 {
944            let mut t = trace(ServedFrom::Attempt, Verdict::Pass);
945            t.request.prompt_hash = format!("h{i}");
946            c.offer(&t, &resp("a"), &ladder(), now);
947        }
948        assert!(c.len() <= 10, "expected <= 10 entries, got {}", c.len());
949    }
950}