use std::collections::HashMap;
use std::sync::Mutex;
use std::time::{Duration, Instant};
use firstpass_core::{ServedFrom, Trace, Verdict};
use crate::provider::ModelResponse;
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct Entry {
pub response: ModelResponse,
pub source: uuid::Uuid,
pub original_cost_usd: f64,
#[serde(skip, default = "Instant::now")]
stored: Instant,
}
#[must_use]
pub fn is_cacheable(trace: &Trace) -> bool {
if trace.final_.served_from != ServedFrom::Attempt {
return false;
}
if trace.final_.cache_source.is_some() {
return false;
}
let Some(served) = trace.final_.served_rung else {
return false;
};
if !trace
.attempts
.iter()
.any(|a| a.rung == served && a.verdict == Verdict::Pass)
{
return false;
}
!trace.deferred.iter().any(|d| d.verdict == Verdict::Fail)
}
impl Entry {
#[must_use]
pub fn new(response: ModelResponse, trace: &Trace, now: Instant) -> Self {
Self {
response,
source: trace.trace_id,
original_cost_usd: trace.final_.total_cost_usd,
stored: now,
}
}
}
fn key(tenant: &str, prompt_hash: &str, ladder: &[String]) -> String {
format!("{tenant}|{prompt_hash}|{}", ladder.join(">"))
}
#[async_trait::async_trait]
pub trait CacheStore: Send + Sync + std::fmt::Debug {
async fn get(&self, key: &str, now: Instant) -> Option<Entry>;
async fn put(&self, key: &str, source: uuid::Uuid, entry: Entry, now: Instant);
async fn retract(&self, source: uuid::Uuid) -> usize;
async fn entry_count(&self) -> usize;
}
#[derive(Debug)]
pub struct VerifiedCache {
entries: Mutex<HashMap<String, Entry>>,
ttl: Duration,
max_entries: usize,
}
impl VerifiedCache {
#[must_use]
pub fn new(ttl: Duration, max_entries: usize) -> Self {
Self {
entries: Mutex::new(HashMap::new()),
ttl,
max_entries,
}
}
#[must_use]
pub fn get(
&self,
tenant: &str,
prompt_hash: &str,
ladder: &[String],
now: Instant,
) -> Option<Entry> {
let Ok(map) = self.entries.lock() else {
return None;
};
map.get(&key(tenant, prompt_hash, ladder))
.filter(|e| now.duration_since(e.stored) < self.ttl)
.cloned()
}
pub fn offer(
&self,
trace: &Trace,
response: &ModelResponse,
ladder: &[String],
now: Instant,
) -> bool {
if !is_cacheable(trace) {
return false;
}
let mut entry = Entry {
response: response.clone(),
source: trace.trace_id,
original_cost_usd: trace.final_.total_cost_usd,
stored: now,
};
entry.stored = now;
self.put_by_key(
&key(&trace.tenant_id, &trace.request.prompt_hash, ladder),
entry,
);
true
}
#[must_use]
pub fn get_by_key(&self, key: &str, now: Instant) -> Option<Entry> {
let Ok(map) = self.entries.lock() else {
return None;
};
map.get(key)
.filter(|e| now.duration_since(e.stored) < self.ttl)
.cloned()
}
pub fn put_by_key(&self, key: &str, entry: Entry) {
let Ok(mut map) = self.entries.lock() else {
return;
};
let now = entry.stored;
if map.len() >= self.max_entries {
map.retain(|_, e| now.duration_since(e.stored) < self.ttl);
if map.len() >= self.max_entries {
let mut by_age: Vec<(String, Instant)> =
map.iter().map(|(k, e)| (k.clone(), e.stored)).collect();
by_age.sort_by_key(|(_, t)| *t);
for (k, _) in by_age.into_iter().take(self.max_entries.max(2) / 2) {
map.remove(&k);
}
}
}
map.insert(key.to_owned(), entry);
}
#[must_use]
pub fn compose_key(tenant: &str, prompt_hash: &str, ladder: &[String]) -> String {
key(tenant, prompt_hash, ladder)
}
pub fn retract(&self, source: uuid::Uuid) -> usize {
let Ok(mut map) = self.entries.lock() else {
return 0;
};
let before = map.len();
map.retain(|_, e| e.source != source);
before - map.len()
}
#[must_use]
pub fn len(&self) -> usize {
self.entries.lock().map(|m| m.len()).unwrap_or(0)
}
#[must_use]
pub fn is_empty(&self) -> bool {
self.len() == 0
}
}
#[async_trait::async_trait]
impl CacheStore for VerifiedCache {
async fn get(&self, key: &str, now: Instant) -> Option<Entry> {
self.get_by_key(key, now)
}
async fn put(&self, key: &str, _source: uuid::Uuid, entry: Entry, _now: Instant) {
self.put_by_key(key, entry);
}
async fn retract(&self, source: uuid::Uuid) -> usize {
Self::retract(self, source)
}
async fn entry_count(&self) -> usize {
Self::len(self)
}
}
#[cfg(feature = "redis-cache")]
pub struct RedisStore {
client: redis::aio::ConnectionManager,
ttl_secs: u64,
}
#[cfg(feature = "redis-cache")]
impl std::fmt::Debug for RedisStore {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("RedisStore")
.field("ttl_secs", &self.ttl_secs)
.finish_non_exhaustive()
}
}
#[cfg(feature = "redis-cache")]
const CONNECT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
#[cfg(feature = "redis-cache")]
#[must_use]
pub fn redact_redis_url(url: &str) -> String {
let Some((scheme, rest)) = url.split_once("://") else {
return "<malformed redis url>".to_owned();
};
let host_part = match rest.rfind('@') {
Some(i) => &rest[i + 1..],
None => rest,
};
format!("{scheme}://{host_part}")
}
#[cfg(feature = "redis-cache")]
impl RedisStore {
pub async fn connect(url: &str, ttl_secs: u64) -> Result<Self, String> {
let safe = redact_redis_url(url);
let client = redis::Client::open(url).map_err(|e| format!("redis url {safe}: {e}"))?;
let mgr = tokio::time::timeout(CONNECT_TIMEOUT, redis::aio::ConnectionManager::new(client))
.await
.map_err(|_| {
format!(
"redis at {safe} did not answer within {}s. The proxy refuses to start rather \
than run with a cache that silently stays per-replica — check the server is \
reachable, or remove redis_url to use the in-process cache.",
CONNECT_TIMEOUT.as_secs()
)
})?
.map_err(|e| format!("redis connect {safe}: {e}"))?;
let mut probe = mgr.clone();
let pong: Result<String, _> = tokio::time::timeout(
CONNECT_TIMEOUT,
redis::cmd("PING").query_async(&mut probe),
)
.await
.map_err(|_| {
format!(
"redis at {safe} did not answer PING within {}s. Refusing to start rather than \
run with a cache that reports itself as shared and never works — check the \
server is reachable, or remove redis_url to use the in-process cache.",
CONNECT_TIMEOUT.as_secs()
)
})?;
pong.map_err(|e| format!("redis at {safe} rejected PING: {e}"))?;
Ok(Self {
client: mgr,
ttl_secs,
})
}
fn entry_key(key: &str) -> String {
format!("fp:vc:{key}")
}
fn source_key(source: uuid::Uuid) -> String {
format!("fp:vc:src:{source}")
}
}
#[cfg(feature = "redis-cache")]
#[async_trait::async_trait]
impl CacheStore for RedisStore {
async fn get(&self, key: &str, _now: Instant) -> Option<Entry> {
let mut c = self.client.clone();
let raw: Option<String> = redis::cmd("GET")
.arg(Self::entry_key(key))
.query_async(&mut c)
.await
.ok()?;
serde_json::from_str(&raw?).ok()
}
async fn put(&self, key: &str, source: uuid::Uuid, entry: Entry, _now: Instant) {
let Ok(body) = serde_json::to_string(&entry) else {
return;
};
let mut c = self.client.clone();
let ek = Self::entry_key(key);
let _: Result<(), _> = redis::pipe()
.cmd("SETEX")
.arg(&ek)
.arg(self.ttl_secs)
.arg(body)
.ignore()
.cmd("SADD")
.arg(Self::source_key(source))
.arg(&ek)
.ignore()
.cmd("EXPIRE")
.arg(Self::source_key(source))
.arg(self.ttl_secs)
.ignore()
.query_async::<()>(&mut c)
.await;
}
async fn retract(&self, source: uuid::Uuid) -> usize {
let mut c = self.client.clone();
let sk = Self::source_key(source);
let keys: Vec<String> = redis::cmd("SMEMBERS")
.arg(&sk)
.query_async(&mut c)
.await
.unwrap_or_default();
if keys.is_empty() {
return 0;
}
let n: usize = redis::cmd("DEL")
.arg(&keys)
.query_async(&mut c)
.await
.unwrap_or(0);
let _: Result<(), _> = redis::cmd("DEL").arg(&sk).query_async(&mut c).await;
n
}
async fn entry_count(&self) -> usize {
let mut c = self.client.clone();
let keys: Vec<String> = redis::cmd("KEYS")
.arg("fp:vc:*")
.query_async(&mut c)
.await
.unwrap_or_default();
keys.iter().filter(|k| !k.starts_with("fp:vc:src:")).count()
}
}
#[cfg(test)]
mod tests {
use super::*;
use firstpass_core::{
Attempt, DeferredVerdict, Features, FinalOutcome, GENESIS_HASH, GateResult, Mode,
PolicyRef, RequestInfo, TaskKind,
};
fn resp(text: &str) -> ModelResponse {
ModelResponse {
model: "anthropic/claude-haiku-4-5".to_owned(),
text: text.to_owned(),
in_tokens: 10,
cache_write_tokens: 0,
cache_read_tokens: 0,
out_tokens: 5,
raw: serde_json::Value::Null,
}
}
fn ladder() -> Vec<String> {
vec!["anthropic/claude-haiku-4-5".to_owned()]
}
fn trace(served_from: ServedFrom, verdict: Verdict) -> Trace {
Trace {
trace_id: uuid::Uuid::now_v7(),
prev_hash: GENESIS_HASH.to_owned(),
tenant_id: "t".to_owned(),
session_id: "s".to_owned(),
ts: jiff::Timestamp::UNIX_EPOCH,
mode: Mode::Enforce,
policy: PolicyRef {
id: "static-ladder@v0".to_owned(),
explore: false,
propensity: None,
mode_profile: None,
},
request: RequestInfo {
api: "anthropic.messages".to_owned(),
prompt_hash: "hash-abc".to_owned(),
features: Features::new(TaskKind::CodeEdit),
},
attempts: vec![Attempt {
rung: 0,
model: "anthropic/claude-haiku-4-5".to_owned(),
provider: "anthropic".to_owned(),
in_tokens: 10,
cache_write_tokens: 0,
cache_read_tokens: 0,
out_tokens: 5,
cost_usd: 0.01,
latency_ms: 100,
gates: vec![GateResult::deterministic("non-empty", verdict, 1)],
verdict,
}],
final_: FinalOutcome {
served_rung: Some(0),
served_from,
total_cost_usd: 0.01,
gate_cost_usd: 0.0,
total_latency_ms: 120,
escalations: 0,
counterfactual_baseline_usd: 0.05,
savings_usd: 0.04,
cache_source: None,
},
deferred: vec![],
predicted_pass: None,
probe: None,
elastic: None,
rollout: None,
shadow: None,
route_ix: None,
}
}
#[test]
fn a_passing_answer_is_stored_and_replayed_with_its_proof() {
let c = VerifiedCache::new(Duration::from_secs(60), 100);
let now = Instant::now();
let t = trace(ServedFrom::Attempt, Verdict::Pass);
assert!(c.offer(&t, &resp("the answer"), &ladder(), now));
let hit = c.get("t", "hash-abc", &ladder(), now).expect("hit");
assert_eq!(hit.response.text, "the answer");
assert_eq!(hit.source, t.trace_id);
assert!((hit.original_cost_usd - 0.01).abs() < f64::EPSILON);
}
#[test]
fn a_failed_answer_is_never_cached() {
let c = VerifiedCache::new(Duration::from_secs(60), 100);
let now = Instant::now();
assert!(!c.offer(
&trace(ServedFrom::Attempt, Verdict::Fail),
&resp("bad"),
&ladder(),
now
));
assert!(
c.get("t", "hash-abc", &ladder(), now).is_none(),
"must not be replayable"
);
}
#[test]
fn an_abstain_is_never_cached() {
let c = VerifiedCache::new(Duration::from_secs(60), 100);
let now = Instant::now();
assert!(!c.offer(
&trace(ServedFrom::Attempt, Verdict::Abstain),
&resp("?"),
&ladder(),
now
));
assert!(c.get("t", "hash-abc", &ladder(), now).is_none());
}
#[test]
fn a_budget_exhausted_best_effort_serve_is_never_cached() {
let c = VerifiedCache::new(Duration::from_secs(60), 100);
let now = Instant::now();
assert!(!c.offer(
&trace(ServedFrom::BestAttempt, Verdict::Pass),
&resp("x"),
&ladder(),
now
));
assert!(c.get("t", "hash-abc", &ladder(), now).is_none());
}
#[test]
fn a_later_failing_verdict_retracts_the_proof() {
let c = VerifiedCache::new(Duration::from_secs(60), 100);
let now = Instant::now();
let mut t = trace(ServedFrom::Attempt, Verdict::Pass);
t.deferred.push(DeferredVerdict {
gate_id: "tests".to_owned(),
verdict: Verdict::Fail,
score: None,
reported_at: jiff::Timestamp::UNIX_EPOCH,
reporter: "ci".to_owned(),
});
assert!(!c.offer(&t, &resp("looked fine"), &ladder(), now));
}
#[cfg(feature = "redis-cache")]
#[test]
fn a_redis_url_never_reaches_a_log_line_with_its_password() {
assert_eq!(
redact_redis_url("redis://user:hunter2@cache:6379/0"),
"redis://cache:6379/0"
);
assert_eq!(
redact_redis_url("redis://user:p@ss@cache:6379/0"),
"redis://cache:6379/0"
);
assert_eq!(
redact_redis_url("redis://cache:6379/0"),
"redis://cache:6379/0"
);
assert_eq!(redact_redis_url("rediss://u:p@h:6380"), "rediss://h:6380");
assert_eq!(redact_redis_url("nonsense"), "<malformed redis url>");
for u in [
"redis://user:hunter2@cache:6379/0",
"redis://user:p@ss@cache:6379/0",
] {
let out = redact_redis_url(u);
assert!(!out.contains("hunter2"), "{out}");
assert!(!out.contains("p@ss"), "{out}");
assert!(!out.contains("user"), "{out}");
}
}
#[cfg(feature = "redis-cache")]
#[tokio::test]
async fn redis_store_round_trips_and_retracts() {
let url = std::env::var("FIRSTPASS_TEST_REDIS").unwrap_or_else(|_| {
panic!(
"set FIRSTPASS_TEST_REDIS (e.g. redis://127.0.0.1:6379/15) to run the redis-cache \
tests. Not skipped on purpose: a green suite that tested nothing is worse than a \
red one."
)
});
let store = RedisStore::connect(&url, 60).await.expect("connect");
let t = trace(ServedFrom::Attempt, Verdict::Pass);
let k = VerifiedCache::compose_key("t", &format!("h-{}", t.trace_id), &ladder());
let now = Instant::now();
let entry = Entry::new(resp("shared answer"), &t, now);
store.put(&k, t.trace_id, entry, now).await;
let got = store
.get(&k, now)
.await
.expect("entry crossed the boundary");
assert_eq!(got.response.text, "shared answer");
assert_eq!(got.source, t.trace_id);
assert_eq!(store.retract(t.trace_id).await, 1);
assert!(
store.get(&k, now).await.is_none(),
"a retracted answer must stop being replayable in the SHARED store too"
);
}
#[cfg(feature = "redis-cache")]
#[tokio::test]
async fn redis_store_isolates_tenants() {
let url = std::env::var("FIRSTPASS_TEST_REDIS")
.expect("set FIRSTPASS_TEST_REDIS to run the redis-cache tests");
let store = RedisStore::connect(&url, 60).await.expect("connect");
let mut t = trace(ServedFrom::Attempt, Verdict::Pass);
t.tenant_id = "tenant-a".to_owned();
let hash = format!("iso-{}", t.trace_id);
let ka = VerifiedCache::compose_key("tenant-a", &hash, &ladder());
let kb = VerifiedCache::compose_key("tenant-b", &hash, &ladder());
let now = Instant::now();
store
.put(&ka, t.trace_id, Entry::new(resp("a"), &t, now), now)
.await;
assert!(
store.get(&kb, now).await.is_none(),
"cross-tenant replay is a data leak"
);
assert!(store.get(&ka, now).await.is_some());
store.retract(t.trace_id).await;
}
#[tokio::test]
async fn the_trait_and_the_concrete_api_agree_on_keys() {
let c = VerifiedCache::new(Duration::from_secs(60), 100);
let now = Instant::now();
let t = trace(ServedFrom::Attempt, Verdict::Pass);
assert!(c.offer(&t, &resp("via offer"), &ladder(), now));
let k = VerifiedCache::compose_key("t", "hash-abc", &ladder());
let via_trait = CacheStore::get(&c, &k, now)
.await
.expect("trait sees offer's entry");
assert_eq!(via_trait.response.text, "via offer");
}
#[tokio::test]
async fn the_trait_path_still_isolates_tenants() {
let a = VerifiedCache::compose_key("tenant-a", "h", &ladder());
let b = VerifiedCache::compose_key("tenant-b", "h", &ladder());
assert_ne!(a, b, "tenant must be part of the key on the trait path too");
assert!(a.starts_with("tenant-a|"), "{a}");
}
#[tokio::test]
async fn an_entry_survives_a_serde_round_trip_without_its_local_clock() {
let e = Entry {
response: resp("answer"),
source: uuid::Uuid::now_v7(),
original_cost_usd: 0.01,
stored: Instant::now(),
};
let wire = serde_json::to_string(&e).expect("serializes");
assert!(
!wire.contains("stored"),
"a local Instant must not go on the wire: {wire}"
);
let back: Entry = serde_json::from_str(&wire).expect("deserializes");
assert_eq!(back.response.text, "answer");
assert_eq!(back.source, e.source, "the PROOF must survive the crossing");
assert!((back.original_cost_usd - 0.01).abs() < f64::EPSILON);
}
#[test]
fn a_retraction_evicts_the_answer_it_disproved() {
let c = VerifiedCache::new(Duration::from_secs(600), 100);
let now = Instant::now();
let t = trace(ServedFrom::Attempt, Verdict::Pass);
assert!(c.offer(&t, &resp("looked fine"), &ladder(), now));
assert!(c.get("t", "hash-abc", &ladder(), now).is_some());
assert_eq!(c.retract(t.trace_id), 1);
assert!(
c.get("t", "hash-abc", &ladder(), now).is_none(),
"an answer whose proof was retracted must stop being replayed immediately"
);
}
#[test]
fn retracting_an_unrelated_decision_evicts_nothing() {
let c = VerifiedCache::new(Duration::from_secs(600), 100);
let now = Instant::now();
c.offer(
&trace(ServedFrom::Attempt, Verdict::Pass),
&resp("a"),
&ladder(),
now,
);
assert_eq!(c.retract(uuid::Uuid::now_v7()), 0);
assert!(c.get("t", "hash-abc", &ladder(), now).is_some());
}
#[test]
fn a_cache_hit_is_not_itself_re_cached() {
let c = VerifiedCache::new(Duration::from_secs(60), 100);
let now = Instant::now();
let mut t = trace(ServedFrom::Attempt, Verdict::Pass);
t.final_.cache_source = Some(uuid::Uuid::now_v7());
assert!(!c.offer(&t, &resp("replayed"), &ladder(), now));
}
#[test]
fn one_tenant_can_never_replay_anothers_answer() {
let c = VerifiedCache::new(Duration::from_secs(60), 100);
let now = Instant::now();
let mut t = trace(ServedFrom::Attempt, Verdict::Pass);
t.tenant_id = "tenant-a".to_owned();
assert!(c.offer(&t, &resp("a's private answer"), &ladder(), now));
assert!(
c.get("tenant-b", "hash-abc", &ladder(), now).is_none(),
"cross-tenant replay — this is a data leak, not a cache miss"
);
assert!(
c.get("tenant-a", "hash-abc", &ladder(), now).is_some(),
"the owning tenant must still hit"
);
}
#[test]
fn the_cached_tenant_comes_from_the_receipt_not_the_caller() {
let c = VerifiedCache::new(Duration::from_secs(60), 100);
let now = Instant::now();
let mut t = trace(ServedFrom::Attempt, Verdict::Pass);
t.tenant_id = "owner".to_owned();
c.offer(&t, &resp("x"), &ladder(), now);
assert!(c.get("owner", "hash-abc", &ladder(), now).is_some());
assert!(c.get("t", "hash-abc", &ladder(), now).is_none());
}
#[test]
fn a_different_ladder_is_a_different_question() {
let c = VerifiedCache::new(Duration::from_secs(60), 100);
let now = Instant::now();
c.offer(
&trace(ServedFrom::Attempt, Verdict::Pass),
&resp("a"),
&ladder(),
now,
);
let other = vec!["openai/gpt-5.5".to_owned()];
assert!(c.get("t", "hash-abc", &other, now).is_none());
assert!(c.get("t", "hash-abc", &ladder(), now).is_some());
}
#[test]
fn an_entry_expires() {
let c = VerifiedCache::new(Duration::from_secs(1), 100);
let t0 = Instant::now();
c.offer(
&trace(ServedFrom::Attempt, Verdict::Pass),
&resp("a"),
&ladder(),
t0,
);
assert!(c.get("t", "hash-abc", &ladder(), t0).is_some());
assert!(
c.get("t", "hash-abc", &ladder(), t0 + Duration::from_secs(2))
.is_none(),
"a stale proof is not a proof"
);
}
#[test]
fn the_cache_cannot_grow_without_bound() {
let c = VerifiedCache::new(Duration::from_secs(600), 10);
let now = Instant::now();
for i in 0..100 {
let mut t = trace(ServedFrom::Attempt, Verdict::Pass);
t.request.prompt_hash = format!("h{i}");
c.offer(&t, &resp("a"), &ladder(), now);
}
assert!(c.len() <= 10, "expected <= 10 entries, got {}", c.len());
}
}