tonin_client/
cache.rs

1//! Optional per-method short-TTL response cache.
2//!
3//! Layered on top of [`crate::coalesce`]: the lookup order is
4//! `cache hit → (miss) coalescing → upstream → populate cache`.
5//!
6//! Only `Ok` responses are cached. Errors are never stored; the next
7//! call always retries upstream.
8//!
9//! ## Implementation
10//!
11//! Uses a `DashMap<Key, CachedEntry<R>>` with **lazy eviction**: stale
12//! entries are removed on read, not by a background sweep. This keeps
13//! the dep tree minimal (no extra async tasks, no `moka`). The tradeoff
14//! is that evicted entries linger in memory until the key is next read —
15//! acceptable for the expected small working sets (hot RPC methods, not
16//! unbounded user data). A hard `capacity` cap is enforced at insert time
17//! via a simple `if len() >= capacity { return }` guard; under capacity
18//! pressure the cache degrades gracefully to a pass-through.
19//!
20//! ## Configuration
21//!
22//! Built from [`CacheConfig`]; see [`ResponseCache::new`].
23
24use std::hash::{Hash, Hasher};
25use std::time::{Duration, Instant};
26
27use dashmap::DashMap;
28
29/// Configuration for the per-method response cache.
30#[derive(Clone, Debug)]
31pub struct CacheConfig {
32    /// How long an `Ok` response is considered fresh.
33    pub ttl: Duration,
34    /// Maximum number of entries. When the map reaches this size, new
35    /// entries are silently dropped (cache degrades to pass-through).
36    /// Prevents unbounded growth under key-space explosion.
37    pub capacity: usize,
38}
39
40impl CacheConfig {
41    pub fn new(ttl: Duration, capacity: usize) -> Self {
42        Self { ttl, capacity }
43    }
44
45    /// 500 ms TTL, 1 000 entries — a sensible default for a hot authz RPC.
46    pub fn default_authz() -> Self {
47        Self::new(Duration::from_millis(500), 1_000)
48    }
49}
50
51// Internal entry stored in the map.
52#[derive(Clone)]
53struct CachedEntry<R> {
54    value: R,
55    expires_at: Instant,
56}
57
58// Newtype so Vec<u8> is usable as a DashMap key.
59#[derive(Clone, PartialEq, Eq)]
60struct CacheKey(Vec<u8>);
61
62impl Hash for CacheKey {
63    fn hash<H: Hasher>(&self, state: &mut H) {
64        self.0.hash(state);
65    }
66}
67
68/// A bounded, TTL-evicting response cache for a single RPC method.
69pub struct ResponseCache<R> {
70    map: DashMap<CacheKey, CachedEntry<R>>,
71    config: CacheConfig,
72}
73
74impl<R: Clone + Send + 'static> ResponseCache<R> {
75    pub fn new(config: CacheConfig) -> Self {
76        Self {
77            map: DashMap::new(),
78            config,
79        }
80    }
81
82    /// Look up a cached response by key. Returns `None` if absent or
83    /// expired (and removes the stale entry in that case).
84    pub fn get(&self, key: &[u8]) -> Option<R> {
85        let k = CacheKey(key.to_vec());
86        // Use entry API to atomically check-and-remove stale entries.
87        if let Some(entry) = self.map.get(&k)
88            && entry.expires_at > Instant::now()
89        {
90            return Some(entry.value.clone());
91        }
92        // Expired — remove and return None.
93        self.map.remove(&k);
94        None
95    }
96
97    /// Store an `Ok` response. Silently drops the insert when the cache
98    /// is at capacity (pass-through degradation).
99    pub fn insert(&self, key: &[u8], value: R) {
100        if self.map.len() >= self.config.capacity {
101            return;
102        }
103        let k = CacheKey(key.to_vec());
104        self.map.insert(
105            k,
106            CachedEntry {
107                value,
108                expires_at: Instant::now() + self.config.ttl,
109            },
110        );
111    }
112
113    /// Remove all entries. Useful in tests.
114    #[cfg(test)]
115    pub fn clear(&self) {
116        self.map.clear();
117    }
118}
119
120impl<R: Clone + Send + 'static> Clone for ResponseCache<R> {
121    fn clone(&self) -> Self {
122        Self {
123            map: self.map.clone(),
124            config: self.config.clone(),
125        }
126    }
127}
128
129#[cfg(test)]
130mod tests {
131    use super::*;
132    use std::time::Duration;
133
134    #[test]
135    fn hit_within_ttl() {
136        let cache: ResponseCache<String> =
137            ResponseCache::new(CacheConfig::new(Duration::from_secs(10), 100));
138        cache.insert(b"key", "value".into());
139        assert_eq!(cache.get(b"key").as_deref(), Some("value"));
140    }
141
142    #[test]
143    fn miss_after_expiry() {
144        let cache: ResponseCache<String> =
145            ResponseCache::new(CacheConfig::new(Duration::from_millis(1), 100));
146        cache.insert(b"key", "value".into());
147        std::thread::sleep(Duration::from_millis(5));
148        assert_eq!(cache.get(b"key"), None);
149    }
150
151    #[test]
152    fn error_is_not_cached() {
153        // Errors are never inserted; the caller is responsible for skipping
154        // insert on Err. This test just verifies a None get returns None.
155        let cache: ResponseCache<String> =
156            ResponseCache::new(CacheConfig::new(Duration::from_secs(10), 100));
157        assert_eq!(cache.get(b"key"), None);
158    }
159
160    #[test]
161    fn capacity_cap_drops_new_inserts() {
162        let cache: ResponseCache<u32> =
163            ResponseCache::new(CacheConfig::new(Duration::from_secs(10), 2));
164        cache.insert(b"a", 1);
165        cache.insert(b"b", 2);
166        cache.insert(b"c", 3); // dropped — at capacity
167        assert!(cache.get(b"a").is_some());
168        assert!(cache.get(b"b").is_some());
169        assert_eq!(cache.get(b"c"), None);
170    }
171}
tonin_client/cache.rs

tonin_client/
cache.rs
