Skip to main content

codewhale_runtime/
llm_response_cache.rs

1//! Small in-process cache for deterministic non-streaming chat responses.
2
3use std::num::NonZeroUsize;
4use std::sync::{Mutex, OnceLock};
5
6use lru::LruCache;
7use sha2::{Digest, Sha256};
8
9use codewhale_models::{MessageRequest, MessageResponse, Usage};
10
11const DEFAULT_CAPACITY: usize = 256;
12
13static RESPONSE_CACHE: OnceLock<ResponseCache> = OnceLock::new();
14
15pub fn response_cache() -> &'static ResponseCache {
16    RESPONSE_CACHE.get_or_init(ResponseCache::new)
17}
18
19pub fn request_is_cacheable(request: &MessageRequest) -> bool {
20    request.stream != Some(true)
21        && request.tools.as_ref().is_none_or(Vec::is_empty)
22        && request.tool_choice.is_none()
23        && request.temperature == Some(0.0)
24        && request.top_p.is_none_or(|top_p| top_p == 1.0)
25}
26
27pub struct ResponseCache {
28    inner: Mutex<LruCache<[u8; 32], MessageResponse>>,
29}
30
31impl ResponseCache {
32    fn new() -> Self {
33        Self::with_capacity(NonZeroUsize::new(DEFAULT_CAPACITY).expect("non-zero capacity"))
34    }
35
36    fn with_capacity(capacity: NonZeroUsize) -> Self {
37        Self {
38            inner: Mutex::new(LruCache::new(capacity)),
39        }
40    }
41
42    pub fn make_key(
43        provider: &str,
44        base_url: &str,
45        path_suffix: Option<&str>,
46        api_key: &str,
47        wire_body: &[u8],
48    ) -> [u8; 32] {
49        let mut hasher = Sha256::new();
50        update_field(&mut hasher, provider.as_bytes());
51        update_field(&mut hasher, base_url.as_bytes());
52        update_field(&mut hasher, path_suffix.unwrap_or("").as_bytes());
53        update_field(&mut hasher, &Sha256::digest(api_key.as_bytes()));
54        update_field(&mut hasher, wire_body);
55        hasher.finalize().into()
56    }
57
58    pub fn get(&self, key: &[u8; 32]) -> Option<MessageResponse> {
59        let mut cache = self.inner.lock().ok()?;
60        cache.get(key).cloned().map(|mut response| {
61            response.usage = Usage::default();
62            response
63        })
64    }
65
66    pub fn put(&self, key: [u8; 32], value: MessageResponse) {
67        if let Ok(mut cache) = self.inner.lock() {
68            cache.put(key, value);
69        }
70    }
71}
72
73fn update_field(hasher: &mut Sha256, bytes: &[u8]) {
74    hasher.update((bytes.len() as u64).to_le_bytes());
75    hasher.update(bytes);
76}
77
78#[cfg(test)]
79mod tests {
80    use super::*;
81
82    fn response_with_usage(id: &str) -> MessageResponse {
83        MessageResponse {
84            id: id.to_string(),
85            r#type: "message".to_string(),
86            role: "assistant".to_string(),
87            content: Vec::new(),
88            model: "test-model".to_string(),
89            stop_reason: Some("end_turn".to_string()),
90            stop_sequence: None,
91            container: None,
92            usage: Usage {
93                input_tokens: 42,
94                output_tokens: 7,
95                prompt_cache_hit_tokens: Some(3),
96                prompt_cache_miss_tokens: Some(39),
97                prompt_cache_write_tokens: None,
98                reasoning_tokens: Some(5),
99                reasoning_replay_tokens: Some(2),
100                server_tool_use: None,
101            },
102        }
103    }
104
105    fn request() -> MessageRequest {
106        MessageRequest {
107            model: "test-model".to_string(),
108            messages: Vec::new(),
109            max_tokens: 16,
110            system: None,
111            tools: None,
112            tool_choice: None,
113            metadata: None,
114            thinking: None,
115            reasoning_effort: None,
116            stream: None,
117            temperature: Some(0.0),
118            top_p: None,
119        }
120    }
121
122    #[test]
123    fn cache_key_separates_provider_route_account_and_wire_body() {
124        let base = ResponseCache::make_key(
125            "deepseek",
126            "https://api.example.com/v1",
127            None,
128            "key-a",
129            br#"{"model":"m","messages":[]}"#,
130        );
131
132        assert_ne!(
133            base,
134            ResponseCache::make_key(
135                "openai",
136                "https://api.example.com/v1",
137                None,
138                "key-a",
139                br#"{"model":"m","messages":[]}"#
140            )
141        );
142        assert_ne!(
143            base,
144            ResponseCache::make_key(
145                "deepseek",
146                "https://proxy.example.com/v1",
147                None,
148                "key-a",
149                br#"{"model":"m","messages":[]}"#
150            )
151        );
152        assert_ne!(
153            base,
154            ResponseCache::make_key(
155                "deepseek",
156                "https://api.example.com/v1",
157                Some("responses"),
158                "key-a",
159                br#"{"model":"m","messages":[]}"#
160            )
161        );
162        assert_ne!(
163            base,
164            ResponseCache::make_key(
165                "deepseek",
166                "https://api.example.com/v1",
167                None,
168                "key-b",
169                br#"{"model":"m","messages":[]}"#
170            )
171        );
172        assert_ne!(
173            base,
174            ResponseCache::make_key(
175                "deepseek",
176                "https://api.example.com/v1",
177                None,
178                "key-a",
179                br#"{"model":"m","messages":[],"reasoning_effort":"high"}"#
180            )
181        );
182    }
183
184    #[test]
185    fn cache_hit_zeroes_usage_to_avoid_fake_spend() {
186        let cache = ResponseCache::with_capacity(NonZeroUsize::new(2).unwrap());
187        let key =
188            ResponseCache::make_key("deepseek", "https://api.example.com", None, "key", b"{}");
189
190        cache.put(key, response_with_usage("cached"));
191
192        let hit = cache.get(&key).expect("cache hit");
193        assert_eq!(hit.id, "cached");
194        assert_eq!(hit.usage, Usage::default());
195    }
196
197    #[test]
198    fn capacity_evicts_oldest_entry() {
199        let cache = ResponseCache::with_capacity(NonZeroUsize::new(2).unwrap());
200        let key1 =
201            ResponseCache::make_key("deepseek", "https://api.example.com", None, "key", b"one");
202        let key2 =
203            ResponseCache::make_key("deepseek", "https://api.example.com", None, "key", b"two");
204        let key3 =
205            ResponseCache::make_key("deepseek", "https://api.example.com", None, "key", b"three");
206
207        cache.put(key1, response_with_usage("one"));
208        cache.put(key2, response_with_usage("two"));
209        cache.put(key3, response_with_usage("three"));
210
211        assert!(cache.get(&key1).is_none());
212        assert!(cache.get(&key2).is_some());
213        assert!(cache.get(&key3).is_some());
214    }
215
216    #[test]
217    fn cacheability_requires_deterministic_tool_free_non_streaming_request() {
218        let mut req = request();
219        assert!(request_is_cacheable(&req));
220
221        req.temperature = None;
222        assert!(!request_is_cacheable(&req));
223
224        req = request();
225        req.temperature = Some(0.2);
226        assert!(!request_is_cacheable(&req));
227
228        req = request();
229        req.stream = Some(true);
230        assert!(!request_is_cacheable(&req));
231
232        req = request();
233        req.top_p = Some(0.5);
234        assert!(!request_is_cacheable(&req));
235
236        req = request();
237        req.tool_choice = Some(serde_json::json!("auto"));
238        assert!(!request_is_cacheable(&req));
239    }
240}