Skip to main content

codewhale_runtime/
llm_response_cache.rs

1//! Small in-process cache for deterministic non-streaming chat responses.
2
3use std::num::NonZeroUsize;
4use std::sync::{Mutex, OnceLock};
5
6use lru::LruCache;
7use sha2::{Digest, Sha256};
8
9use codewhale_models::{MessageRequest, MessageResponse, Usage};
10
11const DEFAULT_CAPACITY: usize = 256;
12
13static RESPONSE_CACHE: OnceLock<ResponseCache> = OnceLock::new();
14
15pub fn response_cache() -> &'static ResponseCache {
16    RESPONSE_CACHE.get_or_init(ResponseCache::new)
17}
18
19pub fn request_is_cacheable(request: &MessageRequest) -> bool {
20    request.stream != Some(true)
21        && request.tools.as_ref().is_none_or(Vec::is_empty)
22        && request.tool_choice.is_none()
23        && request.temperature == Some(0.0)
24        && request.top_p.is_none_or(|top_p| top_p == 1.0)
25}
26
27pub struct ResponseCache {
28    inner: Mutex<LruCache<[u8; 32], MessageResponse>>,
29}
30
31impl ResponseCache {
32    fn new() -> Self {
33        Self::with_capacity(NonZeroUsize::new(DEFAULT_CAPACITY).expect("non-zero capacity"))
34    }
35
36    fn with_capacity(capacity: NonZeroUsize) -> Self {
37        Self {
38            inner: Mutex::new(LruCache::new(capacity)),
39        }
40    }
41
42    pub fn make_key(
43        provider: &str,
44        base_url: &str,
45        path_suffix: Option<&str>,
46        api_key: &str,
47        wire_body: &[u8],
48    ) -> [u8; 32] {
49        let mut hasher = Sha256::new();
50        update_field(&mut hasher, provider.as_bytes());
51        update_field(&mut hasher, base_url.as_bytes());
52        update_field(&mut hasher, path_suffix.unwrap_or("").as_bytes());
53        update_field(&mut hasher, &Sha256::digest(api_key.as_bytes()));
54        update_field(&mut hasher, wire_body);
55        hasher.finalize().into()
56    }
57
58    pub fn get(&self, key: &[u8; 32]) -> Option<MessageResponse> {
59        let mut cache = self.inner.lock().ok()?;
60        cache.get(key).cloned().map(|mut response| {
61            response.usage = Usage::default();
62            response
63        })
64    }
65
66    pub fn put(&self, key: [u8; 32], value: MessageResponse) {
67        if let Ok(mut cache) = self.inner.lock() {
68            cache.put(key, value);
69        }
70    }
71}
72
73fn update_field(hasher: &mut Sha256, bytes: &[u8]) {
74    hasher.update((bytes.len() as u64).to_le_bytes());
75    hasher.update(bytes);
76}
77
78#[cfg(test)]
79mod tests {
80    use super::*;
81
82    fn response_with_usage(id: &str) -> MessageResponse {
83        MessageResponse {
84            id: id.to_string(),
85            r#type: "message".to_string(),
86            role: "assistant".to_string(),
87            content: Vec::new(),
88            model: "test-model".to_string(),
89            stop_reason: Some("end_turn".to_string()),
90            stop_sequence: None,
91            container: None,
92            title: None,
93            stop_details: None,
94            refusal: None,
95            usage: Usage {
96                input_tokens: 42,
97                output_tokens: 7,
98                prompt_cache_hit_tokens: Some(3),
99                prompt_cache_miss_tokens: Some(39),
100                prompt_cache_write_tokens: None,
101                reasoning_tokens: Some(5),
102                reasoning_replay_tokens: Some(2),
103                server_tool_use: None,
104
105                input_audio_tokens: None,
106                output_audio_tokens: None,
107
108                request_usages: Vec::new(),
109                safety_tokens: None,
110            },
111        }
112    }
113
114    fn request() -> MessageRequest {
115        MessageRequest {
116            model: "test-model".to_string(),
117            messages: Vec::new(),
118            max_tokens: 16,
119            system: None,
120            tools: None,
121            tool_choice: None,
122            metadata: None,
123            thinking: None,
124            reasoning_effort: None,
125            stream: None,
126            temperature: Some(0.0),
127            top_p: None,
128            safety_check: None,
129            title: None,
130        }
131    }
132
133    #[test]
134    fn cache_key_separates_provider_route_account_and_wire_body() {
135        let base = ResponseCache::make_key(
136            "deepseek",
137            "https://api.example.com/v1",
138            None,
139            "key-a",
140            br#"{"model":"m","messages":[]}"#,
141        );
142
143        assert_ne!(
144            base,
145            ResponseCache::make_key(
146                "openai",
147                "https://api.example.com/v1",
148                None,
149                "key-a",
150                br#"{"model":"m","messages":[]}"#
151            )
152        );
153        assert_ne!(
154            base,
155            ResponseCache::make_key(
156                "deepseek",
157                "https://proxy.example.com/v1",
158                None,
159                "key-a",
160                br#"{"model":"m","messages":[]}"#
161            )
162        );
163        assert_ne!(
164            base,
165            ResponseCache::make_key(
166                "deepseek",
167                "https://api.example.com/v1",
168                Some("responses"),
169                "key-a",
170                br#"{"model":"m","messages":[]}"#
171            )
172        );
173        assert_ne!(
174            base,
175            ResponseCache::make_key(
176                "deepseek",
177                "https://api.example.com/v1",
178                None,
179                "key-b",
180                br#"{"model":"m","messages":[]}"#
181            )
182        );
183        assert_ne!(
184            base,
185            ResponseCache::make_key(
186                "deepseek",
187                "https://api.example.com/v1",
188                None,
189                "key-a",
190                br#"{"model":"m","messages":[],"reasoning_effort":"high"}"#
191            )
192        );
193    }
194
195    #[test]
196    fn cache_hit_zeroes_usage_to_avoid_fake_spend() {
197        let cache = ResponseCache::with_capacity(NonZeroUsize::new(2).unwrap());
198        let key =
199            ResponseCache::make_key("deepseek", "https://api.example.com", None, "key", b"{}");
200
201        cache.put(key, response_with_usage("cached"));
202
203        let hit = cache.get(&key).expect("cache hit");
204        assert_eq!(hit.id, "cached");
205        assert_eq!(hit.usage, Usage::default());
206    }
207
208    #[test]
209    fn capacity_evicts_oldest_entry() {
210        let cache = ResponseCache::with_capacity(NonZeroUsize::new(2).unwrap());
211        let key1 =
212            ResponseCache::make_key("deepseek", "https://api.example.com", None, "key", b"one");
213        let key2 =
214            ResponseCache::make_key("deepseek", "https://api.example.com", None, "key", b"two");
215        let key3 =
216            ResponseCache::make_key("deepseek", "https://api.example.com", None, "key", b"three");
217
218        cache.put(key1, response_with_usage("one"));
219        cache.put(key2, response_with_usage("two"));
220        cache.put(key3, response_with_usage("three"));
221
222        assert!(cache.get(&key1).is_none());
223        assert!(cache.get(&key2).is_some());
224        assert!(cache.get(&key3).is_some());
225    }
226
227    #[test]
228    fn cacheability_requires_deterministic_tool_free_non_streaming_request() {
229        let mut req = request();
230        assert!(request_is_cacheable(&req));
231
232        req.temperature = None;
233        assert!(!request_is_cacheable(&req));
234
235        req = request();
236        req.temperature = Some(0.2);
237        assert!(!request_is_cacheable(&req));
238
239        req = request();
240        req.stream = Some(true);
241        assert!(!request_is_cacheable(&req));
242
243        req = request();
244        req.top_p = Some(0.5);
245        assert!(!request_is_cacheable(&req));
246
247        req = request();
248        req.tool_choice = Some(serde_json::json!("auto"));
249        assert!(!request_is_cacheable(&req));
250    }
251}