Skip to main content

codewhale_runtime/
llm_response_cache.rs

1//! Small in-process cache for deterministic non-streaming chat responses.
2
3use std::num::NonZeroUsize;
4use std::sync::{Mutex, OnceLock};
5
6use lru::LruCache;
7use sha2::{Digest, Sha256};
8
9use codewhale_models::{MessageRequest, MessageResponse, Usage};
10
11const DEFAULT_CAPACITY: usize = 256;
12
13static RESPONSE_CACHE: OnceLock<ResponseCache> = OnceLock::new();
14
15pub fn response_cache() -> &'static ResponseCache {
16    RESPONSE_CACHE.get_or_init(ResponseCache::new)
17}
18
19pub fn request_is_cacheable(request: &MessageRequest) -> bool {
20    request.stream != Some(true)
21        && request.tools.as_ref().is_none_or(Vec::is_empty)
22        && request.tool_choice.is_none()
23        && request.temperature == Some(0.0)
24        && request.top_p.is_none_or(|top_p| top_p == 1.0)
25}
26
27pub struct ResponseCache {
28    inner: Mutex<LruCache<[u8; 32], MessageResponse>>,
29}
30
31impl ResponseCache {
32    fn new() -> Self {
33        Self::with_capacity(NonZeroUsize::new(DEFAULT_CAPACITY).expect("non-zero capacity"))
34    }
35
36    fn with_capacity(capacity: NonZeroUsize) -> Self {
37        Self {
38            inner: Mutex::new(LruCache::new(capacity)),
39        }
40    }
41
42    pub fn make_key(
43        provider: &str,
44        base_url: &str,
45        path_suffix: Option<&str>,
46        api_key: &str,
47        wire_body: &[u8],
48    ) -> [u8; 32] {
49        let mut hasher = Sha256::new();
50        update_field(&mut hasher, provider.as_bytes());
51        update_field(&mut hasher, base_url.as_bytes());
52        update_field(&mut hasher, path_suffix.unwrap_or("").as_bytes());
53        update_field(&mut hasher, &Sha256::digest(api_key.as_bytes()));
54        update_field(&mut hasher, wire_body);
55        hasher.finalize().into()
56    }
57
58    pub fn get(&self, key: &[u8; 32]) -> Option<MessageResponse> {
59        let mut cache = self.inner.lock().ok()?;
60        cache.get(key).cloned().map(|mut response| {
61            response.usage = Usage::default();
62            response
63        })
64    }
65
66    pub fn put(&self, key: [u8; 32], value: MessageResponse) {
67        if let Ok(mut cache) = self.inner.lock() {
68            cache.put(key, value);
69        }
70    }
71}
72
73fn update_field(hasher: &mut Sha256, bytes: &[u8]) {
74    hasher.update((bytes.len() as u64).to_le_bytes());
75    hasher.update(bytes);
76}
77
78#[cfg(test)]
79mod tests {
80    use super::*;
81
82    fn response_with_usage(id: &str) -> MessageResponse {
83        MessageResponse {
84            id: id.to_string(),
85            r#type: "message".to_string(),
86            role: "assistant".to_string(),
87            content: Vec::new(),
88            model: "test-model".to_string(),
89            stop_reason: Some("end_turn".to_string()),
90            stop_sequence: None,
91            container: None,
92            usage: Usage {
93                input_tokens: 42,
94                output_tokens: 7,
95                prompt_cache_hit_tokens: Some(3),
96                prompt_cache_miss_tokens: Some(39),
97                prompt_cache_write_tokens: None,
98                reasoning_tokens: Some(5),
99                reasoning_replay_tokens: Some(2),
100                server_tool_use: None,
101
102                input_audio_tokens: None,
103                output_audio_tokens: None,
104
105                request_usages: Vec::new(),
106            },
107        }
108    }
109
110    fn request() -> MessageRequest {
111        MessageRequest {
112            model: "test-model".to_string(),
113            messages: Vec::new(),
114            max_tokens: 16,
115            system: None,
116            tools: None,
117            tool_choice: None,
118            metadata: None,
119            thinking: None,
120            reasoning_effort: None,
121            stream: None,
122            temperature: Some(0.0),
123            top_p: None,
124        }
125    }
126
127    #[test]
128    fn cache_key_separates_provider_route_account_and_wire_body() {
129        let base = ResponseCache::make_key(
130            "deepseek",
131            "https://api.example.com/v1",
132            None,
133            "key-a",
134            br#"{"model":"m","messages":[]}"#,
135        );
136
137        assert_ne!(
138            base,
139            ResponseCache::make_key(
140                "openai",
141                "https://api.example.com/v1",
142                None,
143                "key-a",
144                br#"{"model":"m","messages":[]}"#
145            )
146        );
147        assert_ne!(
148            base,
149            ResponseCache::make_key(
150                "deepseek",
151                "https://proxy.example.com/v1",
152                None,
153                "key-a",
154                br#"{"model":"m","messages":[]}"#
155            )
156        );
157        assert_ne!(
158            base,
159            ResponseCache::make_key(
160                "deepseek",
161                "https://api.example.com/v1",
162                Some("responses"),
163                "key-a",
164                br#"{"model":"m","messages":[]}"#
165            )
166        );
167        assert_ne!(
168            base,
169            ResponseCache::make_key(
170                "deepseek",
171                "https://api.example.com/v1",
172                None,
173                "key-b",
174                br#"{"model":"m","messages":[]}"#
175            )
176        );
177        assert_ne!(
178            base,
179            ResponseCache::make_key(
180                "deepseek",
181                "https://api.example.com/v1",
182                None,
183                "key-a",
184                br#"{"model":"m","messages":[],"reasoning_effort":"high"}"#
185            )
186        );
187    }
188
189    #[test]
190    fn cache_hit_zeroes_usage_to_avoid_fake_spend() {
191        let cache = ResponseCache::with_capacity(NonZeroUsize::new(2).unwrap());
192        let key =
193            ResponseCache::make_key("deepseek", "https://api.example.com", None, "key", b"{}");
194
195        cache.put(key, response_with_usage("cached"));
196
197        let hit = cache.get(&key).expect("cache hit");
198        assert_eq!(hit.id, "cached");
199        assert_eq!(hit.usage, Usage::default());
200    }
201
202    #[test]
203    fn capacity_evicts_oldest_entry() {
204        let cache = ResponseCache::with_capacity(NonZeroUsize::new(2).unwrap());
205        let key1 =
206            ResponseCache::make_key("deepseek", "https://api.example.com", None, "key", b"one");
207        let key2 =
208            ResponseCache::make_key("deepseek", "https://api.example.com", None, "key", b"two");
209        let key3 =
210            ResponseCache::make_key("deepseek", "https://api.example.com", None, "key", b"three");
211
212        cache.put(key1, response_with_usage("one"));
213        cache.put(key2, response_with_usage("two"));
214        cache.put(key3, response_with_usage("three"));
215
216        assert!(cache.get(&key1).is_none());
217        assert!(cache.get(&key2).is_some());
218        assert!(cache.get(&key3).is_some());
219    }
220
221    #[test]
222    fn cacheability_requires_deterministic_tool_free_non_streaming_request() {
223        let mut req = request();
224        assert!(request_is_cacheable(&req));
225
226        req.temperature = None;
227        assert!(!request_is_cacheable(&req));
228
229        req = request();
230        req.temperature = Some(0.2);
231        assert!(!request_is_cacheable(&req));
232
233        req = request();
234        req.stream = Some(true);
235        assert!(!request_is_cacheable(&req));
236
237        req = request();
238        req.top_p = Some(0.5);
239        assert!(!request_is_cacheable(&req));
240
241        req = request();
242        req.tool_choice = Some(serde_json::json!("auto"));
243        assert!(!request_is_cacheable(&req));
244    }
245}