1use std::num::NonZeroUsize;
4use std::sync::{Mutex, OnceLock};
5
6use lru::LruCache;
7use sha2::{Digest, Sha256};
8
9use codewhale_models::{MessageRequest, MessageResponse, Usage};
10
11const DEFAULT_CAPACITY: usize = 256;
12
13static RESPONSE_CACHE: OnceLock<ResponseCache> = OnceLock::new();
14
15pub fn response_cache() -> &'static ResponseCache {
16 RESPONSE_CACHE.get_or_init(ResponseCache::new)
17}
18
19pub fn request_is_cacheable(request: &MessageRequest) -> bool {
20 request.stream != Some(true)
21 && request.tools.as_ref().is_none_or(Vec::is_empty)
22 && request.tool_choice.is_none()
23 && request.temperature == Some(0.0)
24 && request.top_p.is_none_or(|top_p| top_p == 1.0)
25}
26
27pub struct ResponseCache {
28 inner: Mutex<LruCache<[u8; 32], MessageResponse>>,
29}
30
31impl ResponseCache {
32 fn new() -> Self {
33 Self::with_capacity(NonZeroUsize::new(DEFAULT_CAPACITY).expect("non-zero capacity"))
34 }
35
36 fn with_capacity(capacity: NonZeroUsize) -> Self {
37 Self {
38 inner: Mutex::new(LruCache::new(capacity)),
39 }
40 }
41
42 pub fn make_key(
43 provider: &str,
44 base_url: &str,
45 path_suffix: Option<&str>,
46 api_key: &str,
47 wire_body: &[u8],
48 ) -> [u8; 32] {
49 let mut hasher = Sha256::new();
50 update_field(&mut hasher, provider.as_bytes());
51 update_field(&mut hasher, base_url.as_bytes());
52 update_field(&mut hasher, path_suffix.unwrap_or("").as_bytes());
53 update_field(&mut hasher, &Sha256::digest(api_key.as_bytes()));
54 update_field(&mut hasher, wire_body);
55 hasher.finalize().into()
56 }
57
58 pub fn get(&self, key: &[u8; 32]) -> Option<MessageResponse> {
59 let mut cache = self.inner.lock().ok()?;
60 cache.get(key).cloned().map(|mut response| {
61 response.usage = Usage::default();
62 response
63 })
64 }
65
66 pub fn put(&self, key: [u8; 32], value: MessageResponse) {
67 if let Ok(mut cache) = self.inner.lock() {
68 cache.put(key, value);
69 }
70 }
71}
72
73fn update_field(hasher: &mut Sha256, bytes: &[u8]) {
74 hasher.update((bytes.len() as u64).to_le_bytes());
75 hasher.update(bytes);
76}
77
78#[cfg(test)]
79mod tests {
80 use super::*;
81
82 fn response_with_usage(id: &str) -> MessageResponse {
83 MessageResponse {
84 id: id.to_string(),
85 r#type: "message".to_string(),
86 role: "assistant".to_string(),
87 content: Vec::new(),
88 model: "test-model".to_string(),
89 stop_reason: Some("end_turn".to_string()),
90 stop_sequence: None,
91 container: None,
92 title: None,
93 stop_details: None,
94 refusal: None,
95 usage: Usage {
96 input_tokens: 42,
97 output_tokens: 7,
98 prompt_cache_hit_tokens: Some(3),
99 prompt_cache_miss_tokens: Some(39),
100 prompt_cache_write_tokens: None,
101 reasoning_tokens: Some(5),
102 reasoning_replay_tokens: Some(2),
103 server_tool_use: None,
104
105 input_audio_tokens: None,
106 output_audio_tokens: None,
107
108 request_usages: Vec::new(),
109 safety_tokens: None,
110 },
111 }
112 }
113
114 fn request() -> MessageRequest {
115 MessageRequest {
116 model: "test-model".to_string(),
117 messages: Vec::new(),
118 max_tokens: 16,
119 system: None,
120 tools: None,
121 tool_choice: None,
122 metadata: None,
123 thinking: None,
124 reasoning_effort: None,
125 stream: None,
126 temperature: Some(0.0),
127 top_p: None,
128 safety_check: None,
129 title: None,
130 }
131 }
132
133 #[test]
134 fn cache_key_separates_provider_route_account_and_wire_body() {
135 let base = ResponseCache::make_key(
136 "deepseek",
137 "https://api.example.com/v1",
138 None,
139 "key-a",
140 br#"{"model":"m","messages":[]}"#,
141 );
142
143 assert_ne!(
144 base,
145 ResponseCache::make_key(
146 "openai",
147 "https://api.example.com/v1",
148 None,
149 "key-a",
150 br#"{"model":"m","messages":[]}"#
151 )
152 );
153 assert_ne!(
154 base,
155 ResponseCache::make_key(
156 "deepseek",
157 "https://proxy.example.com/v1",
158 None,
159 "key-a",
160 br#"{"model":"m","messages":[]}"#
161 )
162 );
163 assert_ne!(
164 base,
165 ResponseCache::make_key(
166 "deepseek",
167 "https://api.example.com/v1",
168 Some("responses"),
169 "key-a",
170 br#"{"model":"m","messages":[]}"#
171 )
172 );
173 assert_ne!(
174 base,
175 ResponseCache::make_key(
176 "deepseek",
177 "https://api.example.com/v1",
178 None,
179 "key-b",
180 br#"{"model":"m","messages":[]}"#
181 )
182 );
183 assert_ne!(
184 base,
185 ResponseCache::make_key(
186 "deepseek",
187 "https://api.example.com/v1",
188 None,
189 "key-a",
190 br#"{"model":"m","messages":[],"reasoning_effort":"high"}"#
191 )
192 );
193 }
194
195 #[test]
196 fn cache_hit_zeroes_usage_to_avoid_fake_spend() {
197 let cache = ResponseCache::with_capacity(NonZeroUsize::new(2).unwrap());
198 let key =
199 ResponseCache::make_key("deepseek", "https://api.example.com", None, "key", b"{}");
200
201 cache.put(key, response_with_usage("cached"));
202
203 let hit = cache.get(&key).expect("cache hit");
204 assert_eq!(hit.id, "cached");
205 assert_eq!(hit.usage, Usage::default());
206 }
207
208 #[test]
209 fn capacity_evicts_oldest_entry() {
210 let cache = ResponseCache::with_capacity(NonZeroUsize::new(2).unwrap());
211 let key1 =
212 ResponseCache::make_key("deepseek", "https://api.example.com", None, "key", b"one");
213 let key2 =
214 ResponseCache::make_key("deepseek", "https://api.example.com", None, "key", b"two");
215 let key3 =
216 ResponseCache::make_key("deepseek", "https://api.example.com", None, "key", b"three");
217
218 cache.put(key1, response_with_usage("one"));
219 cache.put(key2, response_with_usage("two"));
220 cache.put(key3, response_with_usage("three"));
221
222 assert!(cache.get(&key1).is_none());
223 assert!(cache.get(&key2).is_some());
224 assert!(cache.get(&key3).is_some());
225 }
226
227 #[test]
228 fn cacheability_requires_deterministic_tool_free_non_streaming_request() {
229 let mut req = request();
230 assert!(request_is_cacheable(&req));
231
232 req.temperature = None;
233 assert!(!request_is_cacheable(&req));
234
235 req = request();
236 req.temperature = Some(0.2);
237 assert!(!request_is_cacheable(&req));
238
239 req = request();
240 req.stream = Some(true);
241 assert!(!request_is_cacheable(&req));
242
243 req = request();
244 req.top_p = Some(0.5);
245 assert!(!request_is_cacheable(&req));
246
247 req = request();
248 req.tool_choice = Some(serde_json::json!("auto"));
249 assert!(!request_is_cacheable(&req));
250 }
251}