1use std::num::NonZeroUsize;
4use std::sync::{Mutex, OnceLock};
5
6use lru::LruCache;
7use sha2::{Digest, Sha256};
8
9use codewhale_models::{MessageRequest, MessageResponse, Usage};
10
11const DEFAULT_CAPACITY: usize = 256;
12
13static RESPONSE_CACHE: OnceLock<ResponseCache> = OnceLock::new();
14
15pub fn response_cache() -> &'static ResponseCache {
16 RESPONSE_CACHE.get_or_init(ResponseCache::new)
17}
18
19pub fn request_is_cacheable(request: &MessageRequest) -> bool {
20 request.stream != Some(true)
21 && request.tools.as_ref().is_none_or(Vec::is_empty)
22 && request.tool_choice.is_none()
23 && request.temperature == Some(0.0)
24 && request.top_p.is_none_or(|top_p| top_p == 1.0)
25}
26
27pub struct ResponseCache {
28 inner: Mutex<LruCache<[u8; 32], MessageResponse>>,
29}
30
31impl ResponseCache {
32 fn new() -> Self {
33 Self::with_capacity(NonZeroUsize::new(DEFAULT_CAPACITY).expect("non-zero capacity"))
34 }
35
36 fn with_capacity(capacity: NonZeroUsize) -> Self {
37 Self {
38 inner: Mutex::new(LruCache::new(capacity)),
39 }
40 }
41
42 pub fn make_key(
43 provider: &str,
44 base_url: &str,
45 path_suffix: Option<&str>,
46 api_key: &str,
47 wire_body: &[u8],
48 ) -> [u8; 32] {
49 let mut hasher = Sha256::new();
50 update_field(&mut hasher, provider.as_bytes());
51 update_field(&mut hasher, base_url.as_bytes());
52 update_field(&mut hasher, path_suffix.unwrap_or("").as_bytes());
53 update_field(&mut hasher, &Sha256::digest(api_key.as_bytes()));
54 update_field(&mut hasher, wire_body);
55 hasher.finalize().into()
56 }
57
58 pub fn get(&self, key: &[u8; 32]) -> Option<MessageResponse> {
59 let mut cache = self.inner.lock().ok()?;
60 cache.get(key).cloned().map(|mut response| {
61 response.usage = Usage::default();
62 response
63 })
64 }
65
66 pub fn put(&self, key: [u8; 32], value: MessageResponse) {
67 if let Ok(mut cache) = self.inner.lock() {
68 cache.put(key, value);
69 }
70 }
71}
72
73fn update_field(hasher: &mut Sha256, bytes: &[u8]) {
74 hasher.update((bytes.len() as u64).to_le_bytes());
75 hasher.update(bytes);
76}
77
78#[cfg(test)]
79mod tests {
80 use super::*;
81
82 fn response_with_usage(id: &str) -> MessageResponse {
83 MessageResponse {
84 id: id.to_string(),
85 r#type: "message".to_string(),
86 role: "assistant".to_string(),
87 content: Vec::new(),
88 model: "test-model".to_string(),
89 stop_reason: Some("end_turn".to_string()),
90 stop_sequence: None,
91 container: None,
92 usage: Usage {
93 input_tokens: 42,
94 output_tokens: 7,
95 prompt_cache_hit_tokens: Some(3),
96 prompt_cache_miss_tokens: Some(39),
97 prompt_cache_write_tokens: None,
98 reasoning_tokens: Some(5),
99 reasoning_replay_tokens: Some(2),
100 server_tool_use: None,
101
102 input_audio_tokens: None,
103 output_audio_tokens: None,
104
105 request_usages: Vec::new(),
106 },
107 }
108 }
109
110 fn request() -> MessageRequest {
111 MessageRequest {
112 model: "test-model".to_string(),
113 messages: Vec::new(),
114 max_tokens: 16,
115 system: None,
116 tools: None,
117 tool_choice: None,
118 metadata: None,
119 thinking: None,
120 reasoning_effort: None,
121 stream: None,
122 temperature: Some(0.0),
123 top_p: None,
124 }
125 }
126
127 #[test]
128 fn cache_key_separates_provider_route_account_and_wire_body() {
129 let base = ResponseCache::make_key(
130 "deepseek",
131 "https://api.example.com/v1",
132 None,
133 "key-a",
134 br#"{"model":"m","messages":[]}"#,
135 );
136
137 assert_ne!(
138 base,
139 ResponseCache::make_key(
140 "openai",
141 "https://api.example.com/v1",
142 None,
143 "key-a",
144 br#"{"model":"m","messages":[]}"#
145 )
146 );
147 assert_ne!(
148 base,
149 ResponseCache::make_key(
150 "deepseek",
151 "https://proxy.example.com/v1",
152 None,
153 "key-a",
154 br#"{"model":"m","messages":[]}"#
155 )
156 );
157 assert_ne!(
158 base,
159 ResponseCache::make_key(
160 "deepseek",
161 "https://api.example.com/v1",
162 Some("responses"),
163 "key-a",
164 br#"{"model":"m","messages":[]}"#
165 )
166 );
167 assert_ne!(
168 base,
169 ResponseCache::make_key(
170 "deepseek",
171 "https://api.example.com/v1",
172 None,
173 "key-b",
174 br#"{"model":"m","messages":[]}"#
175 )
176 );
177 assert_ne!(
178 base,
179 ResponseCache::make_key(
180 "deepseek",
181 "https://api.example.com/v1",
182 None,
183 "key-a",
184 br#"{"model":"m","messages":[],"reasoning_effort":"high"}"#
185 )
186 );
187 }
188
189 #[test]
190 fn cache_hit_zeroes_usage_to_avoid_fake_spend() {
191 let cache = ResponseCache::with_capacity(NonZeroUsize::new(2).unwrap());
192 let key =
193 ResponseCache::make_key("deepseek", "https://api.example.com", None, "key", b"{}");
194
195 cache.put(key, response_with_usage("cached"));
196
197 let hit = cache.get(&key).expect("cache hit");
198 assert_eq!(hit.id, "cached");
199 assert_eq!(hit.usage, Usage::default());
200 }
201
202 #[test]
203 fn capacity_evicts_oldest_entry() {
204 let cache = ResponseCache::with_capacity(NonZeroUsize::new(2).unwrap());
205 let key1 =
206 ResponseCache::make_key("deepseek", "https://api.example.com", None, "key", b"one");
207 let key2 =
208 ResponseCache::make_key("deepseek", "https://api.example.com", None, "key", b"two");
209 let key3 =
210 ResponseCache::make_key("deepseek", "https://api.example.com", None, "key", b"three");
211
212 cache.put(key1, response_with_usage("one"));
213 cache.put(key2, response_with_usage("two"));
214 cache.put(key3, response_with_usage("three"));
215
216 assert!(cache.get(&key1).is_none());
217 assert!(cache.get(&key2).is_some());
218 assert!(cache.get(&key3).is_some());
219 }
220
221 #[test]
222 fn cacheability_requires_deterministic_tool_free_non_streaming_request() {
223 let mut req = request();
224 assert!(request_is_cacheable(&req));
225
226 req.temperature = None;
227 assert!(!request_is_cacheable(&req));
228
229 req = request();
230 req.temperature = Some(0.2);
231 assert!(!request_is_cacheable(&req));
232
233 req = request();
234 req.stream = Some(true);
235 assert!(!request_is_cacheable(&req));
236
237 req = request();
238 req.top_p = Some(0.5);
239 assert!(!request_is_cacheable(&req));
240
241 req = request();
242 req.tool_choice = Some(serde_json::json!("auto"));
243 assert!(!request_is_cacheable(&req));
244 }
245}