1use futures::StreamExt;
7use serde::Deserialize;
8
9use crate::driver::{Exchange, Model, Opened, Opening, Transport};
10use crate::error::ProviderError;
11use crate::providers::gemini::cached_content::CachedContents;
12use crate::providers::gemini::caching::{
13 CacheBook, Create, DELETE_TIMEOUT, Lease, Parsed, Plan, cache_body, digests, is_user_text,
14 parse, short_digest, stripped,
15};
16use crate::providers::gemini::completion::{GenerateContent, ThoughtReplay};
17use crate::providers::internal::wire::classify_untyped_line;
18use crate::wasm_compat::timeout;
19use crate::wire::{Body, Encoded, Framing, Mode, WireEvent, WireFrame};
20
21impl CacheBook {
22 pub async fn prove<T>(&self, caches: &Model<CachedContents, T>) -> usize
25 where
26 T: Transport<CachedContents>,
27 {
28 let mut survived = 0;
29 for lease in self.leases() {
30 let alive = match caches.get(&lease.name).await {
31 Ok(resource) => resource
32 .display_name
33 .as_deref()
34 .is_some_and(|name| name.ends_with(short_digest(&lease.digest))),
35 Err(_) => false,
36 };
37 if alive {
38 survived += 1;
39 } else {
40 self.lost(&lease);
41 }
42 }
43 survived
44 }
45
46 pub async fn close<T>(&self, caches: &Model<CachedContents, T>)
51 where
52 T: Transport<CachedContents>,
53 {
54 self.book().closes += 1;
55 if timeout(DELETE_TIMEOUT, self.create.lock()).await.is_err() {
56 tracing::warn!(target: "gemini.cache", "close gave up waiting for a cache create");
57 }
58 for lease in self.leases() {
59 let Ok(result) = timeout(DELETE_TIMEOUT, caches.delete(&lease.name)).await else {
60 let left: Vec<String> = self.leases().into_iter().map(|l| l.name).collect();
61 let left = left.join(", ");
62 tracing::warn!(target: "gemini.cache", %left, "cache delete timed out");
63 return;
64 };
65 let gone = matches!(result, Ok(()) | Err(ProviderError::CacheExpired { .. }));
66 if !gone {
67 tracing::warn!(target: "gemini.cache", name = %lease.name, "cache delete failed");
68 continue;
69 }
70 self.retired(&lease);
71 }
72 }
73}
74
75#[derive(Clone)]
78pub struct Caching<T> {
79 inner: T,
80 config: crate::providers::gemini::GeminiConfig,
81 book: CacheBook,
82 thought_replay: ThoughtReplay,
83}
84
85impl<T> std::fmt::Debug for Caching<T> {
86 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
87 f.debug_struct("Caching")
88 .field("book", &self.book)
89 .field("thought_replay", &self.thought_replay)
90 .finish_non_exhaustive()
91 }
92}
93
94impl<T> Model<GenerateContent, T> {
95 pub fn caching(self, book: &CacheBook) -> Model<GenerateContent, Caching<T>> {
98 let caching = Caching {
99 inner: self.transport,
100 config: self.wire.provider.clone(),
101 book: book.clone(),
102 thought_replay: self.wire.thought_replay,
103 };
104 Model::new(self.wire, caching)
105 }
106}
107
108fn model_of(path: &str) -> Option<String> {
109 let rest = path.split("/models/").nth(1)?;
110 let (model, verb) = rest.split_once(':')?;
111 let verb = verb.split('?').next()?;
112 matches!(verb, "generateContent" | "streamGenerateContent").then(|| model.to_owned())
113}
114
115fn with_body(payload: &Encoded, bytes: Vec<u8>) -> Result<Encoded, ProviderError> {
117 let mut builder = http::Request::builder()
118 .method(payload.request.method().clone())
119 .uri(payload.request.uri().clone());
120 for (name, value) in payload.request.headers() {
121 builder = builder.header(name, value);
122 }
123 let request = builder
124 .body(Body::Bytes(bytes))
125 .map_err(|error| ProviderError::request(error.to_string()))?;
126 Ok(Encoded {
127 request,
128 framing: payload.framing,
129 request_id_header: payload.request_id_header,
130 relaxed_content_type: payload.relaxed_content_type,
131 })
132}
133
134struct Reply {
136 status: Option<u16>,
137 body: String,
138 error: Option<String>,
139}
140
141impl<T> Caching<T>
142where
143 T: Transport<GenerateContent>,
144{
145 async fn resource(&self, method: http::Method, path: &str, body: Option<Vec<u8>>) -> Reply {
147 let request = self
148 .config
149 .request(method, path)
150 .header("Content-Type", "application/json")
151 .body(Body::Bytes(body.unwrap_or_default()));
152 let request = match request {
153 Ok(request) => request,
154 Err(error) => {
155 return Reply {
156 status: None,
157 body: String::new(),
158 error: Some(error.to_string()),
159 };
160 }
161 };
162 let exchange = Exchange {
163 mode: Mode::Unary,
164 extensions: http::Extensions::new(),
165 };
166 let opened = match self
167 .inner
168 .send(Encoded::new(request, Framing::Whole), exchange)
169 .await
170 {
171 Ok(opened) => opened,
172 Err(error) => {
173 return Reply {
174 status: error.provider_response_status().map(|s| s.as_u16()),
175 body: String::new(),
176 error: Some(error.to_string()),
177 };
178 }
179 };
180 let mut frames = opened.frames;
181 let mut body = String::new();
182 while let Some(frame) = frames.next().await {
183 match frame {
184 Ok(frame) => body.push_str(&frame.as_str()),
185 Err(error) => {
186 return Reply {
187 status: error
188 .provider_response_status()
189 .map(|status| status.as_u16()),
190 body: error
191 .provider_response_body()
192 .unwrap_or_default()
193 .to_owned(),
194 error: Some(error.to_string()),
195 };
196 }
197 }
198 }
199 Reply {
200 status: None,
201 body,
202 error: None,
203 }
204 }
205
206 async fn delete(&self, lease: &Lease) {
207 let path = format!("/v1beta/{}", lease.name);
208 let request = self.resource(http::Method::DELETE, &path, None);
209 let Ok(reply) = timeout(DELETE_TIMEOUT, request).await else {
210 tracing::warn!(target: "gemini.cache", name = %lease.name, "cache delete timed out");
211 return;
212 };
213 if reply.error.is_some() && reply.status != Some(403) && reply.status != Some(404) {
214 let status = reply.status.unwrap_or_default();
215 tracing::warn!(target: "gemini.cache", name = %lease.name, status, "cache delete failed");
216 return;
217 }
218 self.book.retired(lease);
219 }
220
221 async fn extend(&self, lease: &Lease, ttl_secs: u64) -> bool {
223 let body = format!("{{\"ttl\":\"{ttl_secs}s\"}}").into_bytes();
224 let reply = self
225 .resource(
226 http::Method::PATCH,
227 &format!("/v1beta/{}?updateMask=ttl", lease.name),
228 Some(body),
229 )
230 .await;
231 if reply.error.is_some() {
232 if matches!(reply.status, Some(403 | 404)) && self.book.owns(lease) {
233 self.book.lost(lease);
234 }
235 return self.book.owns(lease);
236 }
237 let owned = self.book.extended(lease, ttl_secs);
238 if !owned {
239 self.delete(lease).await;
240 }
241 owned
242 }
243
244 async fn create(
249 &self,
250 model: &str,
251 parsed: &Parsed,
252 create: &Create,
253 plan: &Plan,
254 ) -> Option<Lease> {
255 let _single = self.book.create.lock().await;
256 if self.book.book().closes != plan.closes {
257 return None;
258 }
259 let existing = self.book.book().leases.get(&create.digest).cloned();
262 if let Some(existing) = existing
263 && existing.live_at(self.book.now())
264 {
265 return Some(existing);
266 }
267 let display_name = format!(
268 "{}{}",
269 self.book.display_prefix,
270 short_digest(&create.digest)
271 );
272 let body = cache_body(model, parsed, create.covers, &display_name, create.ttl_secs)?;
273 let reply = self
274 .resource(http::Method::POST, "/v1beta/cachedContents", Some(body))
275 .await;
276 if reply.error.is_some() {
277 #[derive(Deserialize)]
278 struct Envelope {
279 error: ErrorBody,
280 }
281 #[derive(Deserialize)]
282 struct ErrorBody {
283 message: String,
284 }
285 let message = known(classify_untyped_line::<Envelope>(reply.body.as_bytes()))
286 .map(|envelope| envelope.error.message)
287 .or(reply.error)
288 .unwrap_or_default();
289 self.book
290 .create_failed(&plan.line, plan.coverable, reply.status, message);
291 return None;
292 }
293 #[derive(Deserialize)]
294 #[serde(rename_all = "camelCase")]
295 struct Resource {
296 name: String,
297 #[serde(default)]
298 usage_metadata: Option<Usage>,
299 }
300 #[derive(Deserialize)]
301 #[serde(rename_all = "camelCase")]
302 struct Usage {
303 #[serde(default)]
304 total_token_count: u64,
305 }
306 let resource = known(classify_untyped_line::<Resource>(reply.body.as_bytes()))?;
307 let tokens = resource
308 .usage_metadata
309 .map_or(0, |usage| usage.total_token_count);
310 let lease = self
311 .book
312 .created(&plan.line, create, resource.name, tokens, model);
313 if self.book.book().closes != plan.closes {
314 tracing::warn!(target: "gemini.cache", name = %lease.name, "cache created during close");
315 return None;
316 }
317 Some(lease)
318 }
319}
320
321fn known<T>(event: WireEvent<T>) -> Option<T> {
323 match event {
324 WireEvent::Known(value) => Some(value),
325 _ => None,
326 }
327}
328
329fn final_cached(frame: &WireFrame) -> Option<u64> {
332 #[derive(Deserialize)]
333 #[serde(rename_all = "camelCase")]
334 struct Usage {
335 #[serde(default)]
336 cached_content_token_count: u64,
337 }
338 #[derive(Deserialize)]
339 #[serde(rename_all = "camelCase")]
340 struct Candidate {
341 finish_reason: Option<String>,
342 }
343 #[derive(Deserialize)]
344 #[serde(rename_all = "camelCase")]
345 struct Reply {
346 usage_metadata: Option<Usage>,
347 #[serde(default)]
348 candidates: Vec<Candidate>,
349 }
350 let reply = known(classify_untyped_line::<Reply>(frame.as_str().as_bytes()))?;
351 let finished = reply
352 .candidates
353 .iter()
354 .any(|candidate| candidate.finish_reason.is_some());
355 finished.then_some(reply.usage_metadata?.cached_content_token_count)
356}
357
358impl<T> Transport<GenerateContent> for Caching<T>
359where
360 T: Transport<GenerateContent>,
361{
362 fn send(&self, payload: Encoded, exchange: Exchange) -> Opening<WireFrame> {
363 let this = self.clone();
364 Opening::new(async move {
365 let path = payload.request.uri().path().to_owned();
366 let parsed = match payload.request.body() {
367 Body::Bytes(bytes) => parse(bytes),
368 Body::Multipart(_) => None,
369 };
370 let (Some(model), Some(parsed)) = (model_of(&path), parsed) else {
371 return this.inner.send(payload, exchange).await;
372 };
373 if parsed.has_cached_content {
374 return this.inner.send(payload, exchange).await;
375 }
376
377 let digested = digests(&model, &parsed);
378 let roll_allowed = match this.thought_replay {
379 ThoughtReplay::All => true,
380 ThoughtReplay::CurrentTurn => {
381 parsed.contents.last().is_some_and(|c| is_user_text(c))
382 }
383 };
384 let plan = this.book.plan(&model, &digested, roll_allowed);
385 for idle in &plan.retire {
386 this.delete(idle).await;
387 }
388 let mut read = plan.read.clone();
389 let mut line = plan.line.clone();
390 if let Some(create) = &plan.create
391 && let Some(lease) = this.create(&model, &parsed, create, &plan).await
392 {
393 if let Some(replaced) = &create.replaces
394 && replaced.name != lease.name
395 {
396 this.delete(replaced).await;
397 }
398 if lease.covers > 0 {
399 line = lease.digest.clone();
400 }
401 read = Some(lease);
402 }
403 if let Some((lease, ttl_secs)) = &plan.extend
404 && read.as_ref().is_some_and(|read| read.name == lease.name)
405 && !this.extend(lease, *ttl_secs).await
406 {
407 read = None;
408 }
409
410 let (sent, inline) = match read.as_ref().and_then(|lease| stripped(&parsed, lease)) {
413 Some(body) => (with_body(&payload, body)?, Some(payload)),
414 None => {
415 read = None;
416 (payload, None)
417 }
418 };
419 let mut opened: Opened<WireFrame> = this.inner.send(sent, exchange.clone()).await?;
420 if let Some(lease) = read.clone() {
421 let first = opened.frames.next().await;
422 let forbidden = matches!(
423 &first,
424 Some(Err(error)) if error.provider_response_status() == Some(http::StatusCode::FORBIDDEN)
425 );
426 if let (true, Some(inline)) = (forbidden, inline) {
427 this.book.lost(&lease);
429 read = None;
430 opened = this.inner.send(inline, exchange).await?;
431 } else {
432 this.book.touched(&lease);
433 let rest =
434 std::mem::replace(&mut opened.frames, Box::pin(futures::stream::empty()));
435 opened.frames = Box::pin(futures::stream::iter(first).chain(rest));
436 }
437 }
438
439 let book = this.book.clone();
440 let read_name = read.map(|lease| lease.name);
441 let coverable = plan.coverable;
442 Ok(opened.map_frames(move |frames| {
443 frames.inspect(move |frame| {
444 if let Ok(frame) = frame
445 && let Some(cached) = final_cached(frame)
446 {
447 book.observe(&line, read_name.as_deref(), coverable, cached);
448 }
449 })
450 }))
451 })
452 }
453}