Skip to main content

rig_core/client/
gemini_caching.rs

1//! The [`Caching`] transport and the cache book's I/O: the part of Gemini's
2//! automatic caching that sends requests. What to read, create and retire is
3//! decided by [`CacheBook`] in [`crate::providers::gemini::caching`], from
4//! bytes alone.
5
6use futures::StreamExt;
7use serde::Deserialize;
8
9use crate::driver::{Exchange, Model, Opened, Opening, Transport};
10use crate::error::ProviderError;
11use crate::providers::gemini::cached_content::CachedContents;
12use crate::providers::gemini::caching::{
13    CacheBook, Create, DELETE_TIMEOUT, Lease, Parsed, Plan, cache_body, digests, is_user_text,
14    parse, short_digest, stripped,
15};
16use crate::providers::gemini::completion::{GenerateContent, ThoughtReplay};
17use crate::providers::internal::wire::classify_untyped_line;
18use crate::wasm_compat::timeout;
19use crate::wire::{Body, Encoded, Framing, Mode, WireEvent, WireFrame};
20
21impl CacheBook {
22    /// Keep each lease only if Google still has a cache under its name whose
23    /// display name ends with the lease's digest. Returns how many survived.
24    pub async fn prove<T>(&self, caches: &Model<CachedContents, T>) -> usize
25    where
26        T: Transport<CachedContents>,
27    {
28        let mut survived = 0;
29        for lease in self.leases() {
30            let alive = match caches.get(&lease.name).await {
31                Ok(resource) => resource
32                    .display_name
33                    .as_deref()
34                    .is_some_and(|name| name.ends_with(short_digest(&lease.digest))),
35                Err(_) => false,
36            };
37            if alive {
38                survived += 1;
39            } else {
40                self.lost(&lease);
41            }
42        }
43        survived
44    }
45
46    /// Delete every cache in [`Self::leases`], keeping any whose delete fails,
47    /// after waiting up to 10 seconds for a create in flight. It stops at the
48    /// first delete with no answer in 10 seconds, and logs what is left. A
49    /// cache already gone (403) counts as deleted.
50    pub async fn close<T>(&self, caches: &Model<CachedContents, T>)
51    where
52        T: Transport<CachedContents>,
53    {
54        self.book().closes += 1;
55        if timeout(DELETE_TIMEOUT, self.create.lock()).await.is_err() {
56            tracing::warn!(target: "gemini.cache", "close gave up waiting for a cache create");
57        }
58        for lease in self.leases() {
59            let Ok(result) = timeout(DELETE_TIMEOUT, caches.delete(&lease.name)).await else {
60                let left: Vec<String> = self.leases().into_iter().map(|l| l.name).collect();
61                let left = left.join(", ");
62                tracing::warn!(target: "gemini.cache", %left, "cache delete timed out");
63                return;
64            };
65            let gone = matches!(result, Ok(()) | Err(ProviderError::CacheExpired { .. }));
66            if !gone {
67                tracing::warn!(target: "gemini.cache", name = %lease.name, "cache delete failed");
68                continue;
69            }
70            self.retired(&lease);
71        }
72    }
73}
74
75/// A transport that caches GenerateContent requests through `inner`, as
76/// its [`CacheBook`] decides. Build it with [`Model::caching`].
77#[derive(Clone)]
78pub struct Caching<T> {
79    inner: T,
80    config: crate::providers::gemini::GeminiConfig,
81    book: CacheBook,
82    thought_replay: ThoughtReplay,
83}
84
85impl<T> std::fmt::Debug for Caching<T> {
86    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
87        f.debug_struct("Caching")
88            .field("book", &self.book)
89            .field("thought_replay", &self.thought_replay)
90            .finish_non_exhaustive()
91    }
92}
93
94impl<T> Model<GenerateContent, T> {
95    /// This model, reading and creating explicit caches as `book` decides.
96    /// See [`crate::providers::gemini::caching`].
97    pub fn caching(self, book: &CacheBook) -> Model<GenerateContent, Caching<T>> {
98        let caching = Caching {
99            inner: self.transport,
100            config: self.wire.provider.clone(),
101            book: book.clone(),
102            thought_replay: self.wire.thought_replay,
103        };
104        Model::new(self.wire, caching)
105    }
106}
107
108fn model_of(path: &str) -> Option<String> {
109    let rest = path.split("/models/").nth(1)?;
110    let (model, verb) = rest.split_once(':')?;
111    let verb = verb.split('?').next()?;
112    matches!(verb, "generateContent" | "streamGenerateContent").then(|| model.to_owned())
113}
114
115/// `payload` with `bytes` as its body.
116fn with_body(payload: &Encoded, bytes: Vec<u8>) -> Result<Encoded, ProviderError> {
117    let mut builder = http::Request::builder()
118        .method(payload.request.method().clone())
119        .uri(payload.request.uri().clone());
120    for (name, value) in payload.request.headers() {
121        builder = builder.header(name, value);
122    }
123    let request = builder
124        .body(Body::Bytes(bytes))
125        .map_err(|error| ProviderError::request(error.to_string()))?;
126    Ok(Encoded {
127        request,
128        framing: payload.framing,
129        request_id_header: payload.request_id_header,
130        relaxed_content_type: payload.relaxed_content_type,
131    })
132}
133
134/// A `cachedContents` reply: its status (when it failed) and its body.
135struct Reply {
136    status: Option<u16>,
137    body: String,
138    error: Option<String>,
139}
140
141impl<T> Caching<T>
142where
143    T: Transport<GenerateContent>,
144{
145    /// Send one `cachedContents` request through the inner transport.
146    async fn resource(&self, method: http::Method, path: &str, body: Option<Vec<u8>>) -> Reply {
147        let request = self
148            .config
149            .request(method, path)
150            .header("Content-Type", "application/json")
151            .body(Body::Bytes(body.unwrap_or_default()));
152        let request = match request {
153            Ok(request) => request,
154            Err(error) => {
155                return Reply {
156                    status: None,
157                    body: String::new(),
158                    error: Some(error.to_string()),
159                };
160            }
161        };
162        let exchange = Exchange {
163            mode: Mode::Unary,
164            extensions: http::Extensions::new(),
165        };
166        let opened = match self
167            .inner
168            .send(Encoded::new(request, Framing::Whole), exchange)
169            .await
170        {
171            Ok(opened) => opened,
172            Err(error) => {
173                return Reply {
174                    status: error.provider_response_status().map(|s| s.as_u16()),
175                    body: String::new(),
176                    error: Some(error.to_string()),
177                };
178            }
179        };
180        let mut frames = opened.frames;
181        let mut body = String::new();
182        while let Some(frame) = frames.next().await {
183            match frame {
184                Ok(frame) => body.push_str(&frame.as_str()),
185                Err(error) => {
186                    return Reply {
187                        status: error
188                            .provider_response_status()
189                            .map(|status| status.as_u16()),
190                        body: error
191                            .provider_response_body()
192                            .unwrap_or_default()
193                            .to_owned(),
194                        error: Some(error.to_string()),
195                    };
196                }
197            }
198        }
199        Reply {
200            status: None,
201            body,
202            error: None,
203        }
204    }
205
206    async fn delete(&self, lease: &Lease) {
207        let path = format!("/v1beta/{}", lease.name);
208        let request = self.resource(http::Method::DELETE, &path, None);
209        let Ok(reply) = timeout(DELETE_TIMEOUT, request).await else {
210            tracing::warn!(target: "gemini.cache", name = %lease.name, "cache delete timed out");
211            return;
212        };
213        if reply.error.is_some() && reply.status != Some(403) && reply.status != Some(404) {
214            let status = reply.status.unwrap_or_default();
215            tracing::warn!(target: "gemini.cache", name = %lease.name, status, "cache delete failed");
216            return;
217        }
218        self.book.retired(lease);
219    }
220
221    /// Extend `lease`. Returns whether it can still be read.
222    async fn extend(&self, lease: &Lease, ttl_secs: u64) -> bool {
223        let body = format!("{{\"ttl\":\"{ttl_secs}s\"}}").into_bytes();
224        let reply = self
225            .resource(
226                http::Method::PATCH,
227                &format!("/v1beta/{}?updateMask=ttl", lease.name),
228                Some(body),
229            )
230            .await;
231        if reply.error.is_some() {
232            if matches!(reply.status, Some(403 | 404)) && self.book.owns(lease) {
233                self.book.lost(lease);
234            }
235            return self.book.owns(lease);
236        }
237        let owned = self.book.extended(lease, ttl_secs);
238        if !owned {
239            self.delete(lease).await;
240        }
241        owned
242    }
243
244    /// Create the cache `create` describes, unless a live one already exists
245    /// for its digest or the book closed since `plan`. Returns the lease to
246    /// read. One that lands after a close stays in the book for a close to
247    /// delete, and is not read.
248    async fn create(
249        &self,
250        model: &str,
251        parsed: &Parsed,
252        create: &Create,
253        plan: &Plan,
254    ) -> Option<Lease> {
255        let _single = self.book.create.lock().await;
256        if self.book.book().closes != plan.closes {
257            return None;
258        }
259        // Checked again under the lock: another request may have created it,
260        // or the one the plan saw may have expired while this one waited.
261        let existing = self.book.book().leases.get(&create.digest).cloned();
262        if let Some(existing) = existing
263            && existing.live_at(self.book.now())
264        {
265            return Some(existing);
266        }
267        let display_name = format!(
268            "{}{}",
269            self.book.display_prefix,
270            short_digest(&create.digest)
271        );
272        let body = cache_body(model, parsed, create.covers, &display_name, create.ttl_secs)?;
273        let reply = self
274            .resource(http::Method::POST, "/v1beta/cachedContents", Some(body))
275            .await;
276        if reply.error.is_some() {
277            #[derive(Deserialize)]
278            struct Envelope {
279                error: ErrorBody,
280            }
281            #[derive(Deserialize)]
282            struct ErrorBody {
283                message: String,
284            }
285            let message = known(classify_untyped_line::<Envelope>(reply.body.as_bytes()))
286                .map(|envelope| envelope.error.message)
287                .or(reply.error)
288                .unwrap_or_default();
289            self.book
290                .create_failed(&plan.line, plan.coverable, reply.status, message);
291            return None;
292        }
293        #[derive(Deserialize)]
294        #[serde(rename_all = "camelCase")]
295        struct Resource {
296            name: String,
297            #[serde(default)]
298            usage_metadata: Option<Usage>,
299        }
300        #[derive(Deserialize)]
301        #[serde(rename_all = "camelCase")]
302        struct Usage {
303            #[serde(default)]
304            total_token_count: u64,
305        }
306        let resource = known(classify_untyped_line::<Resource>(reply.body.as_bytes()))?;
307        let tokens = resource
308            .usage_metadata
309            .map_or(0, |usage| usage.total_token_count);
310        let lease = self
311            .book
312            .created(&plan.line, create, resource.name, tokens, model);
313        if self.book.book().closes != plan.closes {
314            tracing::warn!(target: "gemini.cache", name = %lease.name, "cache created during close");
315            return None;
316        }
317        Some(lease)
318    }
319}
320
321/// The decoded value of a classified payload, when it decoded.
322fn known<T>(event: WireEvent<T>) -> Option<T> {
323    match event {
324        WireEvent::Known(value) => Some(value),
325        _ => None,
326    }
327}
328
329/// The final `cachedContentTokenCount` a reply frame reports, when the frame
330/// ends the reply.
331fn final_cached(frame: &WireFrame) -> Option<u64> {
332    #[derive(Deserialize)]
333    #[serde(rename_all = "camelCase")]
334    struct Usage {
335        #[serde(default)]
336        cached_content_token_count: u64,
337    }
338    #[derive(Deserialize)]
339    #[serde(rename_all = "camelCase")]
340    struct Candidate {
341        finish_reason: Option<String>,
342    }
343    #[derive(Deserialize)]
344    #[serde(rename_all = "camelCase")]
345    struct Reply {
346        usage_metadata: Option<Usage>,
347        #[serde(default)]
348        candidates: Vec<Candidate>,
349    }
350    let reply = known(classify_untyped_line::<Reply>(frame.as_str().as_bytes()))?;
351    let finished = reply
352        .candidates
353        .iter()
354        .any(|candidate| candidate.finish_reason.is_some());
355    finished.then_some(reply.usage_metadata?.cached_content_token_count)
356}
357
358impl<T> Transport<GenerateContent> for Caching<T>
359where
360    T: Transport<GenerateContent>,
361{
362    fn send(&self, payload: Encoded, exchange: Exchange) -> Opening<WireFrame> {
363        let this = self.clone();
364        Opening::new(async move {
365            let path = payload.request.uri().path().to_owned();
366            let parsed = match payload.request.body() {
367                Body::Bytes(bytes) => parse(bytes),
368                Body::Multipart(_) => None,
369            };
370            let (Some(model), Some(parsed)) = (model_of(&path), parsed) else {
371                return this.inner.send(payload, exchange).await;
372            };
373            if parsed.has_cached_content {
374                return this.inner.send(payload, exchange).await;
375            }
376
377            let digested = digests(&model, &parsed);
378            let roll_allowed = match this.thought_replay {
379                ThoughtReplay::All => true,
380                ThoughtReplay::CurrentTurn => {
381                    parsed.contents.last().is_some_and(|c| is_user_text(c))
382                }
383            };
384            let plan = this.book.plan(&model, &digested, roll_allowed);
385            for idle in &plan.retire {
386                this.delete(idle).await;
387            }
388            let mut read = plan.read.clone();
389            let mut line = plan.line.clone();
390            if let Some(create) = &plan.create
391                && let Some(lease) = this.create(&model, &parsed, create, &plan).await
392            {
393                if let Some(replaced) = &create.replaces
394                    && replaced.name != lease.name
395                {
396                    this.delete(replaced).await;
397                }
398                if lease.covers > 0 {
399                    line = lease.digest.clone();
400                }
401                read = Some(lease);
402            }
403            if let Some((lease, ttl_secs)) = &plan.extend
404                && read.as_ref().is_some_and(|read| read.name == lease.name)
405                && !this.extend(lease, *ttl_secs).await
406            {
407                read = None;
408            }
409
410            // A request that reads a cache keeps the encoded one to resend
411            // inline; any other goes out as encoded.
412            let (sent, inline) = match read.as_ref().and_then(|lease| stripped(&parsed, lease)) {
413                Some(body) => (with_body(&payload, body)?, Some(payload)),
414                None => {
415                    read = None;
416                    (payload, None)
417                }
418            };
419            let mut opened: Opened<WireFrame> = this.inner.send(sent, exchange.clone()).await?;
420            if let Some(lease) = read.clone() {
421                let first = opened.frames.next().await;
422                let forbidden = matches!(
423                    &first,
424                    Some(Err(error)) if error.provider_response_status() == Some(http::StatusCode::FORBIDDEN)
425                );
426                if let (true, Some(inline)) = (forbidden, inline) {
427                    // Deleted elsewhere or expired early: forget it and send inline, once.
428                    this.book.lost(&lease);
429                    read = None;
430                    opened = this.inner.send(inline, exchange).await?;
431                } else {
432                    this.book.touched(&lease);
433                    let rest =
434                        std::mem::replace(&mut opened.frames, Box::pin(futures::stream::empty()));
435                    opened.frames = Box::pin(futures::stream::iter(first).chain(rest));
436                }
437            }
438
439            let book = this.book.clone();
440            let read_name = read.map(|lease| lease.name);
441            let coverable = plan.coverable;
442            Ok(opened.map_frames(move |frames| {
443                frames.inspect(move |frame| {
444                    if let Ok(frame) = frame
445                        && let Some(cached) = final_cached(frame)
446                    {
447                        book.observe(&line, read_name.as_deref(), coverable, cached);
448                    }
449                })
450            }))
451        })
452    }
453}