Skip to main content

rig_core/providers/gemini/
streaming.rs

1//! The decoder of GenerateContent replies, a whole `generateContent` body or
2//! a stream of `streamGenerateContent` chunks, read as REST JSON. The REST,
3//! Vertex AI and gRPC wires share it: each part becomes a block whose
4//! provider item is the part as Gemini sent it.
5//!
6//! ```
7//! use rig_core::providers::gemini::streaming::GenerateContentDecoder;
8//!
9//! let decoder = GenerateContentDecoder::default();
10//! # let _ = decoder;
11//! ```
12
13use serde_json::{Map, Value};
14
15use super::completion::blocked_prompt_error;
16use super::completion::{map_google_finish_reason, usage_of};
17use crate::error::ProviderError;
18use crate::json_utils::Lenient;
19use crate::message::{CallId, DocumentSourceKind, Image, ImageMediaType, MimeType, ToolName};
20use crate::operation::{Block, Completion, Finish};
21use crate::providers::internal::wire;
22use crate::wire::{Decoder, Flow, Out, WireCitation, WireEvent, WireFrame};
23
24/// The recognizability markers of a `streamGenerateContent` chunk: every
25/// genuine frame carries `candidates`, `usageMetadata` and/or
26/// `promptFeedback` (a blocked prompt's only chunk may carry nothing but
27/// the feedback), and the service's in-band abort carries only `error`. A
28/// frame with any of them is a chunk. A valid ID-only frame is recognized
29/// separately as metadata; other JSON is `Unknown`.
30const RECOGNIZABLE_CHUNK_KEYS: &[&str] =
31    &["candidates", "usageMetadata", "promptFeedback", "error"];
32
33/// A GenerateContent reply document in REST JSON: the whole
34/// `generateContent` body or one `streamGenerateContent` chunk.
35#[derive(Debug, Default, serde::Deserialize)]
36#[serde(transparent)]
37pub struct GenerateContentChunk(pub Map<String, Value>);
38
39/// Decode GenerateContent replies. A text or thought part continues the
40/// block of the part before it while the kind stays the same, and a part
41/// that carries only a thought signature joins that block; any other part is
42/// a block of its own. The provider's end is held until EOF because
43/// hosted-tool rounds can report intermediate finish reasons. The
44/// candidate's grounding and recitation metadata cite the text blocks when
45/// the reply ends.
46#[derive(Debug, Default)]
47pub struct GenerateContentDecoder {
48    /// The latest `finishReason`.
49    finish: Option<String>,
50    /// The latest `usageMetadata`, as Gemini sent it.
51    usage: Option<Value>,
52    model_version: Option<String>,
53    response_id: Option<String>,
54    /// The text or thought run later parts continue, and whether it is
55    /// thought.
56    open: Option<(usize, bool)>,
57    /// Whether the open run holds a signature.
58    signed: bool,
59    /// The index of the block written last.
60    last: Option<usize>,
61    /// A signature sent alone before any block, which the next block takes.
62    signature: Option<String>,
63    /// The answer text written so far, which grounding segments cite.
64    answer: grounding::AnswerText,
65    /// The text block and byte offset in it of each part of the latest
66    /// chunk; `None` for a part that is not answer text.
67    placed: Vec<Option<(usize, usize)>>,
68    /// Where the part being written put its text.
69    placement: Option<(usize, usize)>,
70    /// The latest `groundingMetadata`'s citations, by block.
71    grounding: Vec<(usize, WireCitation)>,
72    /// How many chunks carried parts: a reply over more than one counts a
73    /// grounding segment's bytes in the whole answer, not in a part.
74    chunks: usize,
75    /// The `citationMetadata` sources of every chunk, by block.
76    recitations: Vec<(usize, WireCitation)>,
77}
78
79impl<'id> Decoder<'id, Completion> for GenerateContentDecoder {
80    type Event = GenerateContentChunk;
81
82    fn classify(&self, frame: WireFrame) -> WireEvent<GenerateContentChunk> {
83        match wire::classify_marker_keyed_frame(&frame.as_str(), RECOGNIZABLE_CHUNK_KEYS) {
84            chunk @ WireEvent::Known(_) => chunk,
85            // ID-only metadata must not create an unknown-content truncation tail.
86            _ if Self::is_analysis_only(&frame) => {
87                wire::classify_untyped_line(frame.as_str().as_bytes())
88            }
89            event => event,
90        }
91    }
92
93    fn decode(
94        &mut self,
95        GenerateContentChunk(data): GenerateContentChunk,
96        mut out: Out<'id, Completion>,
97    ) -> Result<Flow, ProviderError> {
98        let mut data = Value::Object(data);
99        // The parts move into their blocks; the rest of the chunk is read
100        // in place.
101        let parts = data
102            .pointer_mut("/candidates/0/content/parts")
103            .map(Value::take);
104        // Gemini repeats both on every chunk. Record a value when it changes:
105        // every span record counts against the exporter's attribute limit.
106        let span = tracing::Span::current();
107        if let Some(id) = data.str("responseId").filter(|id| !id.is_empty())
108            && self.response_id.as_deref() != Some(id)
109        {
110            span.record("gen_ai.response.id", id);
111            self.response_id = Some(id.to_owned());
112        }
113        if let Some(model) = data.str("modelVersion").filter(|model| !model.is_empty())
114            && self.model_version.as_deref() != Some(model)
115        {
116            span.record("gen_ai.response.model", model);
117            self.model_version = Some(model.to_owned());
118        }
119        if let Some(usage) = data.get_mut("usageMetadata") {
120            let usage = usage.take();
121            out.usage(usage_of(&usage));
122            self.usage = Some(usage);
123        }
124        if let Some(error) = data.at("/error") {
125            // An in-band failure is a provider error, not a truncation.
126            let body = serde_json::json!({ "error": error }).to_string();
127            return Err(ProviderError::from_provider_body(body));
128        }
129        if let Some(blocked) = data.get("promptFeedback").and_then(blocked_prompt_error) {
130            return Err(blocked);
131        }
132        // The candidates, their content and its parts hold every block and
133        // the finish, so a wrongly typed one fails the reply.
134        let candidate = match data
135            .get("candidates")
136            .map(|candidates| (candidates, candidates.get(0)))
137        {
138            None | Some((Value::Null, _) | (Value::Array(_), None)) => return Ok(Flow::More),
139            Some((Value::Array(_), Some(candidate @ Value::Object(_)))) => candidate,
140            Some((Value::Array(_), Some(_))) => {
141                return Err(malformed("a candidate that is not an object"));
142            }
143            Some(_) => return Err(malformed("candidates that are not a list")),
144        };
145        // Last one wins: an intermediate `finishReason` is superseded by the
146        // reason the turn actually ended on. Proto3 JSON spells an enum
147        // value its schema does not know as its number.
148        match candidate.get("finishReason") {
149            Some(Value::String(name)) => self.finish = Some(name.clone()),
150            Some(Value::Number(number)) => self.finish = Some(format!("FINISH_REASON_{number}")),
151            _ => {}
152        }
153        let parts = match candidate.get("content") {
154            None | Some(Value::Null) => None,
155            Some(Value::Object(_)) => parts,
156            Some(_) => return Err(malformed("candidate content that is not an object")),
157        };
158        match parts {
159            None | Some(Value::Null) => {}
160            Some(Value::Array(parts)) => {
161                self.chunks += 1;
162                self.placed.clear();
163                for part in parts {
164                    self.part(part, &mut out)?;
165                    self.placed.push(self.placement.take());
166                }
167            }
168            Some(_) => return Err(malformed("candidate parts that are not a list")),
169        }
170        // Each chunk restates the grounding whole; recitation sources add up.
171        if let Some(metadata) = candidate.get("groundingMetadata") {
172            self.grounding =
173                grounding::grounding(metadata, &self.placed, &self.answer, self.chunks > 1);
174        }
175        if let Some(metadata) = candidate.get("citationMetadata") {
176            let recitations = grounding::recitations(metadata, &self.answer);
177            self.recitations.extend(recitations);
178        }
179        // A failure is final: nothing after it is read.
180        use crate::completion::FinishReason::{Length, Stop};
181        let reason = self.finish.as_deref().map(map_google_finish_reason);
182        match (reason, candidate.get("finishReason")) {
183            (None | Some(Stop | Length), _) | (_, None) => Ok(Flow::More),
184            _ => self.end(out),
185        }
186    }
187
188    /// Gemini ends a reply at EOF, not at its first finish reason: a
189    /// hosted-tool round can report one before more content arrives.
190    fn eof(&mut self, out: Out<'id, Completion>) -> Result<Flow, ProviderError> {
191        self.end(out)
192    }
193}
194
195/// A transport that delivers each reply message as its REST chunk, such as
196/// an SDK's reply transcoded once, hands the decoder the chunk itself.
197impl<'id> Decoder<'id, Completion, GenerateContentChunk> for GenerateContentDecoder {
198    type Event = GenerateContentChunk;
199
200    fn classify(&self, chunk: GenerateContentChunk) -> WireEvent<GenerateContentChunk> {
201        WireEvent::Known(chunk)
202    }
203
204    fn decode(
205        &mut self,
206        chunk: GenerateContentChunk,
207        out: Out<'id, Completion>,
208    ) -> Result<Flow, ProviderError> {
209        Decoder::<'id, Completion>::decode(self, chunk, out)
210    }
211
212    fn eof(&mut self, out: Out<'id, Completion>) -> Result<Flow, ProviderError> {
213        self.end(out)
214    }
215}
216
217impl GenerateContentDecoder {
218    /// End the reply on the finish reason the candidate holds. Without one
219    /// the reply did not end.
220    fn end(&mut self, mut out: Out<'_, Completion>) -> Result<Flow, ProviderError> {
221        let Some(reason) = self.finish.take() else {
222            return Err(ProviderError::Truncated);
223        };
224        self.close(&mut out)?;
225        let citations = std::mem::take(&mut self.grounding)
226            .into_iter()
227            .chain(std::mem::take(&mut self.recitations));
228        for (index, citation) in citations {
229            out.cite(index, citation);
230        }
231        let usage = self.usage.as_ref().map(usage_of).unwrap_or_default();
232        let model = self.model_version.take();
233        let response_id = self.response_id.take();
234        Ok(out.end(Finish {
235            usage,
236            reason: Some(map_google_finish_reason(&reason)),
237            response_id,
238            model,
239            ..Finish::default()
240        }))
241    }
242
243    /// Finish the open text or thought run: Gemini states each part whole.
244    fn close(&mut self, out: &mut Out<'_, Completion>) -> Result<(), ProviderError> {
245        self.open
246            .take()
247            .map_or(Ok(()), |(index, _)| out.finish(index))
248    }
249
250    /// Write a block at a fresh index, closing the run before it. A text or
251    /// thought block stays open as the run later parts continue; any other
252    /// is whole.
253    fn open(
254        &mut self,
255        block: Block,
256        item: Value,
257        text: &str,
258        out: &mut Out<'_, Completion>,
259    ) -> Result<(), ProviderError> {
260        self.close(out)?;
261        let index = out.fresh_index();
262        let mut item = item;
263        if let (Some(signature), Some(fields)) = (self.signature.take(), item.as_object_mut()) {
264            fields
265                .entry("thoughtSignature")
266                .or_insert(Value::String(signature));
267        }
268        self.last = Some(index);
269        self.signed = signature(&item).is_some();
270        if matches!(block, Block::Text) {
271            self.answer.open(index, text);
272            self.placement = Some((index, 0));
273        }
274        let run = match &block {
275            Block::Text => Some(false),
276            Block::Reasoning { .. } => Some(true),
277            _ => None,
278        };
279        out.open(index, block, item)?;
280        out.push(index, text)?;
281        self.open = run.map(|thought| (index, thought));
282        self.open.map_or_else(|| out.finish(index), |_| Ok(()))
283    }
284
285    /// Write one part of the candidate's content. A part this decoder does
286    /// not model is an opaque item that replays; one with no data, or that
287    /// is not an object, is kept as one that does not. A signature alone
288    /// joins the block before it, as Gemini attaches it to the part before,
289    /// or the next block when it comes first.
290    fn part(&mut self, part: Value, out: &mut Out<'_, Completion>) -> Result<(), ProviderError> {
291        let Value::Object(fields) = &part else {
292            return self.open(Block::Opaque { replay: false }, part, "", out);
293        };
294        let thought = part.bool("thought") == Some(true);
295        let signature = signature(&part);
296        if let Some(text) = part.str("text") {
297            // An empty part without a signature carries nothing.
298            if text.is_empty() && signature.is_none() {
299                return Ok(());
300            }
301            // A second signed part starts a block of its own, so each
302            // signature stays with the part that carried it.
303            return match self.open {
304                Some((index, kind)) if kind == thought && !(self.signed && signature.is_some()) => {
305                    self.signed |= signature.is_some();
306                    if !thought {
307                        self.placement = self.answer.push(index, text).map(|at| (index, at));
308                    }
309                    out.push(index, text)?;
310                    out.edit(index, |item| merge_part(item, &part))
311                }
312                _ => {
313                    let block = match thought {
314                        true => Block::Reasoning { redacted: false },
315                        false => Block::Text,
316                    };
317                    let text = text.to_owned();
318                    self.open(block, part, &text, out)
319                }
320            };
321        }
322        if let Some(call) = part.obj("functionCall") {
323            let Ok(name) =
324                ToolName::new(call.get("name").and_then(Value::as_str).unwrap_or_default())
325            else {
326                tracing::warn!("Gemini sent a function call without a name; nothing can answer it");
327                return Ok(());
328            };
329            // Proto3 JSON leaves out an empty `args` Struct.
330            let args = call
331                .get("args")
332                .map_or_else(|| "{}".to_owned(), Value::to_string);
333            let id = CallId::from_wire(call.get("id").and_then(Value::as_str).unwrap_or_default());
334            return self.open(Block::Call { id, name }, part, &args, out);
335        }
336        if !thought
337            && let (Some(mime_type), Some(data)) =
338                (part.at("/inlineData/mimeType"), part.at("/inlineData/data"))
339            && let (Some(mime_type), Some(data)) = (mime_type.as_str(), data.as_str())
340            && let Some(media_type) = ImageMediaType::from_mime_type(mime_type)
341        {
342            let image = Image {
343                data: DocumentSourceKind::Base64(data.to_owned()),
344                media_type: Some(media_type),
345                detail: None,
346                native: None,
347            };
348            return self.open(Block::Image(image), part, "", out);
349        }
350        let data = !is_bare(fields);
351        if !data && let Some(signature) = signature {
352            let Some(index) = self.open.map(|(index, _)| index).or(self.last) else {
353                self.signature = Some(signature.to_owned());
354                return Ok(());
355            };
356            return out.edit(index, |item| {
357                if let Some(item) = item.as_object_mut() {
358                    item.insert("thoughtSignature".to_owned(), Value::from(signature));
359                }
360            });
361        }
362        self.open(Block::Opaque { replay: data }, part, "", out)
363    }
364
365    /// Whether `frame` carries a response id and nothing else, a repeated
366    /// key included.
367    fn is_analysis_only(frame: &WireFrame) -> bool {
368        #[derive(serde::Deserialize)]
369        #[serde(deny_unknown_fields)]
370        struct ResponseIdOnly {
371            #[serde(rename = "responseId")]
372            _id: String,
373        }
374        matches!(
375            wire::classify_untyped_line::<ResponseIdOnly>(frame.as_str().as_bytes()),
376            WireEvent::Known(_)
377        )
378    }
379}
380
381/// A part's thought signature, when it is not empty.
382fn signature(part: &Value) -> Option<&str> {
383    part.str("thoughtSignature")
384        .filter(|signature| !signature.is_empty())
385}
386
387/// Whether a part holds nothing but its thought flag, signature and
388/// metadata.
389fn is_bare(part: &Map<String, Value>) -> bool {
390    let bare = ["thought", "thoughtSignature", "partMetadata"];
391    part.keys().all(|key| bare.contains(&key.as_str()))
392}
393
394fn malformed(what: &str) -> ProviderError {
395    ProviderError::Response(format!("Gemini sent {what}"))
396}
397
398/// Merge a streamed part into the part its block holds so far: the text
399/// appends, a non-empty `thoughtSignature` replaces the one held, and any
400/// other field replaces its namesake.
401fn merge_part(item: &mut Value, part: &Value) {
402    let (Value::Object(held), Value::Object(part)) = (&mut *item, part) else {
403        *item = part.clone();
404        return;
405    };
406    for (key, value) in part {
407        if key == "text"
408            && let (Some(Value::String(text)), Value::String(more)) = (held.get_mut("text"), value)
409        {
410            text.push_str(more);
411        } else if key != "thoughtSignature" || value.as_str() != Some("") {
412            held.insert(key.clone(), value.clone());
413        }
414    }
415}
416
417pub mod document;
418mod grounding;
419
420#[cfg(test)]
421mod tests;