Skip to main content

rig_core/providers/gemini/
streaming.rs

1use serde::{Deserialize, Serialize};
2
3use super::completion::gemini_api_types::{
4    ContentCandidate, FinishReason, GenerateContentResponse, Part, PartKind, UsageMetadata,
5    map_finish_reason,
6};
7use super::completion::{blocked_prompt_error, function_call_finish_reason_error, part_kind_name};
8use crate::error::ProviderError;
9use crate::observe::ObservedError;
10use crate::operation::{Completion, Finish, TextPart};
11use crate::providers::internal::thoughts::Thoughts;
12use crate::providers::internal::wire;
13use crate::wire::{
14    AdapterEvent, AdapterUsage, AdapterVerdict, Decoder, Flow, ObservationSink, Out, WireEvent,
15    WireFrame,
16};
17
18/// Part-kind interpretation shared by the Gemini wires whose payloads
19/// coincide: REST `streamGenerateContent` and the Interactions API both
20/// deliver whole function calls and identity-less thought fragments.
21pub(crate) mod shared_parts {
22    use serde_json::Value;
23
24    use crate::error::ProviderError;
25    use crate::message::{CallId, ToolCall, ToolFunction, ToolName};
26    use crate::operation::Completion;
27    use crate::wire::Out;
28
29    /// Write a whole function call. An id-less call gets an id rig issues,
30    /// never a fabricated provider id, even when it shares a tool name; a
31    /// nameless one is dropped.
32    pub(crate) fn function_call(
33        out: &mut Out<'_, Completion>,
34        name: String,
35        args: Value,
36        wire_id: Option<String>,
37        signature: Option<String>,
38    ) -> Result<(), ProviderError> {
39        let Ok(name) = ToolName::new(name) else {
40            return Ok(());
41        };
42        out.tool_call(ToolCall {
43            id: CallId::from_wire(wire_id.unwrap_or_default()),
44            function: ToolFunction {
45                name,
46                arguments: args,
47            },
48            signature,
49            additional_params: None,
50        })
51    }
52}
53
54#[derive(Clone, Debug, Serialize, Deserialize)]
55pub struct StreamingCompletionResponse {
56    pub usage_metadata: UsageMetadata,
57    #[serde(skip_serializing_if = "Option::is_none")]
58    pub finish_reason: Option<FinishReason>,
59    #[serde(skip_serializing_if = "Option::is_none")]
60    pub finish_message: Option<String>,
61    #[serde(skip_serializing_if = "Option::is_none")]
62    pub model_version: Option<String>,
63    #[serde(skip_serializing_if = "Option::is_none")]
64    pub response_id: Option<String>,
65}
66
67impl From<&StreamingCompletionResponse> for crate::completion::Usage {
68    fn from(value: &StreamingCompletionResponse) -> crate::completion::Usage {
69        (&value.usage_metadata).into()
70    }
71}
72
73impl From<StreamingCompletionResponse> for crate::completion::Usage {
74    fn from(value: StreamingCompletionResponse) -> crate::completion::Usage {
75        (&value).into()
76    }
77}
78
79fn tool_protocol_finish_reason_error(choice: &ContentCandidate) -> Option<ProviderError> {
80    let reason = choice.finish_reason.as_ref()?;
81    function_call_finish_reason_error(reason, choice.finish_message.as_deref())
82}
83
84/// The recognizability markers of a `streamGenerateContent` chunk: every
85/// genuine frame carries `candidates`, `usageMetadata` and/or
86/// `promptFeedback` (a blocked prompt's only chunk may carry nothing but
87/// the feedback), and the service's in-band abort carries only `error`. A
88/// frame with any of them must fully decode (else `Corrupt`). A valid ID-only
89/// frame is recognized separately as metadata; other JSON is `Unknown`.
90const RECOGNIZABLE_CHUNK_KEYS: &[&str] =
91    &["candidates", "usageMetadata", "promptFeedback", "error"];
92
93/// Decode GenerateContent replies, a whole body or a stream of chunks.
94/// The provider's end is held until EOF because hosted-tool rounds can
95/// report intermediate finish reasons. A reply without assistant content
96/// fails unless a truncating finish reason permits empty output.
97pub struct GenerateContentDecoder<'id> {
98    /// Thought boundaries inferred from content transitions and signatures.
99    thoughts: Thoughts<'id>,
100    /// The answer text part text chunks extend.
101    text: Option<TextPart<'id>>,
102    final_usage: Option<UsageMetadata>,
103    final_finish_reason: Option<FinishReason>,
104    final_finish_message: Option<String>,
105    final_model_version: Option<String>,
106    final_response_id: Option<String>,
107    /// A provider finish reason was received, possibly before a hosted-tool round ended.
108    saw_finish_reason: bool,
109    /// At least one part mapped to assistant content.
110    delivered: bool,
111}
112
113impl GenerateContentDecoder<'_> {
114    /// A decoder for one reply.
115    pub(super) fn new() -> Self {
116        Self {
117            thoughts: Thoughts::new(),
118            text: None,
119            final_usage: None,
120            final_finish_reason: None,
121            final_finish_message: None,
122            final_model_version: None,
123            final_response_id: None,
124            saw_finish_reason: false,
125            delivered: false,
126        }
127    }
128}
129
130impl<'id> Decoder<'id, Completion> for GenerateContentDecoder<'id> {
131    type Event = GenerateContentResponse;
132
133    fn classify(&self, frame: WireFrame) -> WireEvent<GenerateContentResponse> {
134        // ID-only metadata must not create an unknown-content truncation tail.
135        if GenerateContentDecoder::is_analysis_only(&frame) {
136            return wire::classify_marker_keyed_frame(&frame.as_str(), &["responseId"]);
137        }
138        wire::classify_marker_keyed_frame(&frame.as_str(), RECOGNIZABLE_CHUNK_KEYS)
139    }
140
141    fn decode(
142        &mut self,
143        data: GenerateContentResponse,
144        mut out: Out<'id, Completion>,
145    ) -> Result<Flow, ProviderError> {
146        let span = tracing::Span::current();
147        // The document defaults an absent id to the empty string, which is
148        // the same "not reported" the normalized record's setter filters.
149        if !data.response_id.is_empty() {
150            span.record("gen_ai.response.id", data.response_id.as_str());
151            self.final_response_id = Some(data.response_id.clone());
152        }
153        if let Some(model_version) = &data.model_version {
154            span.record("gen_ai.response.model", model_version.as_str());
155            self.final_model_version = Some(model_version.clone());
156        }
157        if let Some(usage) = data.usage_metadata.as_ref() {
158            self.final_usage = Some(usage.clone());
159        }
160
161        if let Some(error) = data.error {
162            // Preserve in-band failures as provider errors, not unknown-frame truncation.
163            // GenerateContent codes are HTTP statuses; only error statuses
164            // participate in the unary retry policy.
165            let status = error
166                .get("code")
167                .and_then(serde_json::Value::as_u64)
168                .and_then(|code| u16::try_from(code).ok())
169                .and_then(|code| http::StatusCode::from_u16(code).ok())
170                .filter(|status| status.is_client_error() || status.is_server_error());
171            let body = serde_json::json!({ "error": error }).to_string();
172            return Err(match status {
173                Some(status) => ProviderError::from_http_response(status, body),
174                None => crate::error::ProviderError::from_provider_body(body),
175            });
176        }
177
178        if let Some(blocked) = data.prompt_feedback.as_ref().and_then(blocked_prompt_error) {
179            // Preserve the refusal reason rather than reporting an unexplained truncation.
180            return Err(blocked);
181        }
182
183        let Some(choice) = data.candidates.into_iter().next() else {
184            tracing::debug!("There is no content candidate");
185            return Ok(Flow::More);
186        };
187
188        if let Some(finish_reason) = &choice.finish_reason {
189            // Last one wins: an intermediate `finishReason` is superseded by
190            // the reason the turn actually ended on.
191            self.saw_finish_reason = true;
192            self.final_finish_reason = Some(finish_reason.clone());
193        }
194        if let Some(message) = &choice.finish_message {
195            self.final_finish_message = Some(message.clone());
196        }
197
198        if let Some(err) = tool_protocol_finish_reason_error(&choice) {
199            return Err(err);
200        }
201
202        match choice.content {
203            Some(content) => {
204                if content.parts.is_empty() {
205                    tracing::trace!(reason = ?self.final_finish_reason, "There is no part in the streaming content");
206                }
207                // Parts within one document are distinct parts; text chunks
208                // across stream events continue one part.
209                let mut previous_text = false;
210                for part in content.parts {
211                    let text = matches!(part.part, PartKind::Text(_)) && part.thought != Some(true);
212                    if text && previous_text {
213                        self.close_text(&mut out);
214                    }
215                    previous_text = text;
216                    self.interpret_part(part, &mut out)?;
217                }
218            }
219            None => {
220                // Gemini's final chunk may carry finishReason with no content.
221                tracing::debug!(finish_reason = ?self.final_finish_reason, "Streaming candidate missing content");
222            }
223        }
224        Ok(Flow::More)
225    }
226
227    /// Gemini ends a reply at EOF, not at its first finish reason: a
228    /// hosted-tool round can report one before more content arrives.
229    fn eof(&mut self, mut out: Out<'id, Completion>) -> Result<Flow, ProviderError> {
230        // Without a provider finish reason, the reply did not end.
231        if !self.saw_finish_reason {
232            return Err(ProviderError::Truncated);
233        }
234        // An empty reply needs a truncating finish reason to explain it.
235        let cut_short = self
236            .final_finish_reason
237            .as_ref()
238            .and_then(map_finish_reason)
239            .is_some_and(|reason| reason.truncated_output());
240        if !self.delivered && !cut_short {
241            return Err(ProviderError::Response(
242                crate::message::EMPTY_RESPONSE_ERROR.to_owned(),
243            ));
244        }
245        self.close_text(&mut out);
246        self.thoughts.close(&mut out, None);
247        // Defaulting the raw usage shape does not imply reported usage.
248        let usage = self
249            .final_usage
250            .as_ref()
251            .map(crate::completion::Usage::from)
252            .unwrap_or_default();
253        let native = StreamingCompletionResponse {
254            usage_metadata: self.final_usage.take().unwrap_or_default(),
255            finish_reason: self.final_finish_reason.take(),
256            finish_message: self.final_finish_message.take(),
257            model_version: self.final_model_version.take(),
258            response_id: self.final_response_id.take(),
259        };
260        out.raw(serde_json::to_value(&native)?);
261        let finish_reason = native.finish_reason.as_ref().and_then(map_finish_reason);
262        Ok(out.end(Finish {
263            usage,
264            reason: finish_reason,
265            response_id: native.response_id,
266            model: native.model_version,
267            ..Finish::default()
268        }))
269    }
270}
271
272impl GenerateContentDecoder<'_> {
273    pub(crate) fn is_analysis_only(frame: &WireFrame) -> bool {
274        #[derive(Deserialize)]
275        #[serde(deny_unknown_fields)]
276        struct ResponseIdOnly {
277            #[serde(rename = "responseId")]
278            _id: String,
279        }
280        matches!(
281            wire::classify_marker_keyed_frame::<ResponseIdOnly>(&frame.as_str(), &["responseId"]),
282            WireEvent::Known(_)
283        )
284    }
285
286    /// Project provider verdicts, usage, response identity, and errors before normalization.
287    pub(crate) fn project(payload: &[u8], sink: &mut ObservationSink<'_>) {
288        // The observation projection must not inherit native response
289        // defaults: omitted prompt/total counts in UsageMetadata otherwise
290        // become zero. Parsing failure has no effect on the provider's
291        // authoritative decoder.
292        if let Ok(ObservedUsageOnly { usage: Some(usage) }) =
293            serde_json::from_slice::<ObservedUsageOnly>(payload)
294        {
295            sink.emit(AdapterEvent::Usage {
296                usage: AdapterUsage {
297                    input_tokens: usage.prompt_token_count,
298                    output_tokens: usage.candidates_token_count,
299                    total_tokens: usage.total_token_count,
300                    cached_input_tokens: usage.cached_content_token_count,
301                    reasoning_tokens: usage.thoughts_token_count,
302                    tool_input_tokens: usage.tool_use_prompt_token_count,
303                },
304            });
305        }
306        // Project metadata independently so malformed candidate fields cannot
307        // erase an otherwise valid usage report from a rejected response.
308        let Ok(metadata) = serde_json::from_slice::<ObservedMetadata>(payload) else {
309            return;
310        };
311        let candidate = metadata.candidates.into_iter().next().unwrap_or_default();
312        let scrub = |value: String| sink.scrub(&value);
313        let verdict = AdapterVerdict {
314            finish_reason: candidate.finish_reason.map(scrub),
315            block_reason: metadata
316                .prompt_feedback
317                .and_then(|f| f.block_reason)
318                .map(scrub),
319            detail: candidate.finish_message.map(scrub),
320            model: metadata.model_version.map(scrub),
321        };
322        let response_id = metadata.response_id.map(scrub);
323        sink.provider(verdict, response_id);
324        if let Some(error) = metadata.error {
325            error.emit(sink);
326        }
327    }
328}
329
330/// The usage report alone, read off the payload before the verdict so a
331/// malformed candidate cannot erase it.
332#[derive(Deserialize)]
333struct ObservedUsageOnly {
334    #[serde(rename = "usageMetadata")]
335    usage: Option<ObservedUsage>,
336}
337
338// Ignore all unrelated response fields rather than allocating another tree
339// containing the completion text, tools, signatures and media.
340#[derive(Deserialize)]
341#[serde(rename_all = "camelCase")]
342struct ObservedUsage {
343    #[serde(default, deserialize_with = "crate::observe::lenient_count")]
344    prompt_token_count: Option<u64>,
345    #[serde(default, deserialize_with = "crate::observe::lenient_count")]
346    candidates_token_count: Option<u64>,
347    #[serde(default, deserialize_with = "crate::observe::lenient_count")]
348    total_token_count: Option<u64>,
349    #[serde(default, deserialize_with = "crate::observe::lenient_count")]
350    cached_content_token_count: Option<u64>,
351    #[serde(default, deserialize_with = "crate::observe::lenient_count")]
352    thoughts_token_count: Option<u64>,
353    #[serde(default, deserialize_with = "crate::observe::lenient_count")]
354    tool_use_prompt_token_count: Option<u64>,
355}
356
357#[derive(Deserialize)]
358#[serde(rename_all = "camelCase")]
359struct ObservedMetadata {
360    #[serde(default)]
361    candidates: Vec<ObservedCandidate>,
362    prompt_feedback: Option<ObservedFeedback>,
363    model_version: Option<String>,
364    response_id: Option<String>,
365    error: Option<ObservedError>,
366}
367
368#[derive(Default, Deserialize)]
369#[serde(rename_all = "camelCase")]
370struct ObservedCandidate {
371    finish_reason: Option<String>,
372    finish_message: Option<String>,
373}
374
375#[derive(Deserialize)]
376#[serde(rename_all = "camelCase")]
377struct ObservedFeedback {
378    block_reason: Option<String>,
379}
380
381impl<'id> GenerateContentDecoder<'id> {
382    fn close_text(&mut self, out: &mut Out<'id, Completion>) {
383        if let Some(part) = self.text.take() {
384            out.close_text(part);
385        }
386    }
387
388    fn interpret_part(
389        &mut self,
390        part: Part,
391        out: &mut Out<'id, Completion>,
392    ) -> Result<(), ProviderError> {
393        // Hosted code-execution parts alone do not constitute an assistant answer.
394        if !matches!(
395            part.part,
396            PartKind::ExecutableCode(_) | PartKind::CodeExecutionResult(_)
397        ) {
398            self.delivered = true;
399        }
400        match part {
401            Part {
402                part: PartKind::Text(text),
403                thought: Some(true),
404                thought_signature,
405                ..
406            } => {
407                // A signature closes accumulated reasoning or forms a signature-only part.
408                if !text.is_empty() {
409                    self.close_text(out);
410                }
411                self.thoughts.fragment(out, &text);
412                if let Some(signature) = thought_signature {
413                    self.thoughts.signature(out, signature);
414                }
415            }
416            Part {
417                part: PartKind::Text(text),
418                thought_signature,
419                ..
420            } => {
421                // A signature on answer text belongs to that text part, and
422                // must return on it: it rides on the part's extras. A
423                // signature on its own empty part keeps its own part, and a
424                // signed part ends there.
425                let signed = thought_signature.is_some();
426                if signed && text.is_empty() {
427                    self.close_text(out);
428                }
429                let params = thought_signature.and_then(|signature| {
430                    super::text_signature_extras(super::GEMINI_TEXT_EXTRAS_KEY, signature)
431                });
432                if !text.is_empty() || params.is_some() {
433                    self.thoughts.boundary();
434                    let part = self.text.get_or_insert_with(|| out.text());
435                    out.push_text(part, &text);
436                    if let Some(params) = params {
437                        out.text_params(part, params);
438                    }
439                }
440                if signed {
441                    self.close_text(out);
442                }
443            }
444            Part {
445                part: PartKind::FunctionCall(function_call),
446                thought_signature,
447                ..
448            } => {
449                // Tool content interleaving an open thought part stops it.
450                self.thoughts.boundary();
451                self.close_text(out);
452                shared_parts::function_call(
453                    out,
454                    function_call.name,
455                    function_call.args,
456                    function_call.id,
457                    thought_signature,
458                )?;
459            }
460            Part {
461                part: part @ PartKind::InlineData(_),
462                thought_signature,
463                ..
464            } => {
465                // Inline media rides on a text part's metadata: the choice
466                // has no part for it.
467                self.thoughts.boundary();
468                self.close_text(out);
469                let raw = Part {
470                    thought: None,
471                    thought_signature,
472                    part,
473                    additional_params: None,
474                };
475                if let Some(params) = crate::message::AdditionalParams::from_entries([(
476                    super::GEMINI_RAW_CONTENT_KEY,
477                    serde_json::json!(raw),
478                )]) {
479                    let part = out.text();
480                    out.text_params(&part, params);
481                    out.close_text(part);
482                }
483            }
484            Part {
485                part: part @ (PartKind::ExecutableCode(_) | PartKind::CodeExecutionResult(_)),
486                ..
487            } => {
488                // Log only the unmodeled part's kind; hosted-tool payloads may be sensitive.
489                crate::driver::warn_unmodeled("gemini_part", &part_kind_name(&part));
490            }
491            Part { part, .. } => {
492                // Unexpected response parts must fail rather than silently discard content.
493                return Err(ProviderError::Response(format!(
494                    "Gemini response part kind {} carries no assistant content rig can account for",
495                    part_kind_name(&part)
496                )));
497            }
498        }
499        Ok(())
500    }
501}
502
503#[cfg(test)]
504mod tests;