Skip to main content

rig_core/providers/gemini/interactions_api/
streaming.rs

1//! The decoder of Interactions replies: a whole interaction resource, or a
2//! stream of `event_type`-tagged step events, read as JSON.
3//!
4//! ```
5//! use rig_core::providers::gemini::interactions_api::streaming::InteractionsDecoder;
6//!
7//! let decoder = InteractionsDecoder::default();
8//! # let _ = decoder;
9//! ```
10
11use std::borrow::Cow;
12use std::collections::BTreeMap;
13
14use serde_json::{Map, Value, json};
15
16use crate::completion::{FinishReason, Usage, token_sum};
17use crate::error::ProviderError;
18use crate::json_utils::Lenient;
19use crate::json_utils::remove_preserving_order;
20use crate::message::{
21    DocumentRange, DocumentSourceKind, Image, ImageMediaType, MimeType, Source, SourceLocation,
22};
23use crate::operation::completion::merge;
24use crate::operation::{Block, CallFragment, Completion, Finish};
25use crate::providers::internal::wire;
26use crate::wire::{Decoder, Flow, Out, SpanUnit, WireCitation, WireEvent, WireFrame, WireSpan};
27
28/// Recognized Interactions SSE tags; unlisted tags classify as unknown.
29const KNOWN_EVENT_TYPES: &[&str] = &[
30    "interaction.created",
31    "interaction.completed",
32    "interaction.status_update",
33    "step.start",
34    "step.delta",
35    "step.stop",
36    "error",
37];
38
39/// Top-level keys that mark an untagged frame as a whole interaction
40/// resource.
41const INTERACTION_MARKER_KEYS: &[&str] = &["steps", "status", "usage", "object", "id"];
42
43/// The tag of the unknown payload a [`StreamCursor`] travels in.
44const CURSOR_TYPE: &str = "gemini.stream_cursor";
45
46/// Where a resumable stream picks up: the interaction, and the last
47/// `event_id` the API marked as a resume point. A stream carries one after
48/// `interaction.created`, with no event yet, and one after each event with
49/// an `event_id`, as an [`Item::Unknown`](crate::streaming::Item::Unknown)
50/// that [`Self::of`] reads. A stream resumed with
51/// [`Gemini::interaction_resumed`](crate::providers::gemini::Gemini::interaction_resumed)
52/// continues after the event, or from the start without one, so a step
53/// that was open arrives without its start.
54#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
55pub struct StreamCursor {
56    /// The interaction's id, when the stream named it.
57    pub interaction_id: Option<String>,
58    /// The `event_id` to resume after; `None` resumes from the start.
59    pub event_id: Option<String>,
60}
61
62impl StreamCursor {
63    /// The cursor `item` carries, if it is one.
64    pub fn of<E>(item: &crate::streaming::Item<E>) -> Option<Self> {
65        let crate::streaming::Item::Unknown(payload) = item else {
66            return None;
67        };
68        let payload = payload.value();
69        let field = |key: &str| payload.str(key).map(str::to_owned);
70        (payload.str("type") == Some(CURSOR_TYPE)).then(|| Self {
71            interaction_id: field("interaction_id"),
72            event_id: field("event_id"),
73        })
74    }
75
76    /// The unknown payload that carries this cursor.
77    fn payload(&self) -> crate::streaming::UnknownPayload {
78        crate::streaming::UnknownPayload::new(json!({
79            "type": CURSOR_TYPE,
80            "interaction_id": self.interaction_id,
81            "event_id": self.event_id,
82        }))
83    }
84}
85
86/// A decoded frame, verbatim: the decoder reads the fields it needs.
87pub enum InteractionsEvent {
88    /// One `event_type`-tagged streaming event.
89    Sse(SseEvent),
90    /// The whole interaction resource, as the unary reply delivers it.
91    Whole(Map<String, Value>),
92}
93
94/// A streaming event: its tag, and its other fields verbatim.
95#[derive(Debug)]
96pub struct SseEvent {
97    /// The event's `event_type`.
98    pub event_type: String,
99    /// Every other field.
100    pub fields: Map<String, Value>,
101}
102
103/// Read as one object whose `event_type` is taken out, rather than through
104/// `#[serde(flatten)]`, which buffers every field before building the map.
105impl<'de> serde::Deserialize<'de> for SseEvent {
106    fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
107        let mut fields = Map::deserialize(deserializer)?;
108        match remove_preserving_order(&mut fields, "event_type") {
109            Some(Value::String(event_type)) => Ok(Self { event_type, fields }),
110            Some(_) => Err(serde::de::Error::custom("`event_type` is not a string")),
111            None => Err(serde::de::Error::missing_field("event_type")),
112        }
113    }
114}
115
116/// The Gemini Interactions wire's decoder. Steps decode in step order: a
117/// thought, a call and any other step is one block, and a model output is
118/// one block per content item, whose provider item is the model output step
119/// holding that item alone. A step opens on `step.start`, grows with its
120/// `step.delta`s, and becomes its blocks' provider item on `step.stop`, where
121/// the API states it complete. A whole interaction states each step
122/// complete. A step still open when the reply ends keeps no provider item.
123/// A text item's annotations, stated with it or streamed after it, cite its
124/// block when the step stops.
125#[derive(Default)]
126pub struct InteractionsDecoder {
127    /// The open steps, by wire index.
128    steps: BTreeMap<usize, Open>,
129    /// The model `interaction.created` named, for a completion that does
130    /// not.
131    model: Option<String>,
132    /// The last status a stream reported.
133    status: Option<String>,
134    /// The interaction the stream reads, for its cursors.
135    interaction_id: Option<String>,
136}
137
138/// What an open step decodes to.
139enum Open {
140    Thought,
141    /// A model output: the step's other fields, and each content item with
142    /// its block's writer index.
143    Output(Value, Vec<(usize, Value)>),
144    /// A function call, with the argument JSON streamed so far.
145    Call(String),
146    Opaque,
147}
148
149/// Rig's usage for an interaction's `usage`, read leniently: input is
150/// `total_input_tokens` plus the tool-use tokens, output
151/// `total_output_tokens` plus the thought tokens, and the total their sum;
152/// without a base count, that side and the total stay absent. An error when
153/// a sum overflows.
154pub fn usage_of(usage: &Value) -> Result<Usage, ProviderError> {
155    let tool_use = usage.u64("total_tool_use_tokens");
156    let thoughts = usage.u64("total_thought_tokens");
157    let add = |key: &str, more: Option<u64>| {
158        usage
159            .u64(key)
160            .map(|count| token_sum([count, more.unwrap_or(0)]))
161            .transpose()
162    };
163    let (input_tokens, output_tokens) = (
164        add("total_input_tokens", tool_use)?,
165        add("total_output_tokens", thoughts)?,
166    );
167    let total_tokens = input_tokens
168        .zip(output_tokens)
169        .map(|(input, output)| token_sum([input, output]))
170        .transpose()?;
171    Ok(Usage {
172        input_tokens,
173        output_tokens,
174        cached_input_tokens: usage.u64("total_cached_tokens"),
175        reasoning_tokens: thoughts,
176        tool_use_prompt_tokens: tool_use,
177        total_tokens,
178        cache_creation_input_tokens: None,
179        ..Usage::default()
180    })
181}
182
183/// The image a content item states, when it has data or a URI.
184fn image_of(content: &Value) -> Option<Image> {
185    let data = match (content.str("data"), content.str("uri")) {
186        (Some(data), _) => DocumentSourceKind::Base64(data.to_owned()),
187        (None, Some(uri)) => DocumentSourceKind::Url(uri.to_owned()),
188        (None, None) => return None,
189    };
190    Some(Image {
191        data,
192        media_type: content
193            .str("mime_type")
194            .and_then(ImageMediaType::from_mime_type),
195        detail: None,
196        native: None,
197    })
198}
199
200/// Open the block for `content`, a model output's newest content item.
201fn open_content(
202    content: Value,
203    out: &mut Out<'_, Completion>,
204) -> Result<(usize, Value), ProviderError> {
205    let index = out.fresh_index();
206    let block = match content.str("type") {
207        Some("text") => Block::Text,
208        Some("image") => image_of(&content).map_or(Block::Opaque { replay: true }, Block::Image),
209        _ => Block::Opaque { replay: true },
210    };
211    let text = matches!(block, Block::Text);
212    out.open(index, block, Value::Null)?;
213    if let Some(fragment) = content.str("text").filter(|_| text) {
214        out.push(index, fragment)?;
215    }
216    Ok((index, content))
217}
218
219impl InteractionsDecoder {
220    /// The decoder of a stream that reads the interaction `interaction_id`.
221    pub fn resuming(interaction_id: impl Into<String>) -> Self {
222        Self {
223            interaction_id: Some(interaction_id.into()),
224            ..Self::default()
225        }
226    }
227
228    /// Open the step at `index` as it starts, writing the content it
229    /// already states.
230    fn start(
231        &mut self,
232        index: usize,
233        step: Value,
234        out: &mut Out<'_, Completion>,
235    ) -> Result<(), ProviderError> {
236        let open = match step.str("type") {
237            Some("thought") => {
238                let summary: Vec<String> = step
239                    .arr("summary")
240                    .iter()
241                    .filter(|summary| summary.str("type") == Some("text"))
242                    .filter_map(|summary| summary.str("text").map(str::to_owned))
243                    .collect();
244                out.open(index, Block::Reasoning { redacted: false }, step)?;
245                for text in summary {
246                    out.push(index, &text)?;
247                }
248                Open::Thought
249            }
250            Some("model_output") => {
251                let mut step = step;
252                let content = step
253                    .as_object_mut()
254                    .and_then(|step| step.insert("content".to_owned(), Value::Array(Vec::new())));
255                let items = match content {
256                    Some(Value::Array(items)) => items,
257                    _ => Vec::new(),
258                };
259                let items = items.into_iter().map(|content| open_content(content, out));
260                Open::Output(step, items.collect::<Result<_, _>>()?)
261            }
262            Some("function_call") => {
263                call_fields(index, &step, out)?;
264                out.edit(index, |slot| *slot = step)?;
265                Open::Call(String::new())
266            }
267            // Input a reply restates is kept and never sent back; a
268            // hosted-tool step, or one rig does not know, replays.
269            kind => {
270                let replay = !matches!(kind, Some("user_input" | "function_result"));
271                out.open(index, Block::Opaque { replay }, step)?;
272                Open::Opaque
273            }
274        };
275        self.steps.insert(index, open);
276        Ok(())
277    }
278
279    /// Apply one delta to the step at `index`. A delta for a step that
280    /// never started opens the step it implies: a resumed stream can join
281    /// a step after its start.
282    fn delta(
283        &mut self,
284        index: usize,
285        mut delta: Map<String, Value>,
286        out: &mut Out<'_, Completion>,
287    ) -> Result<(), ProviderError> {
288        let kind = delta
289            .get("type")
290            .and_then(Value::as_str)
291            .unwrap_or_default();
292        if !self.steps.contains_key(&index) {
293            self.start(index, json!({ "type": step_kind(kind) }), out)?;
294        }
295        let Some(open) = self.steps.get_mut(&index) else {
296            return Ok(());
297        };
298        match (open, kind) {
299            // Text and its annotations extend the last text item and its
300            // block; any other delta is a content item of its own.
301            (Open::Output(_, items), _) => match items.last_mut() {
302                Some((block, item))
303                    if matches!(kind, "text" | "text_annotation_delta")
304                        && item.str("type") == Some("text") =>
305                {
306                    if let Some(fragment) = delta.get("text").and_then(Value::as_str)
307                        && kind == "text"
308                    {
309                        out.push(*block, fragment)?;
310                    }
311                    merge(item, &delta);
312                    Ok(())
313                }
314                _ => {
315                    items.push(open_content(Value::Object(delta), out)?);
316                    Ok(())
317                }
318            },
319            (Open::Thought, "thought_summary") => {
320                let content = remove_preserving_order(&mut delta, "content").unwrap_or_default();
321                if let Some(text) = content
322                    .str("text")
323                    .filter(|_| content.str("type") == Some("text"))
324                {
325                    out.push(index, text)?;
326                }
327                let summary = Value::Array(vec![content]);
328                out.edit(index, |item| {
329                    merge(item, &Map::from_iter([("summary".to_owned(), summary)]))
330                })
331            }
332            (Open::Call(arguments), "arguments_delta") => {
333                let fragment = arguments_fragment(&delta);
334                arguments.push_str(&fragment);
335                let fragment = CallFragment {
336                    arguments: Some(&fragment),
337                    ..CallFragment::default()
338                };
339                out.fragment(Some(index), fragment)
340            }
341            (open, _) => {
342                if matches!(open, Open::Call(_)) && kind == "function_call" {
343                    call_fields(index, &Value::Object(delta.clone()), out)?;
344                }
345                // A delta that restates its step (it carries the step's own
346                // `type`) replaces the fields it names; any other merges.
347                out.edit(index, |item| {
348                    if item.get("type") != delta.get("type") {
349                        merge(item, &delta);
350                    } else if let Some(item) = item.as_object_mut() {
351                        item.extend(delta);
352                    }
353                })
354            }
355        }
356    }
357
358    /// Finish the step at `index`: the API states it complete.
359    fn stop(&mut self, index: usize, out: &mut Out<'_, Completion>) -> Result<(), ProviderError> {
360        match self.steps.remove(&index) {
361            None => Ok(()),
362            Some(Open::Call(arguments)) => {
363                // Streamed fragments supersede the arguments the start
364                // announced.
365                if !arguments.is_empty()
366                    && let Ok(arguments) = crate::json_utils::parse_tool_arguments(&arguments)
367                {
368                    out.edit(index, |item| {
369                        if let Some(item) = item.as_object_mut() {
370                            item.insert("arguments".to_owned(), arguments);
371                        }
372                    })?;
373                }
374                out.finish(index)
375            }
376            // An output with no content is one empty text block holding it.
377            Some(Open::Output(step, items)) if items.is_empty() => {
378                let block = out.fresh_index();
379                out.open(block, Block::Text, step)?;
380                out.finish(block)
381            }
382            Some(Open::Output(step, items)) => {
383                for (block, item) in items {
384                    for annotation in item.arr("annotations") {
385                        if let Some(citation) = annotation_citation(annotation) {
386                            out.cite(block, citation);
387                        }
388                    }
389                    let mut step = step.clone();
390                    if let Some(step) = step.as_object_mut() {
391                        step.insert("content".to_owned(), Value::Array(vec![item]));
392                    }
393                    out.edit(block, |slot| *slot = step)?;
394                    out.finish(block)?;
395                }
396                Ok(())
397            }
398            Some(Open::Thought | Open::Opaque) => out.finish(index),
399        }
400    }
401
402    /// End the reply with the interaction resource it completed with.
403    fn complete(
404        &mut self,
405        interaction: Map<String, Value>,
406        out: Out<'_, Completion>,
407    ) -> Result<Flow, ProviderError> {
408        let interaction = Value::Object(interaction);
409        let field = |key: &str| interaction.str(key).map(str::to_owned);
410        // An agent interaction names its agent in place of a model.
411        let model = field("model")
412            .or_else(|| field("agent"))
413            .or_else(|| self.model.take());
414        let status = field("status").or_else(|| self.status.take());
415        let (reason, error) = ending(status.as_deref(), interaction.arr("errors"));
416        let finish = Finish {
417            usage: usage_of(interaction.get("usage").unwrap_or(&Value::Null))?,
418            reason,
419            response_id: field("id"),
420            model,
421            error,
422        };
423        Ok(out.end(finish))
424    }
425}
426
427/// The finish an interaction's `status` states, and the failure it reports.
428/// `completed` and `requires_action` (calls wait for results) are
429/// successes; `incomplete` is the token or execution budget running out,
430/// and `budget_exceeded` its deprecated spelling. `failed` and `cancelled`
431/// are failures carrying the interaction's `errors`; `in_progress` and
432/// `queued` are an interaction read before it ended. An unknown status, or
433/// none, is a failure.
434fn ending(status: Option<&str>, errors: &[Value]) -> (Option<FinishReason>, Option<String>) {
435    let messages: Vec<&str> = errors
436        .iter()
437        .filter_map(|error| error.str("message"))
438        .collect();
439    let detail = (!messages.is_empty()).then(|| format!(": {}", messages.join("; ")));
440    let detail = detail.unwrap_or_default();
441    let other = |status: &str| Some(FinishReason::Other(status.to_owned()));
442    match status {
443        Some("completed") => (Some(FinishReason::Stop), None),
444        Some("requires_action") => (Some(FinishReason::ToolCalls), None),
445        Some("incomplete" | "budget_exceeded") => (Some(FinishReason::Length), None),
446        Some(status @ ("failed" | "cancelled")) => (
447            other(status),
448            Some(format!("The interaction {status}{detail}")),
449        ),
450        Some(status @ ("in_progress" | "queued")) => (
451            other(status),
452            Some(format!("The interaction was read while {status}")),
453        ),
454        Some(status) => (other(status), None),
455        None => (None, Some("The interaction states no status".to_owned())),
456    }
457}
458
459/// The citation a text item's annotation states: a `url_citation` cites a
460/// web page, a `file_citation` a document and its page, a `place_citation`
461/// a place by its URL or id. Offsets count bytes of the item's text.
462/// Other annotations cite nothing.
463fn annotation_citation(annotation: &Value) -> Option<WireCitation> {
464    let field = |key: &str| annotation.str(key).map(str::to_owned);
465    let (location, title) = match annotation.str("type")? {
466        "url_citation" => (SourceLocation::Url { url: field("url")? }, field("title")),
467        "file_citation" => {
468            let page = annotation
469                .u64("page_number")
470                .and_then(|page| u32::try_from(page).ok());
471            let within = page.map(|page| DocumentRange::Pages(page..page.saturating_add(1)));
472            let id = field("document_uri");
473            (
474                SourceLocation::Document {
475                    index: None,
476                    id,
477                    within,
478                },
479                field("file_name"),
480            )
481        }
482        "place_citation" => {
483            let location = match (field("url"), field("place_id")) {
484                (Some(url), _) => SourceLocation::Url { url },
485                (None, Some(id)) => SourceLocation::Document {
486                    index: None,
487                    id: Some(id),
488                    within: None,
489                },
490                (None, None) => return None,
491            };
492            (location, field("name"))
493        }
494        _ => return None,
495    };
496    let mut source = Source::new(location);
497    if let Some(title) = title {
498        source = source.title(title);
499    }
500    // Proto3 JSON leaves out a zero start.
501    let span = annotation.u64("end_index").map(|end| {
502        WireSpan::new(
503            annotation.u64("start_index").unwrap_or(0),
504            end,
505            SpanUnit::Bytes,
506        )
507    });
508    Some(WireCitation::new(span, vec![source]))
509}
510
511/// The kind of step a `delta` type belongs to, for a step no `step.start`
512/// opened.
513fn step_kind(delta: &str) -> &str {
514    match delta {
515        "text" | "text_annotation_delta" | "image" | "audio" | "document" | "video" => {
516            "model_output"
517        }
518        "thought_summary" | "thought_signature" => "thought",
519        "arguments_delta" => "function_call",
520        other => other,
521    }
522}
523
524/// The arguments text an `arguments_delta` adds: its string, or the JSON of
525/// any other value.
526fn arguments_fragment(delta: &Map<String, Value>) -> Cow<'_, str> {
527    match delta.get("arguments") {
528        Some(Value::String(fragment)) => Cow::Borrowed(fragment),
529        Some(Value::Null) | None => Cow::Borrowed(""),
530        Some(other) => Cow::Owned(other.to_string()),
531    }
532}
533
534/// Write the id, name and announced arguments a call step states.
535fn call_fields(
536    index: usize,
537    step: &Value,
538    out: &mut Out<'_, Completion>,
539) -> Result<(), ProviderError> {
540    let (id, name) = (step.str("id"), step.str("name"));
541    out.fragment(
542        Some(index),
543        CallFragment {
544            id,
545            name,
546            arguments: None,
547        },
548    )?;
549    step.get("arguments")
550        .map_or(Ok(()), |arguments| out.announce(index, arguments.clone()))
551}
552
553/// EOF without `interaction.completed` is truncation, not successful
554/// completion, so the decoder has nothing to add at the end of the reply.
555impl<'id> Decoder<'id, Completion> for InteractionsDecoder {
556    type Event = InteractionsEvent;
557
558    /// Classify a frame by its `event_type` tag, or as a whole resource by
559    /// its marker keys when it carries no tag. A tagged frame never passes
560    /// for a whole resource.
561    fn classify(&self, frame: WireFrame) -> WireEvent<InteractionsEvent> {
562        wire::classify_or_untagged(
563            &frame.as_str(),
564            "event_type",
565            |tag| KNOWN_EVENT_TYPES.contains(&tag),
566            InteractionsEvent::Sse,
567            |data| {
568                wire::classify_marker_keyed_frame(data, INTERACTION_MARKER_KEYS)
569                    .map(InteractionsEvent::Whole)
570            },
571        )
572    }
573
574    fn decode(
575        &mut self,
576        event: InteractionsEvent,
577        mut out: Out<'id, Completion>,
578    ) -> Result<Flow, ProviderError> {
579        let SseEvent {
580            event_type,
581            fields: event,
582        } = match event {
583            InteractionsEvent::Sse(event) => event,
584            InteractionsEvent::Whole(mut interaction) => {
585                let steps = match remove_preserving_order(&mut interaction, "steps") {
586                    Some(Value::Array(steps)) => steps,
587                    _ => Vec::new(),
588                };
589                for (index, step) in steps.into_iter().enumerate() {
590                    self.start(index, step, &mut out)?;
591                    self.stop(index, &mut out)?;
592                }
593                return self.complete(interaction, out);
594            }
595        };
596        let mut event = Value::Object(event);
597        let index = |event: &Value| {
598            event
599                .u64("index")
600                .and_then(|index| usize::try_from(index).ok())
601                .ok_or_else(|| {
602                    ProviderError::Response(format!(
603                        "an Interactions `{event_type}` event names no step index"
604                    ))
605                })
606        };
607        let malformed = |field: &str| {
608            ProviderError::Response(format!(
609                "an Interactions `{event_type}` event carries no `{field}` object"
610            ))
611        };
612        let event_id = event.str("event_id").map(str::to_owned);
613        let resumable = event_id.is_some() || event_type == "interaction.created";
614        match event_type.as_str() {
615            "step.start" => match event.get_mut("step").map(Value::take) {
616                // A start that states no step leaves its deltas to open it.
617                None => {}
618                Some(step @ Value::Object(_)) => self.start(index(&event)?, step, &mut out)?,
619                Some(_) => return Err(malformed("step")),
620            },
621            "step.delta" => match event.get_mut("delta").map(Value::take) {
622                Some(Value::Object(delta)) => self.delta(index(&event)?, delta, &mut out)?,
623                _ => return Err(malformed("delta")),
624            },
625            "step.stop" => self.stop(index(&event)?, &mut out)?,
626            "interaction.created" => {
627                let field = |key: &str| {
628                    event
629                        .at(&format!("/interaction/{key}"))
630                        .and_then(Value::as_str)
631                        .map(str::to_owned)
632                };
633                self.model = field("model").or_else(|| field("agent"));
634                self.status = field("status");
635                if let Some(id) = field("id") {
636                    self.interaction_id = Some(id);
637                }
638            }
639            "interaction.status_update" => {
640                if let Some(status) = event.str("status") {
641                    self.status = Some(status.to_owned());
642                }
643            }
644            "interaction.completed" => {
645                let interaction = match event.get_mut("interaction").map(Value::take) {
646                    Some(Value::Object(interaction)) => interaction,
647                    _ => Map::new(),
648                };
649                return self.complete(interaction, out);
650            }
651            "error" => {
652                let Value::Object(mut body) = event else {
653                    return Err(ProviderError::from_provider_body(event_type));
654                };
655                body.insert("event_type".to_owned(), Value::from(event_type));
656                return Err(ProviderError::from_provider_body(
657                    Value::Object(body).to_string(),
658                ));
659            }
660            _ => {}
661        }
662        if resumable {
663            let cursor = StreamCursor {
664                interaction_id: self.interaction_id.clone(),
665                event_id,
666            };
667            out.unknown(cursor.payload());
668        }
669        Ok(Flow::More)
670    }
671}
672
673pub mod document;
674
675#[cfg(test)]
676mod tests;