Skip to main content

rig_agent/agent/
streaming.rs

1//! Streamed agent events, bounded event feeds, and terminal response shaping.
2//!
3//! ```no_run
4//! # fn example(agent: &rig_agent::Agent) {
5//! let stream = agent.prompt("Explain ownership.").stream();
6//! # }
7//! ```
8
9use rig_core::{
10    message::AssistantContent,
11    wasm_compat::{WasmBoxedStream, WasmCompatSend},
12};
13
14use crate::{
15    agent::engine::{DriveItem, LiveTool, StreamingTurnSource, Surfaced, drive_agent},
16    agent::runner::AgentRunner,
17    streaming::{Item, StreamEvent},
18};
19use futures::{SinkExt, Stream, StreamExt, channel::mpsc, stream::FusedStream};
20use serde::{Deserialize, Serialize};
21use std::io::Write;
22use std::pin::Pin;
23use tracing_futures::Instrument;
24
25use crate::completion::PromptError;
26use crate::run::response::{CompletionCall, PromptResponse};
27
28/// The stream a streamed run yields: its items, then its ending.
29pub type StreamingResult = WasmBoxedStream<'static, Result<MultiTurnStreamItem, PromptError>>;
30
31#[derive(Serialize, Deserialize, Debug, Clone)]
32#[serde(tag = "type", rename_all = "camelCase")]
33/// One item of a streamed run: a provider stream event, a committed tool
34/// call, a lifecycle marker, or the run's final response. An item reads back
35/// from its serialized form on its own, without checking its order against
36/// the items around it; [`StreamRecord::parse`](rig_core::streaming::StreamRecord::parse)
37/// checks a run's provider stream items.
38#[non_exhaustive]
39pub enum MultiTurnStreamItem {
40    /// A provider stream item containing model-emitted content: part
41    /// starts and ends, text and reasoning fragments, a tool call's
42    /// argument fragments as they stream (a call to a tool the turn does not
43    /// allow is held for its resolution), and unmodeled passthrough
44    /// payloads. A call's end carries the validated call; the same call id
45    /// is reported again as [`ToolCall`](Self::ToolCall) once history holds it.
46    StreamAssistantItem(Item<StreamEvent>),
47    /// A tool call **committed to the run's history**, in commit order.
48    ///
49    /// The stream's tool items are a projection of committed history
50    /// ([`project`](crate::run::project)): every call history holds is
51    /// reported, and later answered by a [`ToolResult`](Self::ToolResult),
52    /// whether it ran, a hook skipped it, or invalid-call recovery or an
53    /// output reprompt answered it; a run resumed mid-batch reports the batch's
54    /// calls again ([`projection_start`](crate::run::AgentRun::projection_start)).
55    /// It is **not** an execution-lifecycle event
56    /// (see [`ToolExecutionCommitted`](Self::ToolExecutionCommitted)). A
57    /// structured-output Tool-mode output-tool call that finalizes the run is
58    /// committed as text, so it is reported in the
59    /// [`FinalResponse`](Self::FinalResponse) instead.
60    ToolCall {
61        /// The call as the model emitted it. Its id is equal on its
62        /// execution commit and its result.
63        tool_call: rig_core::message::ToolCall,
64    },
65    /// A tool call **starts**: Rig dispatches it now, before its dispatch
66    /// hooks and its body run. The call goes on only once the stream's
67    /// consumer has taken this item, so a slow consumer delays the call. A
68    /// [`RunEvents`] feed takes it into its buffer, so there the call may run
69    /// before the feed's reader sees it.
70    ///
71    /// One item per dispatched call, after the turn's
72    /// [`ToolCall`](Self::ToolCall) items and before the batch's commits and
73    /// results: in call order when calls run one at a time, in start order
74    /// under `tool_concurrency`. A call a dispatch hook skips still starts.
75    /// A call answered by invalid-call recovery, or dropped because a
76    /// sibling ended the run, never starts. A start is not a commit: a
77    /// batch that fails surfaces its starts but no commits or results.
78    ToolExecutionStarted {
79        /// The call as the model emitted it, before any hook rewrite. Its
80        /// id is the one its [`ToolCall`](Self::ToolCall), execution commit
81        /// and result carry.
82        tool_call: rig_core::message::ToolCall,
83    },
84    /// One item of a tool's **accepted output**, reported as soon as the
85    /// call's outcome hooks accept it, without waiting for the rest of the
86    /// batch. The call goes on only once the stream's consumer has taken the
87    /// item, as for [`ToolExecutionStarted`](Self::ToolExecutionStarted).
88    ///
89    /// An output that is exactly one JSON array is reported one element at a
90    /// time, each as one JSON block (an empty array reports nothing); any
91    /// other output is one item holding all its blocks. Text is never
92    /// reparsed as JSON. Items come in completion order under
93    /// `tool_concurrency`, so items of different calls may interleave: group
94    /// them by `tool_result.call`. Every started call that answers reports its items,
95    /// a hook-skipped one included; a call answered by invalid-call recovery
96    /// reports none. An item is not a commit: items stay reported when a
97    /// sibling later fails the batch, a call already running when a sibling
98    /// fails still reports its items as it finishes, and the committed
99    /// [`ToolResult`](Self::ToolResult) still follows a batch that settles.
100    /// Show these for progress; record only the `ToolResult`.
101    ToolResultItem {
102        /// This item's content, with the call id, name and error flag of the
103        /// result the batch commits for the call.
104        tool_result: rig_core::message::ToolResult,
105        /// The item's position in the output, from zero.
106        index: usize,
107        /// How many items the output has.
108        count: usize,
109    },
110    /// Confirmation that Rig **executed and committed** a tool call. It is
111    /// surfaced together with its `ToolResult` only after the whole batch
112    /// settles successfully; the live start is
113    /// [`ToolExecutionStarted`](Self::ToolExecutionStarted).
114    ///
115    /// This item is emitted only for a tool whose body actually ran (it passed
116    /// its `ToolCall` hook checks), never for a call dropped by a sibling's
117    /// termination, skipped by a hook, or resolved by invalid-call recovery.
118    /// Correlate it with the model call and result through the call's id.
119    ToolExecutionCommitted {
120        /// The tool call as **executed**: the model's call with any
121        /// [`DispatchAction::Patch`](crate::agent::DispatchAction::Patch) hook rewrite
122        /// applied (so a redaction rewrite is reflected here, not leaked). The
123        /// model's *original* call is reported via
124        /// [`StreamAssistantItem`](Self::StreamAssistantItem).
125        tool_call: rig_core::message::ToolCall,
126    },
127    /// A tool result **committed to the run's history**, answering a reported
128    /// [`ToolCall`](Self::ToolCall). The tool
129    /// batch commits and surfaces atomically at every `tool_concurrency`
130    /// (including the sequential default): results are surfaced in call order
131    /// only after the whole batch settles successfully; a run that terminates
132    /// mid-batch surfaces no successful tool results. The live output is
133    /// [`ToolResultItem`](Self::ToolResultItem).
134    ToolResult {
135        /// The result; `tool_result.call` is the id of the call it answers.
136        tool_result: rig_core::message::ToolResult,
137    },
138    /// Details for one successfully completed completion request made by this agent stream.
139    ///
140    /// This is emitted when a provider call finishes. Usage is the provider's
141    /// final usage for that completion request when available; it is not
142    /// incremental per streamed token.
143    ///
144    /// ```
145    /// use rig_agent::agent::MultiTurnStreamItem;
146    /// fn input_tokens(item: &MultiTurnStreamItem) -> Option<u64> {
147    ///     match item {
148    ///         MultiTurnStreamItem::CompletionCall(call) => call.usage.input_tokens,
149    ///         _ => None,
150    ///     }
151    /// }
152    /// ```
153    CompletionCall(CompletionCall),
154    /// The completed model turn was rejected by a hook for retry.
155    ///
156    /// Text and reasoning deltas emitted for this turn were provisional. A
157    /// consumer should discard or visually reset output associated with `turn`.
158    /// A subsequent attempt is made only if the run's total model-call budget
159    /// permits it.
160    ModelTurnRetried {
161        /// One-based model-call index of the rejected turn.
162        turn: usize,
163    },
164    /// The final result from the stream: the unified [`PromptResponse`] shared
165    /// with the blocking surface.
166    ///
167    /// Terminal for the run: no retry, further turn, or tool execution follows.
168    /// This item is the stream-side
169    /// counterpart of the `on_run_settled` hook's success outcome. Error
170    /// termination surfaces as the stream's `Err` item instead, which is
171    /// equally terminal.
172    FinalResponse(PromptResponse),
173}
174
175/// The one mapping from what the engine surfaces directly; tool items are
176/// built only from committed history.
177impl From<Surfaced> for MultiTurnStreamItem {
178    fn from(surfaced: Surfaced) -> Self {
179        match surfaced {
180            Surfaced::Provider(item) => Self::StreamAssistantItem(item),
181            Surfaced::CompletionCall(call) => Self::CompletionCall(call),
182            Surfaced::ModelTurnRetried { turn } => Self::ModelTurnRetried { turn },
183            Surfaced::Final(response) => Self::FinalResponse(response),
184        }
185    }
186}
187
188impl From<LiveTool> for MultiTurnStreamItem {
189    fn from(live: LiveTool) -> Self {
190        match live {
191            LiveTool::Started(tool_call) => Self::ToolExecutionStarted { tool_call },
192            LiveTool::Output {
193                tool_result,
194                index,
195                count,
196            } => Self::ToolResultItem {
197                tool_result,
198                index,
199                count,
200            },
201        }
202    }
203}
204
205impl AgentRunner {
206    /// Drive the agent loop as a stream of assistant content, tool activity
207    /// and, last, the [`FinalResponse`](MultiTurnStreamItem::FinalResponse).
208    /// Hooks fire at every observable point, including streamed text and
209    /// tool-call deltas.
210    ///
211    /// Like [`run`](AgentRunner::run), this is lazy: agent span creation
212    /// begins only when the stream is first polled, and a stream that is
213    /// dropped unpolled has done nothing. The stream is `Send` on
214    /// native targets, so it can be built in synchronous code and handed to
215    /// whatever polls it; it runs under the span it was built in (a caller's
216    /// enabled span is adopted, otherwise a root `invoke_agent` is created),
217    /// not under whichever span first polls it.
218    ///
219    /// ```rust,no_run
220    /// # use rig_agent::{Agent, agent::StreamingResult};
221    /// fn start(agent: &Agent, prompt: &str) -> StreamingResult {
222    ///     // Nothing runs until whoever holds this polls it.
223    ///     agent.prompt(prompt).stream()
224    /// }
225    /// ```
226    ///
227    /// Shares the drive loop, run construction, tool execution and fail-closed
228    /// hook handling with the blocking [`run`](AgentRunner::run) via
229    /// `drive_agent`, so the two behave identically apart from the streamed
230    /// delta events.
231    #[must_use = "a stream does nothing until polled"]
232    pub fn stream(self) -> StreamingResult {
233        // The span the stream is built under is the one it runs under, not
234        // whichever span first polls it: a host may build the stream in a
235        // request span and hand it to a task of its own.
236        self.stream_under(tracing::Span::current())
237    }
238
239    /// Build a lazy stream under an explicitly captured ambient span.
240    fn stream_under(self, ambient: tracing::Span) -> StreamingResult {
241        let run_under = ambient.clone();
242        let stream = async_stream::stream! {
243            let mut inner = self.start_stream(run_under);
244            while let Some(item) = inner.next().await {
245                yield item;
246            }
247        };
248        Box::pin(stream.instrument(ambient))
249    }
250
251    /// The eager half of [`stream`](Self::stream): build the run and return
252    /// the driver as a stream. Called on the first poll, under `ambient`, the
253    /// span the stream was built in.
254    fn start_stream(self, ambient: tracing::Span) -> StreamingResult {
255        let source = StreamingTurnSource::new(
256            &self.config.hooks,
257            self.agent_name_or_default().to_string(),
258            self.config.record_telemetry_content,
259        );
260        let driver = drive_agent(self, source, ambient).flat_map(|item| {
261            let (head, tools) = match item {
262                Ok(DriveItem::Surfaced(item)) => (Some(Ok(item.into())), None),
263                Ok(DriveItem::Live(live)) => (Some(Ok(live.into())), None),
264                Ok(DriveItem::Projected(items)) => (None, Some(items)),
265                Ok(DriveItem::Done(_)) => (None, None),
266                Err(err) => (Some(Err(err)), None),
267            };
268            futures::stream::iter(head.into_iter().chain(tools.into_iter().flatten().map(Ok)))
269        });
270        Box::pin(driver)
271    }
272}
273
274/// Capacity of the event queue behind [`AgentRunner::run_channel`]: the number
275/// of [`MultiTurnStreamItem`]s the run may buffer ahead of the consumer before
276/// it parks on back-pressure.
277pub const RUN_EVENTS_CAPACITY: usize = 32;
278
279/// Event feed of an agent run started with [`AgentRunner::run_channel`].
280///
281/// Every [`MultiTurnStreamItem`] the run would have streamed is delivered here
282/// in order, ending with [`MultiTurnStreamItem::FinalResponse`]. The feed is a
283/// bounded queue ([`RUN_EVENTS_CAPACITY`]): a slow consumer applies
284/// back-pressure to the run instead of losing events. Poll it as a
285/// [`Stream`] from async code, or drain it with the non-blocking
286/// [`try_next`](RunEvents::try_next) from a synchronous tick, such as a game loop,
287/// UI frame, an ECS system.
288///
289/// Dropping the feed does not cancel the run; it simply stops receiving events
290/// and the run future still resolves with the final
291/// [`PromptResponse`].
292#[derive(Debug)]
293pub struct RunEvents {
294    receiver: mpsc::Receiver<MultiTurnStreamItem>,
295}
296
297impl RunEvents {
298    /// Take the next buffered event without waiting.
299    ///
300    /// Returns `None` both when no event is queued yet and once the run has
301    /// finished and the feed is drained; use [`is_done`](RunEvents::is_done)
302    /// to tell the two apart.
303    pub fn try_next(&mut self) -> Option<MultiTurnStreamItem> {
304        self.receiver.try_recv().ok()
305    }
306
307    /// Whether the run has finished and every event has been taken.
308    pub fn is_done(&self) -> bool {
309        self.receiver.is_terminated()
310    }
311}
312
313impl Stream for RunEvents {
314    type Item = MultiTurnStreamItem;
315
316    fn poll_next(
317        mut self: Pin<&mut Self>,
318        cx: &mut std::task::Context<'_>,
319    ) -> std::task::Poll<Option<Self::Item>> {
320        Pin::new(&mut self.receiver).poll_next(cx)
321    }
322
323    fn size_hint(&self) -> (usize, Option<usize>) {
324        self.receiver.size_hint()
325    }
326}
327
328impl FusedStream for RunEvents {
329    fn is_terminated(&self) -> bool {
330        self.receiver.is_terminated()
331    }
332}
333
334impl AgentRunner {
335    /// Split the run into a driving future and a [`RunEvents`] feed.
336    ///
337    /// The future performs the same agent loop as
338    /// [`run`](AgentRunner::run) and [`stream`](AgentRunner::stream), and
339    /// resolves with the final [`PromptResponse`]; the
340    /// feed receives each intermediate [`MultiTurnStreamItem`] as it happens.
341    /// Spawn the future on any executor and poll the feed from wherever the
342    /// events are consumed; neither side assumes a runtime.
343    ///
344    /// The feed is bounded ([`RUN_EVENTS_CAPACITY`]); when it is full the run
345    /// waits for the consumer rather than dropping events. Dropping the feed
346    /// lets the run continue to completion unobserved. Like
347    /// [`stream`](Self::stream), the run belongs to the span this method was
348    /// called in, wherever the future is later polled.
349    #[must_use = "the run does nothing until the future is driven"]
350    pub fn run_channel(
351        self,
352    ) -> (
353        impl Future<Output = Result<PromptResponse, PromptError>> + WasmCompatSend,
354        RunEvents,
355    ) {
356        let (mut sender, receiver) = mpsc::channel(RUN_EVENTS_CAPACITY);
357        // Captured here, not when the future is first polled: the doc above
358        // says to spawn the future, and the run must still belong to the span
359        // that split it.
360        let ambient = tracing::Span::current();
361        let future = async move {
362            let mut stream = self.stream_under(ambient);
363            let mut response = None;
364            let mut forward = true;
365            while let Some(item) = stream.next().await {
366                let item = match item {
367                    Ok(item) => item,
368                    Err(err) => {
369                        // Drain after the terminal error so span and lineage teardown
370                        // completes instead of being dropped at the yield.
371                        while stream.next().await.is_some() {}
372                        return Err(err);
373                    }
374                };
375                match item {
376                    MultiTurnStreamItem::FinalResponse(done) => {
377                        if forward {
378                            let _ = sender
379                                .send(MultiTurnStreamItem::FinalResponse(done.clone()))
380                                .await;
381                        }
382                        response = Some(done);
383                    }
384                    item => {
385                        if forward && sender.send(item).await.is_err() {
386                            forward = false;
387                        }
388                    }
389                }
390            }
391            response.ok_or_else(super::runner::no_final_response)
392        };
393        (future, RunEvents { receiver })
394    }
395}
396
397/// Why [`stream_to_stdout`] returned no response, or why
398/// [`ChatBot::run`](crate::integrations::cli_chatbot::ChatBot::run) stopped.
399#[derive(Debug, thiserror::Error)]
400pub enum StreamToStdoutError {
401    /// The run failed.
402    #[error(transparent)]
403    Run(#[from] PromptError),
404    /// Writing to stdout failed.
405    #[error(transparent)]
406    Io(#[from] std::io::Error),
407    /// The stream ended without a final response.
408    #[error("the stream ended without a final response")]
409    Incomplete,
410}
411
412/// Print a streamed run's assistant text and reasoning to stdout and return
413/// its final response.
414///
415/// Streaming metadata events, such as `MultiTurnStreamItem::CompletionCall`,
416/// are not printed; metadata is returned on the [`PromptResponse`] via
417/// accessors such as [`PromptResponse::completion_calls`]. A model-turn retry
418/// prints a visible boundary because text already written to stdout cannot be
419/// retracted. Fails with the run's error as soon as the stream yields it, a
420/// stdout write failure, or [`StreamToStdoutError::Incomplete`] when the
421/// stream ends without a final response.
422pub async fn stream_to_stdout(
423    stream: &mut StreamingResult,
424) -> Result<PromptResponse, StreamToStdoutError> {
425    let mut stdout = std::io::stdout();
426    write!(stdout, "Response: ")?;
427    while let Some(content) = stream.next().await {
428        match content? {
429            MultiTurnStreamItem::StreamAssistantItem(Item::Event(StreamEvent::Text {
430                text,
431                ..
432            })) => {
433                write!(stdout, "{text}")?;
434                stdout.flush()?;
435            }
436            MultiTurnStreamItem::StreamAssistantItem(Item::Event(StreamEvent::End {
437                content: AssistantContent::Reasoning(reasoning),
438                ..
439            })) => {
440                write!(stdout, "{}", reasoning.text)?;
441                stdout.flush()?;
442            }
443            MultiTurnStreamItem::FinalResponse(response) => return Ok(response),
444            MultiTurnStreamItem::ModelTurnRetried { turn } => {
445                write!(
446                    stdout,
447                    "\n[model turn {turn} rejected; retry requested]\nResponse: "
448                )?;
449                stdout.flush()?;
450            }
451            _ => {}
452        }
453    }
454    Err(StreamToStdoutError::Incomplete)
455}
456
457#[cfg(test)]
458mod cut_off_tool_call_tests;
459#[cfg(test)]
460mod malformed_tool_args_tests;
461#[cfg(test)]
462mod tests;
463#[cfg(test)]
464mod tool_result_item_tests;
465#[cfg(test)]
466mod tool_started_tests;