Skip to main content

rig_agent/agent/
streaming.rs

1//! Streamed agent events, bounded event feeds, and terminal response shaping.
2//!
3//! ```no_run
4//! # fn example(agent: &rig_agent::Agent) {
5//! let stream = agent.prompt("Explain ownership.").stream();
6//! # }
7//! ```
8
9use rig_core::error::ProviderError;
10use rig_core::{
11    message::AssistantContent,
12    wasm_compat::{WasmBoxedStream, WasmCompatSend},
13};
14
15use crate::{
16    agent::engine::{DriveItem, StreamingTurnSource, Surfaced, drive_agent},
17    agent::runner::AgentRunner,
18    streaming::{Item, StreamEvent},
19};
20use futures::{SinkExt, Stream, StreamExt, channel::mpsc, stream::FusedStream};
21use serde::{Deserialize, Serialize};
22use std::io::Write;
23use std::pin::Pin;
24use tracing_futures::Instrument;
25
26use crate::completion::PromptError;
27use crate::run::response::{CompletionCall, PromptResponse};
28
29/// The stream a streamed run yields: its items, then its ending.
30pub type StreamingResult = WasmBoxedStream<'static, Result<MultiTurnStreamItem, PromptError>>;
31
32#[derive(Serialize, Deserialize, Debug, Clone)]
33#[serde(tag = "type", rename_all = "camelCase")]
34/// One item of a streamed run: a provider stream event, a committed tool
35/// call, a lifecycle marker, or the run's final response. An item reads back
36/// from its serialized form on its own, without checking its order against
37/// the items around it; [`StreamRecord::parse`](rig_core::streaming::StreamRecord::parse)
38/// checks a run's provider stream items.
39#[non_exhaustive]
40pub enum MultiTurnStreamItem {
41    /// A provider stream item containing model-emitted content: part
42    /// starts and ends, text and reasoning fragments, a tool call's
43    /// argument fragments as they stream (a call to a tool the turn does not
44    /// allow is held for its resolution), and unmodeled passthrough
45    /// payloads. A call's end carries the validated call; the same call id
46    /// is reported again as [`ToolCall`](Self::ToolCall) once history holds it.
47    StreamAssistantItem(Item<StreamEvent>),
48    /// A tool call **committed to the run's history**, in commit order.
49    ///
50    /// The stream's tool items are a projection of committed history
51    /// ([`project`](crate::run::project)): every call history holds is
52    /// reported, and later answered by a [`ToolResult`](Self::ToolResult),
53    /// whether it ran, a hook skipped it, or invalid-call recovery or an
54    /// output reprompt answered it; a run resumed mid-batch reports the batch's
55    /// calls again ([`projection_start`](crate::run::AgentRun::projection_start)).
56    /// It is **not** an execution-lifecycle event
57    /// (see [`ToolExecutionCommitted`](Self::ToolExecutionCommitted)). A
58    /// structured-output Tool-mode output-tool call that finalizes the run is
59    /// committed as text, so it is reported in the
60    /// [`FinalResponse`](Self::FinalResponse) instead.
61    ToolCall {
62        /// The call as the model emitted it. Its id is equal on its
63        /// execution commit and its result.
64        tool_call: rig_core::message::ToolCall,
65    },
66    /// Confirmation that Rig **executed and committed** a tool call. This is not
67    /// a real-time start notification: it is surfaced together with its
68    /// `ToolResult` only after the whole batch settles successfully. Use tool
69    /// hooks for live host-side start/result observation.
70    ///
71    /// This item is emitted only for a tool whose body actually ran (it passed
72    /// its `ToolCall` hook checks), never for a call dropped by a sibling's
73    /// termination, skipped by a hook, or resolved by invalid-call recovery.
74    /// Correlate it with the model call and result through the call's id.
75    ToolExecutionCommitted {
76        /// The tool call as **executed**: the model's call with any
77        /// [`DispatchAction::Patch`](crate::agent::DispatchAction::Patch) hook rewrite
78        /// applied (so a redaction rewrite is reflected here, not leaked). The
79        /// model's *original* call is reported via
80        /// [`StreamAssistantItem`](Self::StreamAssistantItem).
81        tool_call: rig_core::message::ToolCall,
82    },
83    /// A tool result **committed to the run's history**, answering a reported
84    /// [`ToolCall`](Self::ToolCall). The tool
85    /// batch commits and surfaces atomically at every `tool_concurrency`
86    /// (including the sequential default): results are surfaced in call order
87    /// only after the whole batch settles successfully; a run that terminates
88    /// mid-batch surfaces no successful tool results.
89    ToolResult {
90        /// The result; `tool_result.call` is the id of the call it answers.
91        tool_result: rig_core::message::ToolResult,
92    },
93    /// Details for one successfully completed completion request made by this agent stream.
94    ///
95    /// This is emitted when a provider call finishes. Usage is the provider's
96    /// final usage for that completion request when available; it is not
97    /// incremental per streamed token.
98    ///
99    /// ```
100    /// use rig_agent::agent::MultiTurnStreamItem;
101    /// fn input_tokens(item: &MultiTurnStreamItem) -> Option<u64> {
102    ///     match item {
103    ///         MultiTurnStreamItem::CompletionCall(call) => call.usage.input_tokens,
104    ///         _ => None,
105    ///     }
106    /// }
107    /// ```
108    CompletionCall(CompletionCall),
109    /// The completed model turn was rejected by a hook for retry.
110    ///
111    /// Text and reasoning deltas emitted for this turn were provisional. A
112    /// consumer should discard or visually reset output associated with `turn`.
113    /// A subsequent attempt is made only if the run's total model-call budget
114    /// permits it.
115    ModelTurnRetried {
116        /// One-based model-call index of the rejected turn.
117        turn: usize,
118    },
119    /// The final result from the stream: the unified [`PromptResponse`] shared
120    /// with the blocking surface.
121    ///
122    /// Terminal for the run: no retry, further turn, or tool execution follows.
123    /// This item is the stream-side
124    /// counterpart of the `on_run_settled` hook's success outcome. Error
125    /// termination surfaces as the stream's `Err` item instead, which is
126    /// equally terminal.
127    FinalResponse(PromptResponse),
128}
129
130/// The one mapping from what the engine surfaces directly; tool items are
131/// built only from committed history.
132impl From<Surfaced> for MultiTurnStreamItem {
133    fn from(surfaced: Surfaced) -> Self {
134        match surfaced {
135            Surfaced::Provider(item) => Self::StreamAssistantItem(item),
136            Surfaced::CompletionCall(call) => Self::CompletionCall(call),
137            Surfaced::ModelTurnRetried { turn } => Self::ModelTurnRetried { turn },
138            Surfaced::Final(response) => Self::FinalResponse(response),
139        }
140    }
141}
142
143impl MultiTurnStreamItem {
144    /// Build a final response from structured content and aggregate usage.
145    /// Concatenates text for output; completion details and history remain empty.
146    pub fn final_response(
147        content: Vec<AssistantContent>,
148        aggregated_usage: crate::completion::Usage,
149    ) -> Self {
150        Self::FinalResponse(PromptResponse::from_content(content, aggregated_usage))
151    }
152}
153
154impl AgentRunner {
155    /// Drive the agent loop as a stream of assistant content, tool activity
156    /// and, last, the [`FinalResponse`](MultiTurnStreamItem::FinalResponse).
157    /// Hooks fire at every observable point, including streamed text and
158    /// tool-call deltas.
159    ///
160    /// Like [`run`](AgentRunner::run), this is lazy: agent span creation
161    /// begins only when the stream is first polled, and a stream that is
162    /// dropped unpolled has done nothing. The stream is `Send` on
163    /// native targets, so it can be built in synchronous code and handed to
164    /// whatever polls it; it runs under the span it was built in (a caller's
165    /// enabled span is adopted, otherwise a root `invoke_agent` is created),
166    /// not under whichever span first polls it.
167    ///
168    /// ```rust,no_run
169    /// # use rig_agent::{Agent, agent::StreamingResult};
170    /// fn start(agent: &Agent, prompt: &str) -> StreamingResult {
171    ///     // Nothing runs until whoever holds this polls it.
172    ///     agent.prompt(prompt).stream()
173    /// }
174    /// ```
175    ///
176    /// Shares the drive loop, run construction, tool execution and fail-closed
177    /// hook handling with the blocking [`run`](AgentRunner::run) via
178    /// `drive_agent`, so the two behave identically apart from the streamed
179    /// delta events.
180    #[must_use = "a stream does nothing until polled"]
181    pub fn stream(self) -> StreamingResult {
182        // The span the stream is built under is the one it runs under, not
183        // whichever span first polls it: a host may build the stream in a
184        // request span and hand it to a task of its own.
185        self.stream_under(tracing::Span::current())
186    }
187
188    /// Build a lazy stream under an explicitly captured ambient span.
189    fn stream_under(self, ambient: tracing::Span) -> StreamingResult {
190        let run_under = ambient.clone();
191        let stream = async_stream::stream! {
192            let mut inner = self.start_stream(run_under);
193            while let Some(item) = inner.next().await {
194                yield item;
195            }
196        };
197        Box::pin(stream.instrument(ambient))
198    }
199
200    /// The eager half of [`stream`](Self::stream): build the run and return
201    /// the driver as a stream. Called on the first poll, under `ambient`, the
202    /// span the stream was built in.
203    fn start_stream(self, ambient: tracing::Span) -> StreamingResult {
204        let (agent_span, created_agent_span) = self.open_agent_span(ambient);
205
206        let hook_ctx = self.hook_context(true);
207        let run = self.build_run();
208        let source = StreamingTurnSource::new(
209            &self.config.hooks,
210            self.agent_name_or_default().to_string(),
211            created_agent_span,
212            self.config.record_telemetry_content,
213        );
214
215        let driver = drive_agent(
216            self,
217            source,
218            run,
219            agent_span.clone(),
220            created_agent_span,
221            hook_ctx,
222        )
223        .flat_map(|item| {
224            let (head, tools) = match item {
225                Ok(DriveItem::Surfaced(item)) => (Some(Ok(item.into())), None),
226                Ok(DriveItem::Projected(items)) => (None, Some(items)),
227                Ok(DriveItem::Done(_)) => (None, None),
228                Err(err) => (Some(Err(err)), None),
229            };
230            futures::stream::iter(head.into_iter().chain(tools.into_iter().flatten().map(Ok)))
231        });
232        Box::pin(driver.instrument(agent_span))
233    }
234}
235
236/// Capacity of the event queue behind [`AgentRunner::run_channel`]: the number
237/// of [`MultiTurnStreamItem`]s the run may buffer ahead of the consumer before
238/// it parks on back-pressure.
239pub const RUN_EVENTS_CAPACITY: usize = 32;
240
241/// Event feed of an agent run started with [`AgentRunner::run_channel`].
242///
243/// Every [`MultiTurnStreamItem`] the run would have streamed is delivered here
244/// in order, ending with [`MultiTurnStreamItem::FinalResponse`]. The feed is a
245/// bounded queue ([`RUN_EVENTS_CAPACITY`]): a slow consumer applies
246/// back-pressure to the run instead of losing events. Poll it as a
247/// [`Stream`] from async code, or drain it with the non-blocking
248/// [`try_next`](RunEvents::try_next) from a synchronous tick, such as a game loop,
249/// UI frame, an ECS system.
250///
251/// Dropping the feed does not cancel the run; it simply stops receiving events
252/// and the run future still resolves with the final
253/// [`PromptResponse`].
254#[derive(Debug)]
255pub struct RunEvents {
256    receiver: mpsc::Receiver<MultiTurnStreamItem>,
257}
258
259impl RunEvents {
260    /// Take the next buffered event without waiting.
261    ///
262    /// Returns `None` both when no event is queued yet and once the run has
263    /// finished and the feed is drained; use [`is_done`](RunEvents::is_done)
264    /// to tell the two apart.
265    pub fn try_next(&mut self) -> Option<MultiTurnStreamItem> {
266        self.receiver.try_recv().ok()
267    }
268
269    /// Whether the run has finished and every event has been taken.
270    pub fn is_done(&self) -> bool {
271        self.receiver.is_terminated()
272    }
273}
274
275impl Stream for RunEvents {
276    type Item = MultiTurnStreamItem;
277
278    fn poll_next(
279        mut self: Pin<&mut Self>,
280        cx: &mut std::task::Context<'_>,
281    ) -> std::task::Poll<Option<Self::Item>> {
282        Pin::new(&mut self.receiver).poll_next(cx)
283    }
284
285    fn size_hint(&self) -> (usize, Option<usize>) {
286        self.receiver.size_hint()
287    }
288}
289
290impl FusedStream for RunEvents {
291    fn is_terminated(&self) -> bool {
292        self.receiver.is_terminated()
293    }
294}
295
296impl AgentRunner {
297    /// Split the run into a driving future and a [`RunEvents`] feed.
298    ///
299    /// The future performs the same agent loop as
300    /// [`run`](AgentRunner::run) and [`stream`](AgentRunner::stream), and
301    /// resolves with the final [`PromptResponse`]; the
302    /// feed receives each intermediate [`MultiTurnStreamItem`] as it happens.
303    /// Spawn the future on any executor and poll the feed from wherever the
304    /// events are consumed; neither side assumes a runtime.
305    ///
306    /// The feed is bounded ([`RUN_EVENTS_CAPACITY`]); when it is full the run
307    /// waits for the consumer rather than dropping events. Dropping the feed
308    /// lets the run continue to completion unobserved. Like
309    /// [`stream`](Self::stream), the run belongs to the span this method was
310    /// called in, wherever the future is later polled.
311    #[must_use = "the run does nothing until the future is driven"]
312    pub fn run_channel(
313        self,
314    ) -> (
315        impl Future<Output = Result<PromptResponse, PromptError>> + WasmCompatSend,
316        RunEvents,
317    ) {
318        let (mut sender, receiver) = mpsc::channel(RUN_EVENTS_CAPACITY);
319        // Captured here, not when the future is first polled: the doc above
320        // says to spawn the future, and the run must still belong to the span
321        // that split it.
322        let ambient = tracing::Span::current();
323        let future = async move {
324            let mut stream = self.stream_under(ambient);
325            let mut response = None;
326            let mut forward = true;
327            while let Some(item) = stream.next().await {
328                let item = match item {
329                    Ok(item) => item,
330                    Err(err) => {
331                        // Drain after the terminal error so span and lineage teardown
332                        // completes instead of being dropped at the yield.
333                        while stream.next().await.is_some() {}
334                        return Err(err);
335                    }
336                };
337                match item {
338                    MultiTurnStreamItem::FinalResponse(done) => {
339                        if forward {
340                            let _ = sender
341                                .send(MultiTurnStreamItem::FinalResponse(done.clone()))
342                                .await;
343                        }
344                        response = Some(done);
345                    }
346                    item => {
347                        if forward && sender.send(item).await.is_err() {
348                            forward = false;
349                        }
350                    }
351                }
352            }
353            response.ok_or_else(|| {
354                PromptError::Provider(ProviderError::Response(
355                    "agent run ended without producing a final response".to_string(),
356                ))
357            })
358        };
359        (future, RunEvents { receiver })
360    }
361}
362
363/// Why [`stream_to_stdout`] returned no response.
364#[derive(Debug, thiserror::Error)]
365pub enum StreamToStdoutError {
366    /// The run failed.
367    #[error(transparent)]
368    Run(#[from] PromptError),
369    /// Writing to stdout failed.
370    #[error(transparent)]
371    Io(#[from] std::io::Error),
372    /// The stream ended without a final response.
373    #[error("the stream ended without a final response")]
374    Incomplete,
375}
376
377/// Print a streamed run's assistant text and reasoning to stdout and return
378/// its final response.
379///
380/// Streaming metadata events, such as `MultiTurnStreamItem::CompletionCall`,
381/// are not printed; metadata is returned on the [`PromptResponse`] via
382/// accessors such as [`PromptResponse::completion_calls`]. A model-turn retry
383/// prints a visible boundary because text already written to stdout cannot be
384/// retracted. Fails with the run's error as soon as the stream yields it, a
385/// stdout write failure, or [`StreamToStdoutError::Incomplete`] when the
386/// stream ends without a final response.
387pub async fn stream_to_stdout(
388    stream: &mut StreamingResult,
389) -> Result<PromptResponse, StreamToStdoutError> {
390    let mut stdout = std::io::stdout();
391    write!(stdout, "Response: ")?;
392    while let Some(content) = stream.next().await {
393        match content? {
394            MultiTurnStreamItem::StreamAssistantItem(Item::Event(StreamEvent::Text {
395                text,
396                ..
397            })) => {
398                write!(stdout, "{text}")?;
399                stdout.flush()?;
400            }
401            MultiTurnStreamItem::StreamAssistantItem(Item::Event(StreamEvent::End {
402                content: AssistantContent::Reasoning(reasoning),
403                ..
404            })) => {
405                write!(stdout, "{}", reasoning.text)?;
406                stdout.flush()?;
407            }
408            MultiTurnStreamItem::FinalResponse(response) => return Ok(response),
409            MultiTurnStreamItem::ModelTurnRetried { turn } => {
410                write!(
411                    stdout,
412                    "\n[model turn {turn} rejected; retry requested]\nResponse: "
413                )?;
414                stdout.flush()?;
415            }
416            _ => {}
417        }
418    }
419    Err(StreamToStdoutError::Incomplete)
420}
421
422#[cfg(test)]
423mod malformed_tool_args_tests;
424#[cfg(test)]
425#[allow(irrefutable_let_patterns, unreachable_patterns)]
426mod tests;