Skip to main content

rig_agent/agent/
streaming.rs

1//! Streamed agent events, bounded event feeds, and terminal response shaping.
2//!
3//! ```no_run
4//! # fn example(agent: &rig_agent::Agent) {
5//! let stream = agent.prompt("Explain ownership.").stream();
6//! # }
7//! ```
8
9use rig_core::error::ProviderError;
10use rig_core::{
11    message::AssistantContent,
12    wasm_compat::{WasmBoxedStream, WasmCompatSend},
13};
14
15use crate::{
16    agent::engine::{DriveItem, StreamingTurnSource, drive_agent},
17    agent::hook::{AgentHook, RunSettled, SettledOutcome, StepEventKind},
18    agent::runner::{AgentRunner, RunOrigin},
19    streaming::{Item, StreamEvent},
20};
21use futures::{SinkExt, Stream, StreamExt, channel::mpsc, stream::FusedStream};
22use serde::{Deserialize, Serialize};
23use std::io::Write;
24use std::pin::Pin;
25use tracing_futures::Instrument;
26
27use crate::completion::PromptError;
28use crate::run::response::{CompletionCall, PromptResponse};
29use rig_core::message::Message;
30
31/// The stream a streamed run yields: its items, then its ending.
32pub type StreamingResult = WasmBoxedStream<'static, Result<MultiTurnStreamItem, PromptError>>;
33
34#[derive(Serialize, Deserialize, Debug, Clone)]
35#[serde(tag = "type", rename_all = "camelCase")]
36/// One item of a streamed run: a provider stream event, a committed tool
37/// call, a lifecycle marker, or the run's final response. An item reads back
38/// from its serialized form on its own, without checking its order against
39/// the items around it; [`Transcript::parse`](rig_core::streaming::Transcript::parse)
40/// checks a run's provider stream items.
41#[non_exhaustive]
42pub enum MultiTurnStreamItem {
43    /// A provider stream item containing model-emitted content: part
44    /// starts and ends, text and reasoning fragments, a tool call's
45    /// argument fragments as they stream (a call to a tool the turn does not
46    /// allow is held for its resolution), and unmodeled passthrough
47    /// payloads. A call's end carries the validated call. The model's
48    /// completed calls are also reported as [`ToolCall`](Self::ToolCall)
49    /// when the turn commits.
50    StreamAssistantItem(Item<StreamEvent>),
51    /// A tool call the **model emitted**, reported when the model turn is
52    /// committed, for each call Rig routes to execution. Such a call is
53    /// reported whether or not the tool body ultimately runs (a hook skip
54    /// still reports it); it is **not** an execution-lifecycle event (see
55    /// [`ToolExecutionCommitted`](Self::ToolExecutionCommitted)).
56    ///
57    /// Two kinds of model tool call are **not** reported here: a call rejected and
58    /// handled by invalid-tool-call recovery (surfaced via that recovery
59    /// path), and a structured-output Tool-mode output-tool call, which
60    /// finalizes the run directly; its structured result is surfaced in
61    /// the [`FinalResponse`](Self::FinalResponse) rather than as a completed
62    /// call.
63    ToolCall {
64        /// The call as the model emitted it. Its id is equal on its
65        /// execution commit and its result.
66        tool_call: rig_core::message::ToolCall,
67    },
68    /// Confirmation that Rig **executed and committed** a tool call. This is not
69    /// a real-time start notification: it is surfaced together with its
70    /// `ToolResult` only after the whole batch settles successfully. Use tool
71    /// hooks for live host-side start/result observation.
72    ///
73    /// This item is emitted only for a tool whose body actually ran (it passed
74    /// its `ToolCall` hook checks), never for a call dropped by a sibling's
75    /// termination, skipped by a hook, or resolved by invalid-call recovery.
76    /// Correlate it with the model call and result through the call's id.
77    ToolExecutionCommitted {
78        /// The tool call as **executed**: the model's call with any
79        /// [`DispatchAction::Patch`](crate::agent::DispatchAction::Patch) hook rewrite
80        /// applied (so a redaction rewrite is reflected here, not leaked). The
81        /// model's *original* call is reported via
82        /// [`StreamAssistantItem`](Self::StreamAssistantItem).
83        tool_call: rig_core::message::ToolCall,
84    },
85    /// The **result** of an executed (or hook-skipped) tool call. The tool
86    /// batch commits and surfaces atomically at every `tool_concurrency`
87    /// (including the sequential default): results are surfaced in call order
88    /// only after the whole batch settles successfully; a run that terminates
89    /// mid-batch surfaces no successful tool results.
90    ToolResult {
91        /// The result; `tool_result.call` is the id of the call it answers.
92        tool_result: rig_core::message::ToolResult,
93    },
94    /// Details for one successfully completed completion request made by this agent stream.
95    ///
96    /// This is emitted when a provider call finishes. Usage is the provider's
97    /// final usage for that completion request when available; it is not
98    /// incremental per streamed token.
99    ///
100    /// ```
101    /// use rig_agent::agent::MultiTurnStreamItem;
102    /// fn input_tokens(item: &MultiTurnStreamItem) -> Option<u64> {
103    ///     match item {
104    ///         MultiTurnStreamItem::CompletionCall(call) => call.usage.input_tokens,
105    ///         _ => None,
106    ///     }
107    /// }
108    /// ```
109    CompletionCall(CompletionCall),
110    /// The completed model turn was rejected by a hook for retry.
111    ///
112    /// Text and reasoning deltas emitted for this turn were provisional. A
113    /// consumer should discard or visually reset output associated with `turn`.
114    /// A subsequent attempt is made only if the run's total model-call budget
115    /// permits it.
116    ModelTurnRetried {
117        /// One-based model-call index of the rejected turn.
118        turn: usize,
119    },
120    /// The final result from the stream: the unified [`PromptResponse`] shared
121    /// with the blocking surface.
122    ///
123    /// Terminal for the run: no retry, further turn, or tool execution follows.
124    /// This item is the stream-side
125    /// counterpart of the `on_run_settled` hook's success outcome. Error
126    /// termination surfaces as the stream's `Err` item instead, which is
127    /// equally terminal.
128    FinalResponse(PromptResponse),
129}
130
131/// Build the unified [`PromptResponse`] for the streaming surface from the
132/// final turn's structured content.
133fn final_response_from_content(
134    content: Vec<AssistantContent>,
135    aggregated_usage: crate::completion::Usage,
136    completion_calls: Vec<CompletionCall>,
137    history: Vec<Message>,
138) -> PromptResponse {
139    PromptResponse::from_content(content, aggregated_usage)
140        .with_completion_calls(completion_calls)
141        .with_messages(history)
142}
143
144impl MultiTurnStreamItem {
145    pub(crate) fn stream_item(item: Item<StreamEvent>) -> Self {
146        Self::StreamAssistantItem(item)
147    }
148
149    /// Stamp a `FinalResponse` item with how the run's memory append
150    /// settled; any other item is returned unchanged.
151    pub(crate) fn with_memory_append(
152        self,
153        memory_append: Option<crate::run::MemoryAppend>,
154    ) -> Self {
155        match self {
156            Self::FinalResponse(response) => {
157                Self::FinalResponse(response.with_memory_append(memory_append))
158            }
159            other => other,
160        }
161    }
162
163    /// Build a final response from structured content and aggregate usage.
164    /// Concatenates text for output; completion details and history remain empty.
165    pub fn final_response(
166        content: Vec<AssistantContent>,
167        aggregated_usage: crate::completion::Usage,
168    ) -> Self {
169        Self::FinalResponse(final_response_from_content(
170            content,
171            aggregated_usage,
172            Vec::new(),
173            Vec::new(),
174        ))
175    }
176
177    pub(crate) fn final_response_with_completion_calls(
178        content: Vec<AssistantContent>,
179        aggregated_usage: crate::completion::Usage,
180        completion_calls: Vec<CompletionCall>,
181        history: Vec<Message>,
182    ) -> Self {
183        Self::FinalResponse(final_response_from_content(
184            content,
185            aggregated_usage,
186            completion_calls,
187            history,
188        ))
189    }
190}
191
192impl AgentRunner {
193    /// Drive the agent loop as a stream of assistant content, tool activity
194    /// and, last, the [`FinalResponse`](MultiTurnStreamItem::FinalResponse).
195    /// Hooks fire at every observable point, including streamed text and
196    /// tool-call deltas.
197    ///
198    /// Like [`run`](AgentRunner::run), this is lazy: memory loading and agent
199    /// span creation begin only when the stream is first polled,
200    /// and a stream that is dropped unpolled has done nothing. A memory-load
201    /// failure is the stream's first (and only) item. The stream is `Send` on
202    /// native targets, so it can be built in synchronous code and handed to
203    /// whatever polls it; it runs under the span it was built in (a caller's
204    /// enabled span is adopted, otherwise a root `invoke_agent` is created),
205    /// not under whichever span first polls it.
206    ///
207    /// ```rust,no_run
208    /// # use rig_agent::{Agent, agent::StreamingResult};
209    /// fn start(agent: &Agent, prompt: &str) -> StreamingResult {
210    ///     // Nothing runs until whoever holds this polls it.
211    ///     agent.prompt(prompt).stream()
212    /// }
213    /// ```
214    ///
215    /// Shares the drive loop, run construction, tool execution and fail-closed
216    /// hook handling with the blocking [`run`](AgentRunner::run) via
217    /// `drive_agent`, so the two behave identically apart from the streamed
218    /// delta events.
219    #[must_use = "a stream does nothing until polled"]
220    pub fn stream(self) -> StreamingResult {
221        // The span the stream is built under is the one it runs under, not
222        // whichever span first polls it: a host may build the stream in a
223        // request span and hand it to a task of its own.
224        self.stream_under(tracing::Span::current())
225    }
226
227    /// Build a lazy stream under an explicitly captured ambient span.
228    fn stream_under(self, ambient: tracing::Span) -> StreamingResult {
229        let run_under = ambient.clone();
230        let stream = async_stream::stream! {
231            let mut inner = self.start_stream(run_under).await;
232            while let Some(item) = inner.next().await {
233                yield item;
234            }
235        };
236        Box::pin(stream.instrument(ambient))
237    }
238
239    /// The eager half of [`stream`](Self::stream): resolve memory, build the
240    /// run and return the driver as a stream. Called on the first poll,
241    /// under `ambient`, the span the stream was built in.
242    async fn start_stream(self, ambient: tracing::Span) -> StreamingResult {
243        let (agent_span, created_agent_span) = self.open_agent_span(ambient);
244
245        let bus = self.config.bus.clone();
246        let hook_ctx = self.hook_context(true);
247        // A resumed run loads nothing and appends what it adds (see `run`).
248        let resolved = match &self.origin {
249            RunOrigin::Resume(_) => self.resumed_memory(),
250            RunOrigin::Prompt(_) => {
251                let resolve = self.resolve_history_and_memory(&hook_ctx);
252                futures::pin_mut!(resolve);
253                let mut driven = bus.drive(futures::stream::once(resolve));
254                driven.next().await.unwrap_or(Ok((None, None)))
255            }
256        };
257        let (history_override, memory_handle) = match resolved {
258            Ok(resolved) => resolved,
259            Err(err) => {
260                // Notify settlement before yielding the error so consumers stopping
261                // at the first failure cannot suppress the lifecycle notification.
262                let hooks = self.config.hooks.clone();
263                let stream = async_stream::stream! {
264                    let err = PromptError::from(err);
265                    if hooks.observes(StepEventKind::RunSettled) {
266                        let reason = err.to_string();
267                        hooks
268                            .on_run_settled(
269                                &hook_ctx,
270                                RunSettled {
271                                    outcome: SettledOutcome::Error(&reason),
272                                    messages: None,
273                                },
274                            )
275                            .await;
276                    }
277                    yield Err(err);
278                };
279                // Instrument under the agent span like the success path so
280                // a load failure stays tied to invoke_agent.
281                return Box::pin(stream.instrument(agent_span));
282            }
283        };
284
285        let run = self.build_run(history_override);
286        let source = StreamingTurnSource::new(
287            &self.config.hooks,
288            self.agent_name_or_default().to_string(),
289            created_agent_span,
290            self.config.record_telemetry_content,
291        );
292
293        let driver = drive_agent(
294            self,
295            source,
296            run,
297            agent_span.clone(),
298            created_agent_span,
299            memory_handle,
300            hook_ctx,
301        )
302        .filter_map(|item| {
303            std::future::ready(match item {
304                Ok(DriveItem::Item(item)) => Some(Ok(item)),
305                Ok(DriveItem::Done(_)) => None,
306                Err(err) => Some(Err(err)),
307            })
308        });
309        // The consumer of this stream drives the agent's bus: every poll that
310        // leaves the run pending polls the driver.
311        let driver = bus.drive(Box::pin(driver));
312
313        Box::pin(driver.instrument(agent_span))
314    }
315}
316
317/// Capacity of the event queue behind [`AgentRunner::run_channel`]: the number
318/// of [`MultiTurnStreamItem`]s the run may buffer ahead of the consumer before
319/// it parks on back-pressure.
320pub const RUN_EVENTS_CAPACITY: usize = 32;
321
322/// Event feed of an agent run started with [`AgentRunner::run_channel`].
323///
324/// Every [`MultiTurnStreamItem`] the run would have streamed is delivered here
325/// in order, ending with [`MultiTurnStreamItem::FinalResponse`]. The feed is a
326/// bounded queue ([`RUN_EVENTS_CAPACITY`]): a slow consumer applies
327/// back-pressure to the run instead of losing events. Poll it as a
328/// [`Stream`] from async code, or drain it with the non-blocking
329/// [`try_next`](RunEvents::try_next) from a synchronous tick, such as a game loop,
330/// UI frame, an ECS system.
331///
332/// Dropping the feed does not cancel the run; it simply stops receiving events
333/// and the run future still resolves with the final
334/// [`PromptResponse`].
335#[derive(Debug)]
336pub struct RunEvents {
337    receiver: mpsc::Receiver<MultiTurnStreamItem>,
338}
339
340impl RunEvents {
341    /// Take the next buffered event without waiting.
342    ///
343    /// Returns `None` both when no event is queued yet and once the run has
344    /// finished and the feed is drained; use [`is_done`](RunEvents::is_done)
345    /// to tell the two apart.
346    pub fn try_next(&mut self) -> Option<MultiTurnStreamItem> {
347        self.receiver.try_recv().ok()
348    }
349
350    /// Whether the run has finished and every event has been taken.
351    pub fn is_done(&self) -> bool {
352        self.receiver.is_terminated()
353    }
354}
355
356impl Stream for RunEvents {
357    type Item = MultiTurnStreamItem;
358
359    fn poll_next(
360        mut self: Pin<&mut Self>,
361        cx: &mut std::task::Context<'_>,
362    ) -> std::task::Poll<Option<Self::Item>> {
363        Pin::new(&mut self.receiver).poll_next(cx)
364    }
365
366    fn size_hint(&self) -> (usize, Option<usize>) {
367        self.receiver.size_hint()
368    }
369}
370
371impl FusedStream for RunEvents {
372    fn is_terminated(&self) -> bool {
373        self.receiver.is_terminated()
374    }
375}
376
377impl AgentRunner {
378    /// Split the run into a driving future and a [`RunEvents`] feed.
379    ///
380    /// The future performs the same agent loop as
381    /// [`run`](AgentRunner::run) and [`stream`](AgentRunner::stream), and
382    /// resolves with the final [`PromptResponse`]; the
383    /// feed receives each intermediate [`MultiTurnStreamItem`] as it happens.
384    /// Spawn the future on any executor and poll the feed from wherever the
385    /// events are consumed; neither side assumes a runtime.
386    ///
387    /// The feed is bounded ([`RUN_EVENTS_CAPACITY`]); when it is full the run
388    /// waits for the consumer rather than dropping events. Dropping the feed
389    /// lets the run continue to completion unobserved. Like
390    /// [`stream`](Self::stream), the run belongs to the span this method was
391    /// called in, wherever the future is later polled.
392    #[must_use = "the run does nothing until the future is driven"]
393    pub fn run_channel(
394        self,
395    ) -> (
396        impl Future<Output = Result<PromptResponse, PromptError>> + WasmCompatSend,
397        RunEvents,
398    ) {
399        let (mut sender, receiver) = mpsc::channel(RUN_EVENTS_CAPACITY);
400        // Captured here, not when the future is first polled: the doc above
401        // says to spawn the future, and the run must still belong to the span
402        // that split it.
403        let ambient = tracing::Span::current();
404        let future = async move {
405            let mut stream = self.stream_under(ambient);
406            let mut response = None;
407            let mut forward = true;
408            while let Some(item) = stream.next().await {
409                let item = match item {
410                    Ok(item) => item,
411                    Err(err) => {
412                        // Drain after the terminal error so span and lineage teardown
413                        // completes instead of being dropped at the yield.
414                        while stream.next().await.is_some() {}
415                        return Err(err);
416                    }
417                };
418                match item {
419                    MultiTurnStreamItem::FinalResponse(done) => {
420                        if forward {
421                            let _ = sender
422                                .send(MultiTurnStreamItem::FinalResponse(done.clone()))
423                                .await;
424                        }
425                        response = Some(done);
426                    }
427                    item => {
428                        if forward && sender.send(item).await.is_err() {
429                            forward = false;
430                        }
431                    }
432                }
433            }
434            response.ok_or_else(|| {
435                PromptError::Provider(ProviderError::Response(
436                    "agent run ended without producing a final response".to_string(),
437                ))
438            })
439        };
440        (future, RunEvents { receiver })
441    }
442}
443
444/// Why [`stream_to_stdout`] returned no response.
445#[derive(Debug, thiserror::Error)]
446pub enum StreamToStdoutError {
447    /// The run failed.
448    #[error(transparent)]
449    Run(#[from] PromptError),
450    /// Writing to stdout failed.
451    #[error(transparent)]
452    Io(#[from] std::io::Error),
453    /// The stream ended without a final response.
454    #[error("the stream ended without a final response")]
455    Incomplete,
456}
457
458/// Print a streamed run's assistant text and reasoning to stdout and return
459/// its final response.
460///
461/// Streaming metadata events, such as `MultiTurnStreamItem::CompletionCall`,
462/// are not printed; metadata is returned on the [`PromptResponse`] via
463/// accessors such as [`PromptResponse::completion_calls`]. A model-turn retry
464/// prints a visible boundary because text already written to stdout cannot be
465/// retracted. Fails with the run's error as soon as the stream yields it, a
466/// stdout write failure, or [`StreamToStdoutError::Incomplete`] when the
467/// stream ends without a final response.
468pub async fn stream_to_stdout(
469    stream: &mut StreamingResult,
470) -> Result<PromptResponse, StreamToStdoutError> {
471    let mut stdout = std::io::stdout();
472    write!(stdout, "Response: ")?;
473    while let Some(content) = stream.next().await {
474        match content? {
475            MultiTurnStreamItem::StreamAssistantItem(Item::Event(StreamEvent::Text {
476                text,
477                ..
478            })) => {
479                write!(stdout, "{text}")?;
480                stdout.flush()?;
481            }
482            MultiTurnStreamItem::StreamAssistantItem(Item::Event(StreamEvent::End {
483                content: AssistantContent::Reasoning(reasoning),
484                ..
485            })) => {
486                write!(stdout, "{}", reasoning.text)?;
487                stdout.flush()?;
488            }
489            MultiTurnStreamItem::FinalResponse(response) => return Ok(response),
490            MultiTurnStreamItem::ModelTurnRetried { turn } => {
491                write!(
492                    stdout,
493                    "\n[model turn {turn} rejected; retry requested]\nResponse: "
494                )?;
495                stdout.flush()?;
496            }
497            _ => {}
498        }
499    }
500    Err(StreamToStdoutError::Incomplete)
501}
502
503#[cfg(test)]
504mod malformed_tool_args_tests;
505#[cfg(test)]
506#[allow(irrefutable_let_patterns, unreachable_patterns)]
507mod tests;