Skip to main content

rig_agent/agent/
streaming.rs

1//! Streamed agent events, bounded event feeds, and terminal response shaping.
2//!
3//! ```no_run
4//! # fn example(agent: &rig_agent::Agent) {
5//! let stream = agent.prompt("Explain ownership.").stream();
6//! # }
7//! ```
8
9use rig_core::error::ProviderError;
10use rig_core::{message::AssistantContent, wasm_compat::WasmCompatSend};
11
12use crate::{
13    agent::engine::{DriveItem, StreamingTurnSource, drive_agent, streaming_error_into_prompt},
14    agent::hook::{AgentHook, RunSettled, SettledOutcome, StepEventKind},
15    agent::runner::{AgentRunner, RunOrigin},
16    streaming::{Item, StreamEvent, StreamedUserContent},
17};
18use futures::{SinkExt, Stream, StreamExt, channel::mpsc, stream::FusedStream};
19use serde::Serialize;
20use std::pin::Pin;
21use tracing_futures::Instrument;
22
23use crate::completion::PromptError;
24use crate::run::response::{CompletionCall, PromptResponse};
25use crate::run::transcript::assistant_text_from_choice;
26use rig_core::message::Message;
27
28// Match core's browser-only relaxed bounds; a bare wasm32 check would also
29// relax Send on WASI, where core still requires it.
30/// The stream a streamed run yields: its items, then its ending.
31#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))]
32pub type StreamingResult =
33    Pin<Box<dyn Stream<Item = Result<MultiTurnStreamItem, StreamingError>> + Send>>;
34
35/// The stream a streamed run yields: its items, then its ending.
36#[cfg(all(target_arch = "wasm32", target_os = "unknown"))]
37pub type StreamingResult = Pin<Box<dyn Stream<Item = Result<MultiTurnStreamItem, StreamingError>>>>;
38
39#[derive(Serialize, Debug, Clone)]
40#[serde(tag = "type", rename_all = "camelCase")]
41/// One item of a streamed run: a provider stream event, a committed tool
42/// call, a lifecycle marker, or the run's final response.
43pub enum MultiTurnStreamItem {
44    /// A provider stream item containing model-emitted content: part
45    /// starts and ends, text and reasoning fragments, the call parts of a
46    /// validated tool call, and unmodeled passthrough payloads. The model's
47    /// completed calls are also reported as [`ToolCall`](Self::ToolCall)
48    /// when the turn commits.
49    StreamAssistantItem(Item<StreamEvent>),
50    /// A tool call the **model emitted**, reported when the model turn is
51    /// committed, for each call Rig routes to execution. Such a call is
52    /// reported whether or not the tool body ultimately runs (a hook skip
53    /// still reports it); it is **not** an execution-lifecycle event (see
54    /// [`ToolExecutionCommitted`](Self::ToolExecutionCommitted)).
55    ///
56    /// Two kinds of model tool call are **not** reported here: a call rejected and
57    /// handled by invalid-tool-call recovery (surfaced via that recovery
58    /// path), and a structured-output Tool-mode output-tool call, which
59    /// finalizes the run directly; its structured result is surfaced in
60    /// the [`FinalResponse`](Self::FinalResponse) rather than as a completed
61    /// call.
62    ToolCall {
63        /// The call as the model emitted it. Its id is equal on its
64        /// execution commit and its result.
65        tool_call: rig_core::message::ToolCall,
66    },
67    /// Confirmation that Rig **executed and committed** a tool call. This is not
68    /// a real-time start notification: it is surfaced together with its
69    /// `ToolResult` only after the whole batch settles successfully. Use tool
70    /// hooks for live host-side start/result observation.
71    ///
72    /// This item is emitted only for a tool whose body actually ran (it passed
73    /// its `ToolCall` hook checks), never for a call dropped by a sibling's
74    /// termination, skipped by a hook, or resolved by invalid-call recovery.
75    /// Correlate it with the model call and result through the call's id.
76    ToolExecutionCommitted {
77        /// The tool call as **executed**: the model's call with any
78        /// [`DispatchAction::Patch`](crate::agent::DispatchAction::Patch) hook rewrite
79        /// applied (so a redaction rewrite is reflected here, not leaked). The
80        /// model's *original* call is reported via
81        /// [`StreamAssistantItem`](Self::StreamAssistantItem).
82        tool_call: rig_core::message::ToolCall,
83    },
84    /// A streamed user content item: the **result** of an executed (or
85    /// hook-skipped) tool call. The tool batch commits and surfaces atomically at
86    /// every `tool_concurrency` (including the sequential default): results are
87    /// surfaced in call order only after the whole batch settles successfully;
88    /// a run that terminates mid-batch surfaces no successful tool results.
89    StreamUserItem(StreamedUserContent),
90    /// Details for one successfully completed completion request made by this agent stream.
91    ///
92    /// This is emitted when a provider call finishes. Usage is the provider's
93    /// final usage for that completion request when available; it is not
94    /// incremental per streamed token.
95    ///
96    /// ```
97    /// use rig_agent::agent::MultiTurnStreamItem;
98    /// fn input_tokens(item: &MultiTurnStreamItem) -> Option<u64> {
99    ///     match item {
100    ///         MultiTurnStreamItem::CompletionCall(call) => call.usage.input_tokens,
101    ///         _ => None,
102    ///     }
103    /// }
104    /// ```
105    CompletionCall(CompletionCall),
106    /// The completed model turn was rejected by a hook for retry.
107    ///
108    /// Text and reasoning deltas emitted for this turn were provisional. A
109    /// consumer should discard or visually reset output associated with `turn`.
110    /// A subsequent attempt is made only if the run's total model-call budget
111    /// permits it.
112    ModelTurnRetried {
113        /// One-based model-call index of the rejected turn.
114        turn: usize,
115    },
116    /// The final result from the stream: the unified [`PromptResponse`] shared
117    /// with the blocking surface.
118    ///
119    /// Terminal for the run: no retry, further turn, or tool execution follows.
120    /// This item is the stream-side
121    /// counterpart of the `on_run_settled` hook's success outcome. Error
122    /// termination surfaces as the stream's `Err` item instead, which is
123    /// equally terminal.
124    FinalResponse(PromptResponse),
125}
126
127/// Build the unified [`PromptResponse`] for the streaming surface from the
128/// final turn's structured content.
129fn final_response_from_content(
130    content: Vec<AssistantContent>,
131    aggregated_usage: crate::completion::Usage,
132    completion_calls: Vec<CompletionCall>,
133    history: Option<Vec<Message>>,
134) -> PromptResponse {
135    let mut response = PromptResponse::new(assistant_text_from_choice(&content), aggregated_usage)
136        .with_content(content)
137        .with_completion_calls(completion_calls);
138    response.messages = history;
139    response
140}
141
142impl MultiTurnStreamItem {
143    pub(crate) fn stream_item(item: Item<StreamEvent>) -> Self {
144        Self::StreamAssistantItem(item)
145    }
146
147    /// Stamp a `FinalResponse` item with how the run's memory append
148    /// settled; any other item is returned unchanged.
149    pub(crate) fn with_memory_append(
150        self,
151        memory_append: Option<crate::run::MemoryAppend>,
152    ) -> Self {
153        match self {
154            Self::FinalResponse(response) => {
155                Self::FinalResponse(response.with_memory_append(memory_append))
156            }
157            other => other,
158        }
159    }
160
161    /// Build a final response from structured content and aggregate usage.
162    /// Concatenates text for output; completion details and history remain unset.
163    pub fn final_response(
164        content: Vec<AssistantContent>,
165        aggregated_usage: crate::completion::Usage,
166    ) -> Self {
167        Self::FinalResponse(final_response_from_content(
168            content,
169            aggregated_usage,
170            Vec::new(),
171            None,
172        ))
173    }
174
175    pub(crate) fn final_response_with_completion_calls(
176        content: Vec<AssistantContent>,
177        aggregated_usage: crate::completion::Usage,
178        completion_calls: Vec<CompletionCall>,
179        history: Option<Vec<Message>>,
180    ) -> Self {
181        Self::FinalResponse(final_response_from_content(
182            content,
183            aggregated_usage,
184            completion_calls,
185            history,
186        ))
187    }
188}
189
190/// Replace calls and prose in a finished output-tool turn with final output text,
191/// retaining reasoning and images. Call only after finalization, when remaining
192/// calls are output-tool calls. Persisted history retains prose independently;
193/// returns `None` if no call is present.
194pub(crate) fn finalize_streamed_choice(
195    last_final_choice: &[AssistantContent],
196    output: &str,
197) -> Option<Vec<AssistantContent>> {
198    let finalized_via_output_tool = last_final_choice
199        .iter()
200        .any(|item| matches!(item, AssistantContent::ToolCall(_)));
201    if !finalized_via_output_tool {
202        return None;
203    }
204    let mut items: Vec<AssistantContent> = last_final_choice
205        .iter()
206        .filter(|item| {
207            !matches!(
208                item,
209                AssistantContent::ToolCall(_) | AssistantContent::Text(_)
210            )
211        })
212        .cloned()
213        .collect();
214    items.push(AssistantContent::text(output.to_string()));
215    Some(items)
216}
217
218/// Why a streamed run ended before its final response.
219#[derive(Debug, thiserror::Error)]
220pub enum StreamingError {
221    /// The provider stream failed.
222    #[error("CompletionError: {0}")]
223    Completion(ProviderError),
224    /// Structured failure from the bus, a handler, a hook, or a stream item.
225    #[error("{0}")]
226    Report(#[from] rig_core::error::ErrorReport),
227    /// The run failed for a reason the blocking surface reports the same way.
228    #[error("PromptError: {0}")]
229    Prompt(#[from] PromptError),
230}
231
232/// A failure relayed over the bus is the report the origin sent.
233impl From<ProviderError> for StreamingError {
234    fn from(error: ProviderError) -> Self {
235        match error {
236            ProviderError::Relayed(report) => Self::Report(*report),
237            error => Self::Completion(error),
238        }
239    }
240}
241
242impl From<rig_core::memory::MemoryError> for StreamingError {
243    fn from(err: rig_core::memory::MemoryError) -> Self {
244        Self::Prompt(PromptError::MemoryError(err))
245    }
246}
247
248impl AgentRunner {
249    /// Drive the agent loop as a stream of assistant content, tool activity
250    /// and, last, the [`FinalResponse`](MultiTurnStreamItem::FinalResponse).
251    /// Hooks fire at every observable point, including streamed text and
252    /// tool-call deltas.
253    ///
254    /// Like [`run`](AgentRunner::run), this is lazy: memory loading and agent
255    /// span creation begin only when the stream is first polled,
256    /// and a stream that is dropped unpolled has done nothing. A memory-load
257    /// failure is the stream's first (and only) item. The stream is `Send` on
258    /// native targets, so it can be built in synchronous code and handed to
259    /// whatever polls it; it runs under the span it was built in (a caller's
260    /// enabled span is adopted, otherwise a root `invoke_agent` is created),
261    /// not under whichever span first polls it.
262    ///
263    /// ```rust,no_run
264    /// # use rig_agent::{Agent, agent::StreamingResult};
265    /// fn start(agent: &Agent, prompt: &str) -> StreamingResult {
266    ///     // Nothing runs until whoever holds this polls it.
267    ///     agent.prompt(prompt).stream()
268    /// }
269    /// ```
270    ///
271    /// Shares the drive loop, run construction, tool execution and fail-closed
272    /// hook handling with the blocking [`run`](AgentRunner::run) via
273    /// `drive_agent`, so the two behave identically apart from the streamed
274    /// delta events.
275    #[must_use = "a stream does nothing until polled"]
276    pub fn stream(self) -> StreamingResult {
277        // The span the stream is built under is the one it runs under, not
278        // whichever span first polls it: a host may build the stream in a
279        // request span and hand it to a task of its own.
280        self.stream_under(tracing::Span::current())
281    }
282
283    /// Build a lazy stream under an explicitly captured ambient span.
284    fn stream_under(self, ambient: tracing::Span) -> StreamingResult {
285        let run_under = ambient.clone();
286        let stream = async_stream::stream! {
287            let mut inner = self.start_stream(run_under).await;
288            while let Some(item) = inner.next().await {
289                yield item;
290            }
291        };
292        Box::pin(stream.instrument(ambient))
293    }
294
295    /// The eager half of [`stream`](Self::stream): resolve memory, build the
296    /// run and return the driver as a stream. Called on the first poll,
297    /// under `ambient`, the span the stream was built in.
298    async fn start_stream(self, ambient: tracing::Span) -> StreamingResult {
299        let (agent_span, created_agent_span) = self.open_agent_span(ambient);
300
301        let bus = self.config.bus.clone();
302        let hook_ctx = self.hook_context(true);
303        // A resumed run loads nothing and saves nothing (see `run`).
304        let resolved = match &self.origin {
305            RunOrigin::Resume(_) => Ok((None, None)),
306            RunOrigin::Prompt(_) => {
307                let resolve = self.resolve_history_and_memory(&hook_ctx);
308                futures::pin_mut!(resolve);
309                let mut driven = bus.drive(futures::stream::once(resolve));
310                driven.next().await.unwrap_or(Ok((None, None)))
311            }
312        };
313        let (history_override, memory_handle) = match resolved {
314            Ok(resolved) => resolved,
315            Err(err) => {
316                // Notify settlement before yielding the error so consumers stopping
317                // at the first failure cannot suppress the lifecycle notification.
318                let hooks = self.config.hooks.clone();
319                let stream = async_stream::stream! {
320                    let err = StreamingError::from(err);
321                    if hooks.observes(StepEventKind::RunSettled) {
322                        let reason = err.to_string();
323                        hooks
324                            .on_run_settled(
325                                &hook_ctx,
326                                RunSettled {
327                                    outcome: SettledOutcome::Error(&reason),
328                                    messages: None,
329                                },
330                            )
331                            .await;
332                    }
333                    yield Err(err);
334                };
335                // Instrument under the agent span like the success path so
336                // a load failure stays tied to invoke_agent.
337                return Box::pin(stream.instrument(agent_span));
338            }
339        };
340
341        let run = self.build_run(history_override);
342        let source = StreamingTurnSource::new(
343            &self.config.hooks,
344            self.agent_name_or_default().to_string(),
345            created_agent_span,
346            self.config.record_telemetry_content,
347        );
348
349        let driver = drive_agent(
350            self,
351            source,
352            run,
353            agent_span.clone(),
354            created_agent_span,
355            memory_handle,
356            hook_ctx,
357        )
358        .filter_map(|item| {
359            std::future::ready(match item {
360                Ok(DriveItem::Item(item)) => Some(Ok(item)),
361                Ok(DriveItem::Done(_)) => None,
362                Err(err) => Some(Err(err)),
363            })
364        });
365        // The consumer of this stream drives the agent's bus: every poll that
366        // leaves the run pending polls the driver.
367        let driver = bus.drive(Box::pin(driver));
368
369        Box::pin(driver.instrument(agent_span))
370    }
371}
372
373/// Capacity of the event queue behind [`AgentRunner::run_channel`]: the number
374/// of [`MultiTurnStreamItem`]s the run may buffer ahead of the consumer before
375/// it parks on back-pressure.
376pub const RUN_EVENTS_CAPACITY: usize = 32;
377
378/// Event feed of an agent run started with [`AgentRunner::run_channel`].
379///
380/// Every [`MultiTurnStreamItem`] the run would have streamed is delivered here
381/// in order, ending with [`MultiTurnStreamItem::FinalResponse`]. The feed is a
382/// bounded queue ([`RUN_EVENTS_CAPACITY`]): a slow consumer applies
383/// back-pressure to the run instead of losing events. Poll it as a
384/// [`Stream`] from async code, or drain it with the non-blocking
385/// [`try_next`](RunEvents::try_next) from a synchronous tick, such as a game loop,
386/// UI frame, an ECS system.
387///
388/// Dropping the feed does not cancel the run; it simply stops receiving events
389/// and the run future still resolves with the final
390/// [`PromptResponse`].
391#[derive(Debug)]
392pub struct RunEvents {
393    receiver: mpsc::Receiver<MultiTurnStreamItem>,
394}
395
396impl RunEvents {
397    /// Take the next buffered event without waiting.
398    ///
399    /// Returns `None` both when no event is queued yet and once the run has
400    /// finished and the feed is drained; use [`is_done`](RunEvents::is_done)
401    /// to tell the two apart.
402    pub fn try_next(&mut self) -> Option<MultiTurnStreamItem> {
403        self.receiver.try_recv().ok()
404    }
405
406    /// Whether the run has finished and every event has been taken.
407    pub fn is_done(&self) -> bool {
408        self.receiver.is_terminated()
409    }
410}
411
412impl Stream for RunEvents {
413    type Item = MultiTurnStreamItem;
414
415    fn poll_next(
416        mut self: Pin<&mut Self>,
417        cx: &mut std::task::Context<'_>,
418    ) -> std::task::Poll<Option<Self::Item>> {
419        Pin::new(&mut self.receiver).poll_next(cx)
420    }
421
422    fn size_hint(&self) -> (usize, Option<usize>) {
423        self.receiver.size_hint()
424    }
425}
426
427impl FusedStream for RunEvents {
428    fn is_terminated(&self) -> bool {
429        self.receiver.is_terminated()
430    }
431}
432
433impl AgentRunner {
434    /// Split the run into a driving future and a [`RunEvents`] feed.
435    ///
436    /// The future performs the same agent loop as
437    /// [`run`](AgentRunner::run) and [`stream`](AgentRunner::stream), and
438    /// resolves with the final [`PromptResponse`]; the
439    /// feed receives each intermediate [`MultiTurnStreamItem`] as it happens.
440    /// Spawn the future on any executor and poll the feed from wherever the
441    /// events are consumed; neither side assumes a runtime.
442    ///
443    /// The feed is bounded ([`RUN_EVENTS_CAPACITY`]); when it is full the run
444    /// waits for the consumer rather than dropping events. Dropping the feed
445    /// lets the run continue to completion unobserved. Like
446    /// [`stream`](Self::stream), the run belongs to the span this method was
447    /// called in, wherever the future is later polled.
448    #[must_use = "the run does nothing until the future is driven"]
449    pub fn run_channel(
450        self,
451    ) -> (
452        impl Future<Output = Result<PromptResponse, PromptError>> + WasmCompatSend,
453        RunEvents,
454    ) {
455        let (mut sender, receiver) = mpsc::channel(RUN_EVENTS_CAPACITY);
456        // Captured here, not when the future is first polled: the doc above
457        // says to spawn the future, and the run must still belong to the span
458        // that split it.
459        let ambient = tracing::Span::current();
460        let future = async move {
461            let mut stream = self.stream_under(ambient);
462            let mut response = None;
463            let mut forward = true;
464            while let Some(item) = stream.next().await {
465                let item = match item {
466                    Ok(item) => item,
467                    Err(err) => {
468                        // Drain after the terminal error so span and lineage teardown
469                        // completes instead of being dropped at the yield.
470                        let error = streaming_error_into_prompt(err);
471                        while stream.next().await.is_some() {}
472                        return Err(error);
473                    }
474                };
475                match item {
476                    MultiTurnStreamItem::FinalResponse(done) => {
477                        if forward {
478                            let _ = sender
479                                .send(MultiTurnStreamItem::FinalResponse(done.clone()))
480                                .await;
481                        }
482                        response = Some(done);
483                    }
484                    item => {
485                        if forward && sender.send(item).await.is_err() {
486                            forward = false;
487                        }
488                    }
489                }
490            }
491            response.ok_or_else(|| {
492                PromptError::CompletionError(ProviderError::Response(
493                    "agent run ended without producing a final response".to_string(),
494                ))
495            })
496        };
497        (future, RunEvents { receiver })
498    }
499}
500
501/// Helper function to stream assistant-visible completion output to stdout.
502///
503/// This helper prints streamed assistant text and reasoning. Streaming metadata
504/// events, such as `MultiTurnStreamItem::CompletionCall`, are not printed;
505/// metadata is returned on the [`PromptResponse`] via accessors such as
506/// [`PromptResponse::completion_calls`]. A model-turn retry prints a visible
507/// boundary because text already written to stdout cannot be retracted.
508pub async fn stream_to_stdout(
509    stream: &mut StreamingResult,
510) -> Result<PromptResponse, std::io::Error> {
511    let mut final_res = PromptResponse::empty();
512    print!("Response: ");
513    while let Some(content) = stream.next().await {
514        match content {
515            Ok(MultiTurnStreamItem::StreamAssistantItem(Item::Event(StreamEvent::Text {
516                text,
517                ..
518            }))) => {
519                print!("{text}");
520                std::io::Write::flush(&mut std::io::stdout())?;
521            }
522            Ok(MultiTurnStreamItem::StreamAssistantItem(Item::Event(StreamEvent::End {
523                content: AssistantContent::Reasoning(reasoning),
524                ..
525            }))) => {
526                let reasoning = reasoning
527                    .open(reasoning.issuer())
528                    .map(|reasoning| reasoning.display_text())
529                    .unwrap_or_default();
530                print!("{reasoning}");
531                std::io::Write::flush(&mut std::io::stdout())?;
532            }
533            Ok(MultiTurnStreamItem::FinalResponse(res)) => {
534                final_res = res;
535            }
536            Ok(MultiTurnStreamItem::ModelTurnRetried { turn }) => {
537                print!("\n[model turn {turn} rejected; retry requested]\nResponse: ");
538                std::io::Write::flush(&mut std::io::stdout())?;
539            }
540            Err(err) => {
541                eprintln!("Error: {err}");
542            }
543            _ => {}
544        }
545    }
546
547    Ok(final_res)
548}
549
550#[cfg(test)]
551mod malformed_tool_args_tests;
552#[cfg(test)]
553#[allow(irrefutable_let_patterns, unreachable_patterns)]
554mod tests;