rig_agent/agent/streaming.rs
1//! Streamed agent events, bounded event feeds, and terminal response shaping.
2//!
3//! ```no_run
4//! # fn example(agent: &rig_agent::Agent) {
5//! let stream = agent.prompt("Explain ownership.").stream();
6//! # }
7//! ```
8
9use rig_core::{
10 message::AssistantContent,
11 wasm_compat::{WasmBoxedStream, WasmCompatSend},
12};
13
14use crate::{
15 agent::engine::{DriveItem, LiveTool, StreamingTurnSource, Surfaced, drive_agent},
16 agent::runner::AgentRunner,
17 streaming::{Item, StreamEvent},
18};
19use futures::{SinkExt, Stream, StreamExt, channel::mpsc, stream::FusedStream};
20use serde::{Deserialize, Serialize};
21use std::io::Write;
22use std::pin::Pin;
23use tracing_futures::Instrument;
24
25use crate::completion::PromptError;
26use crate::run::response::{CompletionCall, PromptResponse};
27
28/// The stream a streamed run yields: its items, then its ending.
29pub type StreamingResult = WasmBoxedStream<'static, Result<MultiTurnStreamItem, PromptError>>;
30
31#[derive(Serialize, Deserialize, Debug, Clone)]
32#[serde(tag = "type", rename_all = "camelCase")]
33/// One item of a streamed run: a provider stream event, a committed tool
34/// call, a lifecycle marker, or the run's final response. An item reads back
35/// from its serialized form on its own, without checking its order against
36/// the items around it; [`StreamRecord::parse`](rig_core::streaming::StreamRecord::parse)
37/// checks a run's provider stream items.
38#[non_exhaustive]
39pub enum MultiTurnStreamItem {
40 /// A provider stream item containing model-emitted content: part
41 /// starts and ends, text and reasoning fragments, a tool call's
42 /// argument fragments as they stream (a call to a tool the turn does not
43 /// allow is held for its resolution), and unmodeled passthrough
44 /// payloads. A call's end carries the validated call; the same call id
45 /// is reported again as [`ToolCall`](Self::ToolCall) once history holds it.
46 StreamAssistantItem(Item<StreamEvent>),
47 /// A tool call **committed to the run's history**, in commit order.
48 ///
49 /// The stream's tool items are a projection of committed history
50 /// ([`project`](crate::run::project)): every call history holds is
51 /// reported, and later answered by a [`ToolResult`](Self::ToolResult),
52 /// whether it ran, a hook skipped it, or invalid-call recovery or an
53 /// output reprompt answered it; a run resumed mid-batch reports the batch's
54 /// calls again ([`projection_start`](crate::run::AgentRun::projection_start)).
55 /// It is **not** an execution-lifecycle event
56 /// (see [`ToolExecutionCommitted`](Self::ToolExecutionCommitted)). A
57 /// structured-output Tool-mode output-tool call that finalizes the run is
58 /// committed as text, so it is reported in the
59 /// [`FinalResponse`](Self::FinalResponse) instead.
60 ToolCall {
61 /// The call as the model emitted it. Its id is equal on its
62 /// execution commit and its result.
63 tool_call: rig_core::message::ToolCall,
64 },
65 /// A tool call **starts**: Rig dispatches it now, before its dispatch
66 /// hooks and its body run. The call goes on only once the stream's
67 /// consumer has taken this item, so a slow consumer delays the call. A
68 /// [`RunEvents`] feed takes it into its buffer, so there the call may run
69 /// before the feed's reader sees it.
70 ///
71 /// One item per dispatched call, after the turn's
72 /// [`ToolCall`](Self::ToolCall) items and before the batch's commits and
73 /// results: in call order when calls run one at a time, in start order
74 /// under `tool_concurrency`. A call a dispatch hook skips still starts.
75 /// A call answered by invalid-call recovery, or dropped because a
76 /// sibling ended the run, never starts. A start is not a commit: a
77 /// batch that fails surfaces its starts but no commits or results.
78 ToolExecutionStarted {
79 /// The call as the model emitted it, before any hook rewrite. Its
80 /// id is the one its [`ToolCall`](Self::ToolCall), execution commit
81 /// and result carry.
82 tool_call: rig_core::message::ToolCall,
83 },
84 /// One item of a tool's **accepted output**, reported as soon as the
85 /// call's outcome hooks accept it, without waiting for the rest of the
86 /// batch. The call goes on only once the stream's consumer has taken the
87 /// item, as for [`ToolExecutionStarted`](Self::ToolExecutionStarted).
88 ///
89 /// An output that is exactly one JSON array is reported one element at a
90 /// time, each as one JSON block (an empty array reports nothing); any
91 /// other output is one item holding all its blocks. Text is never
92 /// reparsed as JSON. Items come in completion order under
93 /// `tool_concurrency`, so items of different calls may interleave: group
94 /// them by `tool_result.call`. Every started call that answers reports its items,
95 /// a hook-skipped one included; a call answered by invalid-call recovery
96 /// reports none. An item is not a commit: items stay reported when a
97 /// sibling later fails the batch, a call already running when a sibling
98 /// fails still reports its items as it finishes, and the committed
99 /// [`ToolResult`](Self::ToolResult) still follows a batch that settles.
100 /// Show these for progress; record only the `ToolResult`.
101 ToolResultItem {
102 /// This item's content, with the call id, name and error flag of the
103 /// result the batch commits for the call.
104 tool_result: rig_core::message::ToolResult,
105 /// The item's position in the output, from zero.
106 index: usize,
107 /// How many items the output has.
108 count: usize,
109 },
110 /// Confirmation that Rig **executed and committed** a tool call. It is
111 /// surfaced together with its `ToolResult` only after the whole batch
112 /// settles successfully; the live start is
113 /// [`ToolExecutionStarted`](Self::ToolExecutionStarted).
114 ///
115 /// This item is emitted only for a tool whose body actually ran (it passed
116 /// its `ToolCall` hook checks), never for a call dropped by a sibling's
117 /// termination, skipped by a hook, or resolved by invalid-call recovery.
118 /// Correlate it with the model call and result through the call's id.
119 ToolExecutionCommitted {
120 /// The tool call as **executed**: the model's call with any
121 /// [`DispatchAction::Patch`](crate::agent::DispatchAction::Patch) hook rewrite
122 /// applied (so a redaction rewrite is reflected here, not leaked). The
123 /// model's *original* call is reported via
124 /// [`StreamAssistantItem`](Self::StreamAssistantItem).
125 tool_call: rig_core::message::ToolCall,
126 },
127 /// A tool result **committed to the run's history**, answering a reported
128 /// [`ToolCall`](Self::ToolCall). The tool
129 /// batch commits and surfaces atomically at every `tool_concurrency`
130 /// (including the sequential default): results are surfaced in call order
131 /// only after the whole batch settles successfully; a run that terminates
132 /// mid-batch surfaces no successful tool results. The live output is
133 /// [`ToolResultItem`](Self::ToolResultItem).
134 ToolResult {
135 /// The result; `tool_result.call` is the id of the call it answers.
136 tool_result: rig_core::message::ToolResult,
137 },
138 /// Details for one successfully completed completion request made by this agent stream.
139 ///
140 /// This is emitted when a provider call finishes. Usage is the provider's
141 /// final usage for that completion request when available; it is not
142 /// incremental per streamed token.
143 ///
144 /// ```
145 /// use rig_agent::agent::MultiTurnStreamItem;
146 /// fn input_tokens(item: &MultiTurnStreamItem) -> Option<u64> {
147 /// match item {
148 /// MultiTurnStreamItem::CompletionCall(call) => call.usage.input_tokens,
149 /// _ => None,
150 /// }
151 /// }
152 /// ```
153 CompletionCall(CompletionCall),
154 /// The completed model turn was rejected by a hook for retry.
155 ///
156 /// Text and reasoning deltas emitted for this turn were provisional. A
157 /// consumer should discard or visually reset output associated with `turn`.
158 /// A subsequent attempt is made only if the run's total model-call budget
159 /// permits it.
160 ModelTurnRetried {
161 /// One-based model-call index of the rejected turn.
162 turn: usize,
163 },
164 /// The final result from the stream: the unified [`PromptResponse`] shared
165 /// with the blocking surface.
166 ///
167 /// Terminal for the run: no retry, further turn, or tool execution follows.
168 /// This item is the stream-side
169 /// counterpart of the `on_run_settled` hook's success outcome. Error
170 /// termination surfaces as the stream's `Err` item instead, which is
171 /// equally terminal.
172 FinalResponse(PromptResponse),
173}
174
175/// The one mapping from what the engine surfaces directly; tool items are
176/// built only from committed history.
177impl From<Surfaced> for MultiTurnStreamItem {
178 fn from(surfaced: Surfaced) -> Self {
179 match surfaced {
180 Surfaced::Provider(item) => Self::StreamAssistantItem(item),
181 Surfaced::CompletionCall(call) => Self::CompletionCall(call),
182 Surfaced::ModelTurnRetried { turn } => Self::ModelTurnRetried { turn },
183 Surfaced::Final(response) => Self::FinalResponse(response),
184 }
185 }
186}
187
188impl From<LiveTool> for MultiTurnStreamItem {
189 fn from(live: LiveTool) -> Self {
190 match live {
191 LiveTool::Started(tool_call) => Self::ToolExecutionStarted { tool_call },
192 LiveTool::Output {
193 tool_result,
194 index,
195 count,
196 } => Self::ToolResultItem {
197 tool_result,
198 index,
199 count,
200 },
201 }
202 }
203}
204
205impl AgentRunner {
206 /// Drive the agent loop as a stream of assistant content, tool activity
207 /// and, last, the [`FinalResponse`](MultiTurnStreamItem::FinalResponse).
208 /// Hooks fire at every observable point, including streamed text and
209 /// tool-call deltas.
210 ///
211 /// Like [`run`](AgentRunner::run), this is lazy: agent span creation
212 /// begins only when the stream is first polled, and a stream that is
213 /// dropped unpolled has done nothing. The stream is `Send` on
214 /// native targets, so it can be built in synchronous code and handed to
215 /// whatever polls it; it runs under the span it was built in (a caller's
216 /// enabled span is adopted, otherwise a root `invoke_agent` is created),
217 /// not under whichever span first polls it.
218 ///
219 /// ```rust,no_run
220 /// # use rig_agent::{Agent, agent::StreamingResult};
221 /// fn start(agent: &Agent, prompt: &str) -> StreamingResult {
222 /// // Nothing runs until whoever holds this polls it.
223 /// agent.prompt(prompt).stream()
224 /// }
225 /// ```
226 ///
227 /// Shares the drive loop, run construction, tool execution and fail-closed
228 /// hook handling with the blocking [`run`](AgentRunner::run) via
229 /// `drive_agent`, so the two behave identically apart from the streamed
230 /// delta events.
231 #[must_use = "a stream does nothing until polled"]
232 pub fn stream(self) -> StreamingResult {
233 // The span the stream is built under is the one it runs under, not
234 // whichever span first polls it: a host may build the stream in a
235 // request span and hand it to a task of its own.
236 self.stream_under(tracing::Span::current())
237 }
238
239 /// Build a lazy stream under an explicitly captured ambient span.
240 fn stream_under(self, ambient: tracing::Span) -> StreamingResult {
241 let run_under = ambient.clone();
242 let stream = async_stream::stream! {
243 let mut inner = self.start_stream(run_under);
244 while let Some(item) = inner.next().await {
245 yield item;
246 }
247 };
248 Box::pin(stream.instrument(ambient))
249 }
250
251 /// The eager half of [`stream`](Self::stream): build the run and return
252 /// the driver as a stream. Called on the first poll, under `ambient`, the
253 /// span the stream was built in.
254 fn start_stream(self, ambient: tracing::Span) -> StreamingResult {
255 let source = StreamingTurnSource::new(
256 &self.config.hooks,
257 self.agent_name_or_default().to_string(),
258 self.config.record_telemetry_content,
259 );
260 let driver = drive_agent(self, source, ambient).flat_map(|item| {
261 let (head, tools) = match item {
262 Ok(DriveItem::Surfaced(item)) => (Some(Ok(item.into())), None),
263 Ok(DriveItem::Live(live)) => (Some(Ok(live.into())), None),
264 Ok(DriveItem::Projected(items)) => (None, Some(items)),
265 Ok(DriveItem::Done(_)) => (None, None),
266 Err(err) => (Some(Err(err)), None),
267 };
268 futures::stream::iter(head.into_iter().chain(tools.into_iter().flatten().map(Ok)))
269 });
270 Box::pin(driver)
271 }
272}
273
274/// Capacity of the event queue behind [`AgentRunner::run_channel`]: the number
275/// of [`MultiTurnStreamItem`]s the run may buffer ahead of the consumer before
276/// it parks on back-pressure.
277pub const RUN_EVENTS_CAPACITY: usize = 32;
278
279/// Event feed of an agent run started with [`AgentRunner::run_channel`].
280///
281/// Every [`MultiTurnStreamItem`] the run would have streamed is delivered here
282/// in order, ending with [`MultiTurnStreamItem::FinalResponse`]. The feed is a
283/// bounded queue ([`RUN_EVENTS_CAPACITY`]): a slow consumer applies
284/// back-pressure to the run instead of losing events. Poll it as a
285/// [`Stream`] from async code, or drain it with the non-blocking
286/// [`try_next`](RunEvents::try_next) from a synchronous tick, such as a game loop,
287/// UI frame, an ECS system.
288///
289/// Dropping the feed does not cancel the run; it simply stops receiving events
290/// and the run future still resolves with the final
291/// [`PromptResponse`].
292#[derive(Debug)]
293pub struct RunEvents {
294 receiver: mpsc::Receiver<MultiTurnStreamItem>,
295}
296
297impl RunEvents {
298 /// Take the next buffered event without waiting.
299 ///
300 /// Returns `None` both when no event is queued yet and once the run has
301 /// finished and the feed is drained; use [`is_done`](RunEvents::is_done)
302 /// to tell the two apart.
303 pub fn try_next(&mut self) -> Option<MultiTurnStreamItem> {
304 self.receiver.try_recv().ok()
305 }
306
307 /// Whether the run has finished and every event has been taken.
308 pub fn is_done(&self) -> bool {
309 self.receiver.is_terminated()
310 }
311}
312
313impl Stream for RunEvents {
314 type Item = MultiTurnStreamItem;
315
316 fn poll_next(
317 mut self: Pin<&mut Self>,
318 cx: &mut std::task::Context<'_>,
319 ) -> std::task::Poll<Option<Self::Item>> {
320 Pin::new(&mut self.receiver).poll_next(cx)
321 }
322
323 fn size_hint(&self) -> (usize, Option<usize>) {
324 self.receiver.size_hint()
325 }
326}
327
328impl FusedStream for RunEvents {
329 fn is_terminated(&self) -> bool {
330 self.receiver.is_terminated()
331 }
332}
333
334impl AgentRunner {
335 /// Split the run into a driving future and a [`RunEvents`] feed.
336 ///
337 /// The future performs the same agent loop as
338 /// [`run`](AgentRunner::run) and [`stream`](AgentRunner::stream), and
339 /// resolves with the final [`PromptResponse`]; the
340 /// feed receives each intermediate [`MultiTurnStreamItem`] as it happens.
341 /// Spawn the future on any executor and poll the feed from wherever the
342 /// events are consumed; neither side assumes a runtime.
343 ///
344 /// The feed is bounded ([`RUN_EVENTS_CAPACITY`]); when it is full the run
345 /// waits for the consumer rather than dropping events. Dropping the feed
346 /// lets the run continue to completion unobserved. Like
347 /// [`stream`](Self::stream), the run belongs to the span this method was
348 /// called in, wherever the future is later polled.
349 #[must_use = "the run does nothing until the future is driven"]
350 pub fn run_channel(
351 self,
352 ) -> (
353 impl Future<Output = Result<PromptResponse, PromptError>> + WasmCompatSend,
354 RunEvents,
355 ) {
356 let (mut sender, receiver) = mpsc::channel(RUN_EVENTS_CAPACITY);
357 // Captured here, not when the future is first polled: the doc above
358 // says to spawn the future, and the run must still belong to the span
359 // that split it.
360 let ambient = tracing::Span::current();
361 let future = async move {
362 let mut stream = self.stream_under(ambient);
363 let mut response = None;
364 let mut forward = true;
365 while let Some(item) = stream.next().await {
366 let item = match item {
367 Ok(item) => item,
368 Err(err) => {
369 // Drain after the terminal error so span and lineage teardown
370 // completes instead of being dropped at the yield.
371 while stream.next().await.is_some() {}
372 return Err(err);
373 }
374 };
375 match item {
376 MultiTurnStreamItem::FinalResponse(done) => {
377 if forward {
378 let _ = sender
379 .send(MultiTurnStreamItem::FinalResponse(done.clone()))
380 .await;
381 }
382 response = Some(done);
383 }
384 item => {
385 if forward && sender.send(item).await.is_err() {
386 forward = false;
387 }
388 }
389 }
390 }
391 response.ok_or_else(super::runner::no_final_response)
392 };
393 (future, RunEvents { receiver })
394 }
395}
396
397/// Why [`stream_to_stdout`] returned no response, or why
398/// [`ChatBot::run`](crate::integrations::cli_chatbot::ChatBot::run) stopped.
399#[derive(Debug, thiserror::Error)]
400pub enum StreamToStdoutError {
401 /// The run failed.
402 #[error(transparent)]
403 Run(#[from] PromptError),
404 /// Writing to stdout failed.
405 #[error(transparent)]
406 Io(#[from] std::io::Error),
407 /// The stream ended without a final response.
408 #[error("the stream ended without a final response")]
409 Incomplete,
410}
411
412/// Print a streamed run's assistant text and reasoning to stdout and return
413/// its final response.
414///
415/// Streaming metadata events, such as `MultiTurnStreamItem::CompletionCall`,
416/// are not printed; metadata is returned on the [`PromptResponse`] via
417/// accessors such as [`PromptResponse::completion_calls`]. A model-turn retry
418/// prints a visible boundary because text already written to stdout cannot be
419/// retracted. Fails with the run's error as soon as the stream yields it, a
420/// stdout write failure, or [`StreamToStdoutError::Incomplete`] when the
421/// stream ends without a final response.
422pub async fn stream_to_stdout(
423 stream: &mut StreamingResult,
424) -> Result<PromptResponse, StreamToStdoutError> {
425 let mut stdout = std::io::stdout();
426 write!(stdout, "Response: ")?;
427 while let Some(content) = stream.next().await {
428 match content? {
429 MultiTurnStreamItem::StreamAssistantItem(Item::Event(StreamEvent::Text {
430 text,
431 ..
432 })) => {
433 write!(stdout, "{text}")?;
434 stdout.flush()?;
435 }
436 MultiTurnStreamItem::StreamAssistantItem(Item::Event(StreamEvent::End {
437 content: AssistantContent::Reasoning(reasoning),
438 ..
439 })) => {
440 write!(stdout, "{}", reasoning.text)?;
441 stdout.flush()?;
442 }
443 MultiTurnStreamItem::FinalResponse(response) => return Ok(response),
444 MultiTurnStreamItem::ModelTurnRetried { turn } => {
445 write!(
446 stdout,
447 "\n[model turn {turn} rejected; retry requested]\nResponse: "
448 )?;
449 stdout.flush()?;
450 }
451 _ => {}
452 }
453 }
454 Err(StreamToStdoutError::Incomplete)
455}
456
457#[cfg(test)]
458mod cut_off_tool_call_tests;
459#[cfg(test)]
460mod malformed_tool_args_tests;
461#[cfg(test)]
462mod tests;
463#[cfg(test)]
464mod tool_result_item_tests;
465#[cfg(test)]
466mod tool_started_tests;