Skip to main content

rig_core/streaming/
mod.rs

1//! Completion events, and [`Streamed`], the one stream every reply arrives
2//! as.
3//!
4//! ```
5//! use rig_core::streaming::{Item, StreamEvent};
6//!
7//! fn text(item: &Item<StreamEvent>) -> Option<&str> {
8//!     match item {
9//!         Item::Event(StreamEvent::Text { text, .. }) => Some(text),
10//!         _ => None,
11//!     }
12//! }
13//! # let _ = text;
14//! ```
15
16mod event;
17
18use std::pin::Pin;
19use std::sync::{Arc, Mutex};
20use std::task::{Context, Poll};
21
22use futures::{Stream, StreamExt};
23use serde::{Deserialize, Serialize};
24
25use crate::completion::CompletionResponse;
26use crate::driver::{lock, record_request_id};
27use crate::error::{ErrorReport, ProviderError};
28use crate::message::ToolResult;
29use crate::operation::{Completion, Turn};
30use crate::wasm_compat::WasmBoxedStream;
31use crate::wire::{Fold, Operation, Reply, Shared};
32pub use event::{Item, Part, PartKind, SequenceError, StreamEvent, Transcript};
33
34/// Unmodeled JSON payload with content-redacted `Debug` output.
35/// Serialization preserves the payload; [`Self::value`] explicitly exposes it.
36#[derive(Clone, PartialEq, Serialize, Deserialize)]
37#[serde(transparent)]
38pub struct UnknownPayload(serde_json::Value);
39
40impl UnknownPayload {
41    /// Wrap a raw unmodeled payload.
42    pub fn new(value: serde_json::Value) -> Self {
43        Self(value)
44    }
45
46    /// The raw payload, for consumers who opt in to the content.
47    pub fn value(&self) -> &serde_json::Value {
48        &self.0
49    }
50}
51
52impl std::fmt::Debug for UnknownPayload {
53    /// Reports serialized size without exposing payload content.
54    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
55        let bytes = serde_json::to_vec(&self.0).map_or(0, |json| json.len());
56        write!(f, "UnknownPayload({bytes} bytes redacted)")
57    }
58}
59
60impl From<serde_json::Value> for UnknownPayload {
61    fn from(value: serde_json::Value) -> Self {
62        Self(value)
63    }
64}
65
66#[cfg(test)]
67mod unknown_payload_tests;
68
69/// One item of a completion stream relayed over the bus: an item of the
70/// reply, or the response the origin folded when the provider ended it.
71#[derive(Debug, Clone, PartialEq)]
72pub enum Relayed {
73    /// An event, or a payload the origin's decoder did not model.
74    Item(Item<StreamEvent>),
75    /// The provider ended the reply; the origin's response.
76    Done(Box<CompletionResponse>),
77}
78
79/// A completion stream as the bus carries it.
80pub type StreamEvents = WasmBoxedStream<'static, Result<Relayed, ErrorReport>>;
81
82/// One reply as it arrives: its events, and the fold that has seen each of
83/// them. It is what [`Model::stream`](crate::Model::stream) returns for
84/// every operation, and what a call finishes.
85///
86/// An error is the last item: the stream ends after it. Stop polling to
87/// pause; drop the stream to cancel.
88pub struct Streamed<Op: Operation> {
89    /// The reading of the reply; `None` once it ended.
90    reading: Option<WasmBoxedStream<'static, ()>>,
91    shared: Arc<Mutex<Shared<Op>>>,
92    span: tracing::Span,
93    provider: String,
94    /// The error the stream yielded, which ends it.
95    failed: Option<ProviderError>,
96}
97
98/// A streamed completion.
99pub type CompletionStream = Streamed<Completion>;
100
101impl<Op: Operation> Streamed<Op> {
102    /// The reply `reading` writes into `shared`, under `span`.
103    pub(crate) fn new(
104        reading: WasmBoxedStream<'static, ()>,
105        shared: Arc<Mutex<Shared<Op>>>,
106        span: tracing::Span,
107        provider: impl Into<String>,
108    ) -> Self {
109        Self {
110            reading: Some(reading),
111            shared,
112            span,
113            provider: provider.into(),
114            failed: None,
115        }
116    }
117
118    /// The one place an item is taken: an event is absorbed by the fold
119    /// before it leaves, and an error, stamped with the request id, ends the
120    /// stream.
121    fn poll_item(
122        &mut self,
123        cx: &mut Context<'_>,
124    ) -> Poll<Option<Result<Item<Op::Event>, ProviderError>>> {
125        loop {
126            if self.failed.is_some() {
127                return Poll::Ready(None);
128            }
129            {
130                let mut shared = lock(&self.shared);
131                if let Some(item) = shared.items.pop_front() {
132                    let item = match item {
133                        Ok(Item::Event(event)) => {
134                            shared.fold.absorb(&event).map(|()| Item::Event(event))
135                        }
136                        Ok(unknown) => Ok(unknown),
137                        // An id an upstream constructor already attached
138                        // wins: it saw the reply.
139                        Err(error) => {
140                            Err(error.with_provider_request_id(shared.request_id.clone()))
141                        }
142                    };
143                    if let Err(error) = &item {
144                        record_request_id(&self.span, error.provider_request_id());
145                        shared.items.clear();
146                        self.failed = Some(error.clone());
147                        self.reading = None;
148                    }
149                    return Poll::Ready(Some(item));
150                }
151            }
152            let Some(reading) = &mut self.reading else {
153                return Poll::Ready(None);
154            };
155            match reading.as_mut().poll_next(cx) {
156                Poll::Pending => return Poll::Pending,
157                Poll::Ready(Some(())) => {}
158                Poll::Ready(None) => self.reading = None,
159            }
160        }
161    }
162
163    /// What the driver learned about the reply so far.
164    fn reply(&self, shared: &mut Shared<Op>) -> Reply {
165        Reply {
166            provider: self.provider.clone(),
167            // A whole body is the reply's document; a stream's is what its
168            // decoder recorded.
169            raw: shared
170                .document
171                .clone()
172                .or_else(|| shared.raw.clone())
173                .unwrap_or(serde_json::Value::Null),
174            provider_request_id: shared.request_id.clone(),
175        }
176    }
177
178    /// Read the rest of the reply, then fold it with the provider's end into
179    /// the response: what [`Model::call`](crate::Model::call) returns for
180    /// the same reply. An error the stream yielded, now or before, is the
181    /// result; a reply the provider did not end is
182    /// [`ProviderError::Truncated`].
183    pub async fn finish(self) -> Result<Op::Response, ProviderError> {
184        self.finish_routed().await.map_err(|(error, _)| error)
185    }
186
187    /// [`Self::finish`], with a failure paired with the request path of the
188    /// reply that failed.
189    pub(crate) async fn finish_routed(mut self) -> Result<Op::Response, (ProviderError, String)> {
190        let route = |stream: &Self| lock(&stream.shared).route.clone();
191        while let Some(item) = futures::future::poll_fn(|cx| self.poll_item(cx)).await {
192            if let Err(error) = item {
193                return Err((error, route(&self)));
194            }
195        }
196        if let Some(error) = self.failed.take() {
197            return Err((error, route(&self)));
198        }
199        let (end, response, reply, path) = {
200            let mut shared = lock(&self.shared);
201            let reply = self.reply(&mut shared);
202            (
203                shared.end.take(),
204                shared.response.take(),
205                reply,
206                shared.route.clone(),
207            )
208        };
209        if let Some(response) = response {
210            return Ok(response);
211        }
212        let Some(end) = end else {
213            return Err((ProviderError::Truncated, path));
214        };
215        let fold = match Arc::try_unwrap(self.shared) {
216            Ok(shared) => {
217                shared
218                    .into_inner()
219                    .unwrap_or_else(std::sync::PoisonError::into_inner)
220                    .fold
221            }
222            Err(_) => {
223                return Err((
224                    ProviderError::Response("the reply is still being read".to_owned()),
225                    path,
226                ));
227            }
228        };
229        fold.finish(end, reply).map_err(|error| (error, path))
230    }
231}
232
233impl Streamed<Completion> {
234    /// A stream relayed over the bus under `label`: its events, then the
235    /// response the origin folded. A relay that ends without one was cut
236    /// short.
237    pub fn relay(label: impl Into<String>, mut events: StreamEvents) -> Self {
238        let label = label.into();
239        let shared = Arc::new(Mutex::new(Shared::new(Turn::relayed(label.clone()))));
240        let writer = Arc::clone(&shared);
241        let reading = async_stream::stream! {
242            while let Some(item) = events.next().await {
243                // The lock is released before the stream yields.
244                let ended = {
245                    let mut shared = lock(&writer);
246                    match item {
247                        Ok(Relayed::Item(item)) => {
248                            shared.items.push_back(Ok(item));
249                            false
250                        }
251                        Ok(Relayed::Done(response)) => {
252                            shared.response = Some(*response);
253                            true
254                        }
255                        Err(report) => {
256                            shared
257                                .items
258                                .push_back(Err(ProviderError::Relayed(Box::new(report))));
259                            true
260                        }
261                    }
262                };
263                if ended {
264                    return;
265                }
266                yield ();
267            }
268            lock(&writer).items.push_back(Err(ProviderError::Truncated));
269        };
270        Self::new(Box::pin(reading), shared, tracing::Span::none(), label)
271    }
272
273    /// This stream as the bus carries it: its items, then the response it
274    /// folds into, or the error that ended it. A reply cut short closes the
275    /// relay without a response, as its transport closed.
276    pub fn into_relay(mut self) -> StreamEvents {
277        Box::pin(async_stream::stream! {
278            while let Some(item) = self.next().await {
279                match item {
280                    Ok(item) => yield Ok(Relayed::Item(item)),
281                    Err(ProviderError::Truncated) => return,
282                    Err(error) => {
283                        yield Err(ErrorReport::from(&error));
284                        return;
285                    }
286                }
287            }
288            match self.finish().await {
289                Ok(response) => yield Ok(Relayed::Done(Box::new(response))),
290                Err(ProviderError::Truncated) => {}
291                Err(error) => yield Err(ErrorReport::from(&error)),
292            }
293        })
294    }
295
296    /// What arrived so far: every part that ended, and the provider's end
297    /// once it arrived. Valid after an error, and after the caller stopped
298    /// polling.
299    pub fn partial(&self) -> CompletionResponse {
300        let mut shared = lock(&self.shared);
301        if let Some(response) = &shared.response {
302            return response.clone();
303        }
304        let reply = self.reply(&mut shared);
305        shared.fold.partial(shared.end.as_ref(), &reply)
306    }
307
308    /// The assistant message id the reply recorded so far.
309    pub fn message_id(&self) -> Option<String> {
310        let shared = lock(&self.shared);
311        shared
312            .response
313            .as_ref()
314            .and_then(|response| response.message_id.clone())
315            .or_else(|| shared.fold.message_id().map(str::to_owned))
316    }
317
318    /// The issuer this reply's reasoning is sealed to.
319    pub fn reasoning_issuer(&self) -> crate::message::Issuer {
320        lock(&self.shared).fold.reasoning_issuer()
321    }
322}
323
324impl<Op: Operation> Stream for Streamed<Op> {
325    type Item = Result<Item<Op::Event>, ProviderError>;
326
327    fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
328        self.get_mut().poll_item(cx)
329    }
330}
331
332/// Streamed user content. This content is primarily used to represent tool results from tool calls made during a multi-turn/step agent prompt.
333#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
334#[serde(untagged)]
335pub enum StreamedUserContent {
336    /// Tool result emitted during a multi-turn streaming agent loop.
337    ToolResult {
338        /// The result; `tool_result.call` is the id of the call it answers.
339        tool_result: ToolResult,
340    },
341}
342
343impl StreamedUserContent {
344    /// A streamed tool result.
345    pub fn tool_result(tool_result: ToolResult) -> Self {
346        Self::ToolResult { tool_result }
347    }
348}
349
350#[cfg(test)]
351mod tests;