Skip to main content

rig_core/streaming/
mod.rs

1//! Completion events, and [`Streamed`], the one stream every reply arrives
2//! as.
3//!
4//! ```
5//! use rig_core::streaming::{Item, StreamEvent};
6//!
7//! fn text(item: &Item<StreamEvent>) -> Option<&str> {
8//!     match item {
9//!         Item::Event(StreamEvent::Text { text, .. }) => Some(text),
10//!         _ => None,
11//!     }
12//! }
13//! # let _ = text;
14//! ```
15
16mod event;
17
18use std::pin::Pin;
19use std::sync::{Arc, Mutex};
20use std::task::{Context, Poll};
21
22use futures::{Stream, StreamExt};
23use serde::{Deserialize, Serialize};
24
25use crate::completion::CompletionResponse;
26use crate::driver::record_request_id;
27use crate::error::{ErrorReport, ProviderError};
28pub use crate::json_utils::parse_partial_arguments;
29use crate::lock;
30use crate::operation::{Completion, ReplyFold};
31use crate::wasm_compat::WasmBoxedStream;
32use crate::wire::{Operation, Shared};
33pub use event::{Item, Part, PartKind, SequenceError, StreamEvent, StreamRecord};
34
35/// Unmodeled JSON payload with content-redacted `Debug` output.
36/// Serialization preserves the payload; [`Self::value`] explicitly exposes it.
37#[derive(Clone, PartialEq, Serialize, Deserialize)]
38#[serde(transparent)]
39pub struct UnknownPayload(serde_json::Value);
40
41impl UnknownPayload {
42    /// Wrap a raw unmodeled payload.
43    pub fn new(value: serde_json::Value) -> Self {
44        Self(value)
45    }
46
47    /// The raw payload, for consumers who opt in to the content.
48    pub fn value(&self) -> &serde_json::Value {
49        &self.0
50    }
51}
52
53impl std::fmt::Debug for UnknownPayload {
54    /// Reports serialized size without exposing payload content.
55    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
56        let bytes = serde_json::to_vec(&self.0).map_or(0, |json| json.len());
57        write!(f, "UnknownPayload({bytes} bytes redacted)")
58    }
59}
60
61impl From<serde_json::Value> for UnknownPayload {
62    fn from(value: serde_json::Value) -> Self {
63        Self(value)
64    }
65}
66
67#[cfg(test)]
68mod unknown_payload_tests;
69
70/// One item of a completion stream relayed by a handler: who the reply is
71/// from, an item of the reply, the usage the provider reported so far, or
72/// the response the origin folded when the provider ended it.
73#[derive(Debug, Clone, PartialEq)]
74pub enum Relayed {
75    /// The wire, provider and model the reply is from, sent before its
76    /// first item, so a reply cut short still knows its origin.
77    Origin(crate::message::Origin),
78    /// An event, or a payload the origin's decoder did not model.
79    Item(Item<StreamEvent>),
80    /// What the provider reported the reply used so far, before its end,
81    /// such as its input as the reply starts: all of it until now, so a
82    /// later report replaces an earlier one.
83    Usage(crate::completion::Usage),
84    /// The provider ended the reply; the origin's response, shared.
85    Done(Arc<CompletionResponse>),
86}
87
88/// A completion stream as a handler relays it.
89pub type StreamEvents = WasmBoxedStream<'static, Result<Relayed, ErrorReport>>;
90
91/// One reply as it arrives: its events, and the fold that has seen each of
92/// them. It is what [`Model::stream`](crate::Model::stream) returns for
93/// every operation, and what a call finishes.
94///
95/// An error is the last item: the stream ends after it. Stop polling to
96/// pause; drop the stream to cancel.
97pub struct Streamed<Op: Operation> {
98    /// The reading of the reply; `None` once it ended.
99    reading: Option<WasmBoxedStream<'static, ()>>,
100    shared: Arc<Mutex<Shared<Op>>>,
101    span: tracing::Span,
102    provider: String,
103    /// The error the stream yielded, which ends it.
104    failed: Option<ProviderError>,
105}
106
107/// What [`Streamed`] takes next: an item, or the usage the provider
108/// reported since the last one.
109enum Taken<E> {
110    Item(Result<Item<E>, ProviderError>),
111    Usage(crate::completion::Usage),
112}
113
114/// A streamed completion.
115pub type CompletionStream = Streamed<Completion>;
116
117impl<Op: Operation> Streamed<Op> {
118    /// The reply `reading` writes into `shared`, under `span`.
119    pub(crate) fn new(
120        reading: WasmBoxedStream<'static, ()>,
121        shared: Arc<Mutex<Shared<Op>>>,
122        span: tracing::Span,
123        provider: impl Into<String>,
124    ) -> Self {
125        Self {
126            reading: Some(reading),
127            shared,
128            span,
129            provider: provider.into(),
130            failed: None,
131        }
132    }
133
134    /// The one place an item is taken: an event is absorbed by the fold
135    /// before it leaves, and an error, stamped with the request id, ends the
136    /// stream.
137    fn poll_item(
138        &mut self,
139        cx: &mut Context<'_>,
140    ) -> Poll<Option<Result<Item<Op::Event>, ProviderError>>> {
141        self.poll_taken(cx, false).map(|taken| match taken? {
142            Taken::Item(item) => Some(item),
143            Taken::Usage(_) => None,
144        })
145    }
146
147    /// [`Self::poll_item`], or with `usage` the usage the provider reported
148    /// since the last look, before the items.
149    fn poll_taken(&mut self, cx: &mut Context<'_>, usage: bool) -> Poll<Option<Taken<Op::Event>>> {
150        loop {
151            if self.failed.is_some() {
152                return Poll::Ready(None);
153            }
154            {
155                let mut shared = lock(&self.shared);
156                if usage && let Some(reported) = shared.usage.take() {
157                    return Poll::Ready(Some(Taken::Usage(reported)));
158                }
159                if let Some(item) = shared.take().map(Taken::Item) {
160                    if let Taken::Item(Err(error)) = &item {
161                        record_request_id(&self.span, error.provider_request_id());
162                        shared.items.clear();
163                        self.failed = Some(error.clone());
164                        self.reading = None;
165                    }
166                    return Poll::Ready(Some(item));
167                }
168            }
169            let Some(reading) = &mut self.reading else {
170                return Poll::Ready(None);
171            };
172            match reading.as_mut().poll_next(cx) {
173                Poll::Pending => return Poll::Pending,
174                Poll::Ready(Some(())) => {}
175                Poll::Ready(None) => self.reading = None,
176            }
177        }
178    }
179
180    /// Read the rest of the reply, then fold it with the provider's end into
181    /// the response: what [`Model::call`](crate::Model::call) returns for
182    /// the same reply. An error the stream yielded, now or before, is the
183    /// result; a reply the provider did not end is
184    /// [`ProviderError::Truncated`].
185    pub async fn finish(self) -> Result<Op::Response, ProviderError> {
186        self.finish_routed().await.map_err(|(error, _)| error)
187    }
188
189    /// [`Self::finish`], with a failure paired with the request path of the
190    /// reply that failed.
191    pub(crate) async fn finish_routed(mut self) -> Result<Op::Response, (ProviderError, String)> {
192        let route = |stream: &Self| lock(&stream.shared).route.clone();
193        while let Some(item) = futures::future::poll_fn(|cx| self.poll_item(cx)).await {
194            if let Err(error) = item {
195                return Err((error, route(&self)));
196            }
197        }
198        if let Some(error) = self.failed.take() {
199            return Err((error, route(&self)));
200        }
201        let path = route(&self);
202        let Ok(shared) = Arc::try_unwrap(self.shared) else {
203            return Err((
204                ProviderError::Response("the reply is still being read".to_owned()),
205                path,
206            ));
207        };
208        shared
209            .into_inner()
210            .unwrap_or_else(std::sync::PoisonError::into_inner)
211            .conclude(&self.provider)
212            .map_err(|error| (error, path))
213    }
214}
215
216/// The assistant content a stream's `items` delivered, as its partial reply
217/// holds it: every part that ended, in start order, and the text of a text
218/// or reasoning part still open. The items hold no provider end, so no
219/// block keeps its provider item and the content replays canonically.
220pub fn delivered(items: &[Item<StreamEvent>]) -> Vec<crate::message::AssistantContent> {
221    use crate::wire::Fold;
222    let mut turn = ReplyFold::relayed("delivered");
223    for item in items {
224        if let Item::Event(event) = item
225            && turn.absorb(event).is_err()
226        {
227            break;
228        }
229    }
230    let reply = crate::wire::Reply {
231        provider: String::new(),
232        raw: serde_json::Value::Null,
233        provider_request_id: None,
234    };
235    turn.partial(None, &reply, None).choice
236}
237
238impl Streamed<Completion> {
239    /// A stream relayed under `label`: its events, then the
240    /// response the origin folded. A relay that ends without one was cut
241    /// short.
242    pub fn relay(label: impl Into<String>, mut events: StreamEvents) -> Self {
243        let label = label.into();
244        let shared = Arc::new(Mutex::new(Shared::new(ReplyFold::relayed(label.clone()))));
245        let writer = Arc::clone(&shared);
246        let reading = async_stream::stream! {
247            while let Some(item) = events.next().await {
248                // The lock is released before the stream yields.
249                let ended = {
250                    let mut shared = lock(&writer);
251                    match item {
252                        Ok(Relayed::Origin(origin)) => {
253                            ReplyFold::set_origin(&mut shared.fold, origin);
254                            false
255                        }
256                        Ok(Relayed::Item(item)) => {
257                            shared.items.push_back(Ok(item));
258                            false
259                        }
260                        Ok(Relayed::Usage(usage)) => {
261                            shared.usage = Some(usage);
262                            false
263                        }
264                        Ok(Relayed::Done(response)) => {
265                            shared.response = Some(Arc::unwrap_or_clone(response));
266                            true
267                        }
268                        Err(report) => {
269                            shared
270                                .items
271                                .push_back(Err(ProviderError::Relayed(Box::new(report))));
272                            true
273                        }
274                    }
275                };
276                if ended {
277                    return;
278                }
279                yield ();
280            }
281            lock(&writer).items.push_back(Err(ProviderError::Truncated));
282        };
283        Self::new(Box::pin(reading), shared, tracing::Span::none(), label)
284    }
285
286    /// This stream as a handler relays it: its items and the usage its
287    /// provider reported before the end, then the response it folds into,
288    /// or the error that ended it. A reply cut short closes the
289    /// relay without a response, as its transport closed.
290    pub fn into_relay(mut self) -> StreamEvents {
291        let origin = lock(&self.shared).fold.origin().clone();
292        Box::pin(async_stream::stream! {
293            yield Ok(Relayed::Origin(origin));
294            while let Some(taken) = futures::future::poll_fn(|cx| self.poll_taken(cx, true)).await {
295                let item = match taken {
296                    Taken::Usage(usage) => {
297                        yield Ok(Relayed::Usage(usage));
298                        continue;
299                    }
300                    Taken::Item(item) => item,
301                };
302                match item {
303                    Ok(item) => yield Ok(Relayed::Item(item)),
304                    Err(ProviderError::Truncated) => return,
305                    Err(error) => {
306                        yield Err(ErrorReport::from(&error));
307                        return;
308                    }
309                }
310            }
311            match self.finish().await {
312                Ok(response) => yield Ok(Relayed::Done(Arc::new(response))),
313                Err(ProviderError::Truncated) => {}
314                Err(error) => yield Err(ErrorReport::from(&error)),
315            }
316        })
317    }
318
319    /// What arrived so far: every part that ended, and the provider's end
320    /// once it arrived. Valid after an error, and after the caller stopped
321    /// polling.
322    pub fn partial(&self) -> CompletionResponse {
323        let shared = lock(&self.shared);
324        if let Some(response) = &shared.response {
325            return response.clone();
326        }
327        shared.fold.partial(
328            shared.end.as_ref(),
329            &shared.reply(&self.provider),
330            self.failed.as_ref(),
331        )
332    }
333}
334
335impl<Op: Operation> Stream for Streamed<Op> {
336    type Item = Result<Item<Op::Event>, ProviderError>;
337
338    fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
339        self.get_mut().poll_item(cx)
340    }
341}
342
343#[cfg(test)]
344mod tests;