1mod event;
17
18use std::pin::Pin;
19use std::sync::{Arc, Mutex};
20use std::task::{Context, Poll};
21
22use futures::{Stream, StreamExt};
23use serde::{Deserialize, Serialize};
24
25use crate::completion::CompletionResponse;
26use crate::driver::record_request_id;
27use crate::error::{ErrorReport, ProviderError};
28pub use crate::json_utils::parse_partial_arguments;
29use crate::lock;
30use crate::operation::{Completion, ReplyFold};
31use crate::wasm_compat::WasmBoxedStream;
32use crate::wire::{Operation, Shared};
33pub use event::{Item, Part, PartKind, SequenceError, StreamEvent, StreamRecord};
34
35#[derive(Clone, PartialEq, Serialize, Deserialize)]
38#[serde(transparent)]
39pub struct UnknownPayload(serde_json::Value);
40
41impl UnknownPayload {
42 pub fn new(value: serde_json::Value) -> Self {
44 Self(value)
45 }
46
47 pub fn value(&self) -> &serde_json::Value {
49 &self.0
50 }
51}
52
53impl std::fmt::Debug for UnknownPayload {
54 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
56 let bytes = serde_json::to_vec(&self.0).map_or(0, |json| json.len());
57 write!(f, "UnknownPayload({bytes} bytes redacted)")
58 }
59}
60
61impl From<serde_json::Value> for UnknownPayload {
62 fn from(value: serde_json::Value) -> Self {
63 Self(value)
64 }
65}
66
67#[cfg(test)]
68mod unknown_payload_tests;
69
70#[derive(Debug, Clone, PartialEq)]
74pub enum Relayed {
75 Origin(crate::message::Origin),
78 Item(Item<StreamEvent>),
80 Usage(crate::completion::Usage),
84 Done(Arc<CompletionResponse>),
86}
87
88pub type StreamEvents = WasmBoxedStream<'static, Result<Relayed, ErrorReport>>;
90
91pub struct Streamed<Op: Operation> {
98 reading: Option<WasmBoxedStream<'static, ()>>,
100 shared: Arc<Mutex<Shared<Op>>>,
101 span: tracing::Span,
102 provider: String,
103 failed: Option<ProviderError>,
105}
106
107enum Taken<E> {
110 Item(Result<Item<E>, ProviderError>),
111 Usage(crate::completion::Usage),
112}
113
114pub type CompletionStream = Streamed<Completion>;
116
117impl<Op: Operation> Streamed<Op> {
118 pub(crate) fn new(
120 reading: WasmBoxedStream<'static, ()>,
121 shared: Arc<Mutex<Shared<Op>>>,
122 span: tracing::Span,
123 provider: impl Into<String>,
124 ) -> Self {
125 Self {
126 reading: Some(reading),
127 shared,
128 span,
129 provider: provider.into(),
130 failed: None,
131 }
132 }
133
134 fn poll_item(
138 &mut self,
139 cx: &mut Context<'_>,
140 ) -> Poll<Option<Result<Item<Op::Event>, ProviderError>>> {
141 self.poll_taken(cx, false).map(|taken| match taken? {
142 Taken::Item(item) => Some(item),
143 Taken::Usage(_) => None,
144 })
145 }
146
147 fn poll_taken(&mut self, cx: &mut Context<'_>, usage: bool) -> Poll<Option<Taken<Op::Event>>> {
150 loop {
151 if self.failed.is_some() {
152 return Poll::Ready(None);
153 }
154 {
155 let mut shared = lock(&self.shared);
156 if usage && let Some(reported) = shared.usage.take() {
157 return Poll::Ready(Some(Taken::Usage(reported)));
158 }
159 if let Some(item) = shared.take().map(Taken::Item) {
160 if let Taken::Item(Err(error)) = &item {
161 record_request_id(&self.span, error.provider_request_id());
162 shared.items.clear();
163 self.failed = Some(error.clone());
164 self.reading = None;
165 }
166 return Poll::Ready(Some(item));
167 }
168 }
169 let Some(reading) = &mut self.reading else {
170 return Poll::Ready(None);
171 };
172 match reading.as_mut().poll_next(cx) {
173 Poll::Pending => return Poll::Pending,
174 Poll::Ready(Some(())) => {}
175 Poll::Ready(None) => self.reading = None,
176 }
177 }
178 }
179
180 pub async fn finish(self) -> Result<Op::Response, ProviderError> {
186 self.finish_routed().await.map_err(|(error, _)| error)
187 }
188
189 pub(crate) async fn finish_routed(mut self) -> Result<Op::Response, (ProviderError, String)> {
192 let route = |stream: &Self| lock(&stream.shared).route.clone();
193 while let Some(item) = futures::future::poll_fn(|cx| self.poll_item(cx)).await {
194 if let Err(error) = item {
195 return Err((error, route(&self)));
196 }
197 }
198 if let Some(error) = self.failed.take() {
199 return Err((error, route(&self)));
200 }
201 let path = route(&self);
202 let Ok(shared) = Arc::try_unwrap(self.shared) else {
203 return Err((
204 ProviderError::Response("the reply is still being read".to_owned()),
205 path,
206 ));
207 };
208 shared
209 .into_inner()
210 .unwrap_or_else(std::sync::PoisonError::into_inner)
211 .conclude(&self.provider)
212 .map_err(|error| (error, path))
213 }
214}
215
216pub fn delivered(items: &[Item<StreamEvent>]) -> Vec<crate::message::AssistantContent> {
221 use crate::wire::Fold;
222 let mut turn = ReplyFold::relayed("delivered");
223 for item in items {
224 if let Item::Event(event) = item
225 && turn.absorb(event).is_err()
226 {
227 break;
228 }
229 }
230 let reply = crate::wire::Reply {
231 provider: String::new(),
232 raw: serde_json::Value::Null,
233 provider_request_id: None,
234 };
235 turn.partial(None, &reply, None).choice
236}
237
238impl Streamed<Completion> {
239 pub fn relay(label: impl Into<String>, mut events: StreamEvents) -> Self {
243 let label = label.into();
244 let shared = Arc::new(Mutex::new(Shared::new(ReplyFold::relayed(label.clone()))));
245 let writer = Arc::clone(&shared);
246 let reading = async_stream::stream! {
247 while let Some(item) = events.next().await {
248 let ended = {
250 let mut shared = lock(&writer);
251 match item {
252 Ok(Relayed::Origin(origin)) => {
253 ReplyFold::set_origin(&mut shared.fold, origin);
254 false
255 }
256 Ok(Relayed::Item(item)) => {
257 shared.items.push_back(Ok(item));
258 false
259 }
260 Ok(Relayed::Usage(usage)) => {
261 shared.usage = Some(usage);
262 false
263 }
264 Ok(Relayed::Done(response)) => {
265 shared.response = Some(Arc::unwrap_or_clone(response));
266 true
267 }
268 Err(report) => {
269 shared
270 .items
271 .push_back(Err(ProviderError::Relayed(Box::new(report))));
272 true
273 }
274 }
275 };
276 if ended {
277 return;
278 }
279 yield ();
280 }
281 lock(&writer).items.push_back(Err(ProviderError::Truncated));
282 };
283 Self::new(Box::pin(reading), shared, tracing::Span::none(), label)
284 }
285
286 pub fn into_relay(mut self) -> StreamEvents {
291 let origin = lock(&self.shared).fold.origin().clone();
292 Box::pin(async_stream::stream! {
293 yield Ok(Relayed::Origin(origin));
294 while let Some(taken) = futures::future::poll_fn(|cx| self.poll_taken(cx, true)).await {
295 let item = match taken {
296 Taken::Usage(usage) => {
297 yield Ok(Relayed::Usage(usage));
298 continue;
299 }
300 Taken::Item(item) => item,
301 };
302 match item {
303 Ok(item) => yield Ok(Relayed::Item(item)),
304 Err(ProviderError::Truncated) => return,
305 Err(error) => {
306 yield Err(ErrorReport::from(&error));
307 return;
308 }
309 }
310 }
311 match self.finish().await {
312 Ok(response) => yield Ok(Relayed::Done(Arc::new(response))),
313 Err(ProviderError::Truncated) => {}
314 Err(error) => yield Err(ErrorReport::from(&error)),
315 }
316 })
317 }
318
319 pub fn partial(&self) -> CompletionResponse {
323 let shared = lock(&self.shared);
324 if let Some(response) = &shared.response {
325 return response.clone();
326 }
327 shared.fold.partial(
328 shared.end.as_ref(),
329 &shared.reply(&self.provider),
330 self.failed.as_ref(),
331 )
332 }
333}
334
335impl<Op: Operation> Stream for Streamed<Op> {
336 type Item = Result<Item<Op::Event>, ProviderError>;
337
338 fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
339 self.get_mut().poll_item(cx)
340 }
341}
342
343#[cfg(test)]
344mod tests;