1use std::collections::VecDeque;
4use std::future::Future;
5use std::pin::Pin;
6use std::sync::{Arc, Mutex, MutexGuard, Weak};
7
8use tokio::sync::{mpsc, watch};
9
10use crate::errors::{MuseError, ProtocolError};
11use crate::msp::{
12 Item, ItemDeltaParams, LastTurn, TurnCompletedParams, TurnErrorKind, TurnTerminal,
13 TurnUnqueuedParams,
14};
15
16#[allow(clippy::large_enum_variant)] #[derive(Debug, Clone, PartialEq)]
19#[non_exhaustive]
20pub enum TurnOutcome {
21 Completed {
23 params: TurnCompletedParams,
25 observed_start: bool,
28 },
29 Unqueued(TurnUnqueuedParams),
31 Ended(LastTurn),
40}
41
42pub fn is_launch_failure(outcome: &TurnOutcome) -> bool {
45 let (terminal, error) = match outcome {
46 TurnOutcome::Completed { params, .. } => (¶ms.terminal, ¶ms.error),
47 TurnOutcome::Ended(last_turn) => (&last_turn.terminal, &last_turn.error),
48 TurnOutcome::Unqueued(_) => return false,
49 };
50 *terminal == TurnTerminal::Failed
51 && error.as_ref().map(|error| &error.kind) == Some(&TurnErrorKind::LaunchError)
52}
53
54#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
56#[non_exhaustive]
57pub enum TurnWaitError {
58 #[error("turn event source closed before a terminal outcome")]
60 SourceClosed,
61 #[error("client closed before a terminal outcome")]
63 ClientClosed,
64 #[error("turn event source failed before a terminal outcome: {message}")]
66 EventSourceFailed {
67 message: String,
69 },
70}
71
72#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
74#[non_exhaustive]
75pub enum TurnStreamError {
76 #[error(transparent)]
78 Turn(#[from] TurnWaitError),
79}
80
81pub type TurnOutcomeFuture =
83 Pin<Box<dyn Future<Output = Result<TurnOutcome, MuseError>> + Send + 'static>>;
84
85#[derive(Debug, Clone)]
86enum TurnCompletion {
87 Outcome(Box<TurnOutcome>),
88 WaitFailed(TurnWaitError),
89}
90
91impl TurnCompletion {
92 fn into_result(self) -> Result<TurnOutcome, MuseError> {
93 match self {
94 Self::Outcome(outcome) => Ok(*outcome),
95 Self::WaitFailed(TurnWaitError::SourceClosed) => Err(MuseError::TransportEnded),
96 Self::WaitFailed(TurnWaitError::ClientClosed) => Err(MuseError::ClientClosed),
97 Self::WaitFailed(TurnWaitError::EventSourceFailed { message }) => {
98 Err(ProtocolError::new(message, None).into())
99 }
100 }
101 }
102}
103
104#[derive(Debug, Clone)]
105enum StreamEvent<T> {
106 Value(T),
107 End,
108 Failed(TurnWaitError),
109}
110
111pub struct TurnStream<T: Clone> {
113 replay: VecDeque<T>,
114 receiver: Option<mpsc::UnboundedReceiver<StreamEvent<T>>>,
115 terminal: Option<Result<(), TurnStreamError>>,
116 _keepalive: Option<Arc<dyn Send + Sync>>,
117}
118
119impl<T: Clone> TurnStream<T> {
120 pub async fn recv(&mut self) -> Result<Option<T>, TurnStreamError> {
124 if let Some(value) = self.replay.pop_front() {
125 return Ok(Some(value));
126 }
127 if let Some(terminal) = &self.terminal {
128 return terminal.clone().map(|()| None);
129 }
130
131 let Some(receiver) = &mut self.receiver else {
132 return Ok(None);
133 };
134 match receiver.recv().await {
135 Some(StreamEvent::Value(value)) => Ok(Some(value)),
136 Some(StreamEvent::End) => self.record_terminal(Ok(())),
137 Some(StreamEvent::Failed(error)) => self.record_terminal(Err(error.into())),
138 None => self.record_terminal(Err(TurnWaitError::SourceClosed.into())),
139 }
140 }
141
142 fn record_terminal(
143 &mut self,
144 terminal: Result<(), TurnStreamError>,
145 ) -> Result<Option<T>, TurnStreamError> {
146 self.receiver = None;
147 self.terminal = Some(terminal.clone());
148 terminal.map(|()| None)
149 }
150}
151
152struct TurnState {
153 observed_start: bool,
154 server_settlement: Option<Result<(), TurnWaitError>>,
155 discarded_command_id: Option<String>,
156 discard: Option<Weak<DiscardBinding>>,
157 items: Vec<Item>,
158 item_streams: Vec<mpsc::UnboundedSender<StreamEvent<Item>>>,
159 delta_streams: Vec<mpsc::UnboundedSender<StreamEvent<ItemDeltaParams>>>,
160}
161
162struct TurnInner {
163 turn_id: String,
164 state: Mutex<TurnState>,
165 outcome: watch::Sender<Option<TurnCompletion>>,
166}
167
168impl TurnInner {
169 fn discard(&self, discarded: &watch::Sender<bool>, command_id: &str) -> bool {
170 let mut state = lock_state(&self.state);
171 if state.server_settlement.is_some() {
172 return false;
173 }
174 state.discarded_command_id = Some(command_id.to_owned());
175 discarded.send_replace(true);
176 true
177 }
178}
179
180struct DiscardBinding {
181 inner: Weak<TurnInner>,
182 command_id: String,
183 discarded: watch::Sender<bool>,
184}
185
186impl DiscardBinding {
187 fn discard(&self) {
188 let Some(inner) = self.inner.upgrade() else {
189 return;
190 };
191 inner.discard(&self.discarded, &self.command_id);
192 }
193}
194
195impl Drop for DiscardBinding {
196 fn drop(&mut self) {
197 self.discard();
198 }
199}
200
201#[derive(Clone)]
207pub struct Turn {
208 inner: Arc<TurnInner>,
209 discard: Option<Arc<DiscardBinding>>,
210 keepalive: Option<Arc<dyn Send + Sync>>,
211}
212
213impl Turn {
214 pub fn turn_id(&self) -> &str {
216 &self.inner.turn_id
217 }
218
219 pub fn observed_start(&self) -> bool {
221 self.lock_state().observed_start
222 }
223
224 pub(crate) fn with_discard(mut self, command_id: String) -> Self {
225 let (discarded, _) = watch::channel(false);
226 let discard = Arc::new(DiscardBinding {
227 inner: Arc::downgrade(&self.inner),
228 command_id,
229 discarded,
230 });
231 self.lock_state().discard = Some(Arc::downgrade(&discard));
232 self.discard = Some(discard);
233 self
234 }
235
236 pub fn completed(&self) -> TurnOutcomeFuture {
242 let mut receiver = self.inner.outcome.subscribe();
243 let mut discard = self
244 .discard
245 .as_ref()
246 .map(|discard| (discard.discarded.subscribe(), discard.command_id.clone()));
247 let keepalive = self.keepalive.clone();
248 Box::pin(async move {
249 let _keepalive = keepalive;
250 let mut discard_open = discard.is_some();
251 loop {
252 if let Some((discarded, command_id)) = &discard
253 && *discarded.borrow()
254 {
255 return Err(MuseError::Discarded {
256 command_id: command_id.clone(),
257 });
258 }
259 if let Some(outcome) = receiver.borrow().clone() {
260 return outcome.into_result();
261 }
262 tokio::select! {
263 biased;
264 changed = async {
265 let Some((discarded, _)) = discard.as_mut() else {
266 return std::future::pending().await;
267 };
268 discarded.changed().await
269 }, if discard_open => {
270 if changed.is_err() {
271 discard_open = false;
272 }
273 }
274 changed = receiver.changed() => {
275 if changed.is_err() {
276 return Err(MuseError::TransportEnded);
277 }
278 }
279 }
280 }
281 })
282 }
283
284 pub fn items(&self) -> TurnStream<Item> {
287 let (replay, receiver, terminal) = {
288 let mut state = self.lock_state();
289 let replay = state.items.clone().into();
290 let (receiver, terminal) = match &state.server_settlement {
291 None => {
292 let (sender, receiver) = mpsc::unbounded_channel();
293 state.item_streams.push(sender);
294 (Some(receiver), None)
295 }
296 Some(Ok(_)) => (None, Some(Ok(()))),
297 Some(Err(error)) => (None, Some(Err(error.clone().into()))),
298 };
299 (replay, receiver, terminal)
300 };
301 TurnStream {
302 replay,
303 receiver,
304 terminal,
305 _keepalive: self.keepalive.clone(),
306 }
307 }
308
309 pub fn deltas(&self) -> TurnStream<ItemDeltaParams> {
312 let mut state = self.lock_state();
313 let (receiver, terminal) = match &state.server_settlement {
314 None => {
315 let (sender, receiver) = mpsc::unbounded_channel();
316 state.delta_streams.push(sender);
317 (Some(receiver), None)
318 }
319 Some(Ok(_)) => (None, Some(Ok(()))),
320 Some(Err(error)) => (None, Some(Err(error.clone().into()))),
321 };
322 TurnStream {
323 replay: VecDeque::new(),
324 receiver,
325 terminal,
326 _keepalive: self.keepalive.clone(),
327 }
328 }
329
330 fn lock_state(&self) -> MutexGuard<'_, TurnState> {
331 lock_state(&self.inner.state)
332 }
333}
334
335#[derive(Clone)]
337pub(crate) struct TurnHandle {
338 inner: Arc<TurnInner>,
339}
340
341impl TurnHandle {
342 pub(crate) fn new(turn_id: impl Into<String>) -> Self {
343 let (outcome, _) = watch::channel(None);
344 Self {
345 inner: Arc::new(TurnInner {
346 turn_id: turn_id.into(),
347 state: Mutex::new(TurnState {
348 observed_start: false,
349 server_settlement: None,
350 discarded_command_id: None,
351 discard: None,
352 items: Vec::new(),
353 item_streams: Vec::new(),
354 delta_streams: Vec::new(),
355 }),
356 outcome,
357 }),
358 }
359 }
360
361 pub(crate) fn turn(&self) -> Turn {
362 Turn {
363 inner: Arc::clone(&self.inner),
364 discard: self.discard(),
365 keepalive: None,
366 }
367 }
368
369 pub(crate) fn turn_with_keepalive<T>(&self, keepalive: Arc<T>) -> Turn
370 where
371 T: Send + Sync + 'static,
372 {
373 Turn {
374 inner: Arc::clone(&self.inner),
375 discard: self.discard(),
376 keepalive: Some(keepalive),
377 }
378 }
379
380 fn discard(&self) -> Option<Arc<DiscardBinding>> {
381 self.lock_state().discard.as_ref().and_then(Weak::upgrade)
382 }
383
384 pub(crate) fn mark_started(&self) {
385 self.lock_state().observed_start = true;
386 }
387
388 pub(crate) fn observed_start(&self) -> bool {
389 self.lock_state().observed_start
390 }
391
392 pub(crate) fn is_settled(&self) -> bool {
393 self.lock_state().server_settlement.is_some()
394 }
395
396 pub(crate) fn has_server_outcome(&self) -> bool {
397 matches!(self.lock_state().server_settlement, Some(Ok(())))
398 }
399
400 pub(crate) fn is_command_discarded(&self, command_id: &str) -> bool {
401 self.lock_state().discarded_command_id.as_deref() == Some(command_id)
402 }
403
404 pub(crate) fn discarded_command_id(&self) -> Option<String> {
405 self.lock_state().discarded_command_id.clone()
406 }
407
408 pub(crate) fn held_items(&self) -> Vec<Item> {
410 self.lock_state().items.clone()
411 }
412
413 pub(crate) fn push_item(&self, item: Item) {
414 let mut state = self.lock_state();
415 if state.server_settlement.is_none() {
416 state.items.push(item.clone());
417 state
418 .item_streams
419 .retain(|stream| stream.send(StreamEvent::Value(item.clone())).is_ok());
420 }
421 }
422
423 pub(crate) fn push_delta(&self, delta: ItemDeltaParams) {
424 let mut state = self.lock_state();
425 if state.server_settlement.is_none() {
426 state
427 .delta_streams
428 .retain(|stream| stream.send(StreamEvent::Value(delta.clone())).is_ok());
429 }
430 }
431
432 pub(crate) fn settle_completed(&self, params: TurnCompletedParams) {
433 self.settle(|state| {
434 Ok(TurnOutcome::Completed {
435 params,
436 observed_start: state.observed_start,
437 })
438 });
439 }
440
441 pub(crate) fn settle_unqueued(&self, params: TurnUnqueuedParams) {
442 self.settle(|_| Ok(TurnOutcome::Unqueued(params)));
443 }
444
445 pub(crate) fn settle_ended(&self, last_turn: LastTurn) {
446 self.settle(|_| Ok(TurnOutcome::Ended(last_turn)));
447 }
448
449 pub(crate) fn fail(&self, error: TurnWaitError) {
450 self.settle(|_| Err(error));
451 }
452
453 fn settle(
454 &self,
455 make_settlement: impl FnOnce(&TurnState) -> Result<TurnOutcome, TurnWaitError>,
456 ) {
457 let mut state = self.lock_state();
458 if state.server_settlement.is_some() {
459 return;
460 }
461 let settlement = make_settlement(&state);
462 state.server_settlement = Some(settlement.clone().map(|_| ()));
463 let completion = match &settlement {
464 Ok(outcome) => TurnCompletion::Outcome(Box::new(outcome.clone())),
465 Err(error) => TurnCompletion::WaitFailed(error.clone()),
466 };
467 self.inner.outcome.send_replace(Some(completion));
468 match settlement {
469 Ok(_) => {
470 for stream in state.item_streams.drain(..) {
471 let _ = stream.send(StreamEvent::End);
472 }
473 for stream in state.delta_streams.drain(..) {
474 let _ = stream.send(StreamEvent::End);
475 }
476 }
477 Err(error) => {
478 for stream in state.item_streams.drain(..) {
479 let _ = stream.send(StreamEvent::Failed(error.clone()));
480 }
481 for stream in state.delta_streams.drain(..) {
482 let _ = stream.send(StreamEvent::Failed(error.clone()));
483 }
484 }
485 }
486 }
487
488 fn lock_state(&self) -> MutexGuard<'_, TurnState> {
489 lock_state(&self.inner.state)
490 }
491}
492
493fn lock_state<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
494 match mutex.lock() {
495 Ok(state) => state,
496 Err(poisoned) => poisoned.into_inner(),
497 }
498}
499
500#[cfg(test)]
501mod tests {
502 use std::task::{Context, Poll, Waker};
503
504 use super::*;
505
506 #[test]
511 fn turn_outcome_contains_only_server_authored_terminals() {
512 fn assert_exhaustive(outcome: TurnOutcome) {
513 match outcome {
514 TurnOutcome::Completed { .. }
515 | TurnOutcome::Unqueued(_)
516 | TurnOutcome::Ended(_) => {}
517 }
518 }
519
520 let _ = assert_exhaustive;
521 }
522
523 #[test]
526 fn a_launch_failure_carried_by_ended_is_a_launch_failure() {
527 let last_turn = |kind| {
528 serde_json::from_value::<LastTurn>(serde_json::json!({
529 "turnId": "turn-1",
530 "terminal": "failed",
531 "error": {"kind": kind, "message": "launch failed", "retryable": true}
532 }))
533 };
534 let launch = last_turn("launchError");
535 let other = last_turn("modelError");
536 assert!(launch.is_ok() && other.is_ok(), "the fixtures decode");
537 let (Ok(launch), Ok(other)) = (launch, other) else {
538 return;
539 };
540 assert!(is_launch_failure(&TurnOutcome::Ended(launch)));
541 assert!(!is_launch_failure(&TurnOutcome::Ended(other)));
542 }
543
544 fn ready<Output>(future: impl Future<Output = Output>) -> Option<Output> {
545 let waker = Waker::noop();
546 let mut context = Context::from_waker(waker);
547 let mut future = std::pin::pin!(future);
548 match future.as_mut().poll(&mut context) {
549 Poll::Ready(output) => Some(output),
550 Poll::Pending => None,
551 }
552 }
553
554 fn assert_sticky_error(mut stream: TurnStream<u8>, expected: TurnStreamError) {
555 assert_eq!(ready(stream.recv()), Some(Err(expected.clone())));
556 assert_eq!(ready(stream.recv()), Some(Err(expected)));
557 }
558
559 #[test]
560 fn stream_terminal_errors_remain_sticky() {
561 let source_failure = TurnWaitError::EventSourceFailed {
562 message: "connection lost".to_owned(),
563 };
564 let (failed_sender, failed_receiver) = mpsc::unbounded_channel();
565 assert!(
566 failed_sender
567 .send(StreamEvent::Failed(source_failure.clone()))
568 .is_ok(),
569 "the stream receiver is active"
570 );
571 assert_sticky_error(
572 TurnStream {
573 replay: VecDeque::new(),
574 receiver: Some(failed_receiver),
575 terminal: None,
576 _keepalive: None,
577 },
578 TurnStreamError::Turn(source_failure),
579 );
580
581 let (closed_sender, closed_receiver) = mpsc::unbounded_channel();
582 drop(closed_sender);
583 assert_sticky_error(
584 TurnStream {
585 replay: VecDeque::new(),
586 receiver: Some(closed_receiver),
587 terminal: None,
588 _keepalive: None,
589 },
590 TurnStreamError::Turn(TurnWaitError::SourceClosed),
591 );
592 }
593
594 #[test]
595 fn item_replay_is_independent_for_each_reader() -> Result<(), serde_json::Error> {
596 let handle = TurnHandle::new("turn-1");
597 let item: Item = serde_json::from_value(serde_json::json!({
598 "itemId": "item-1",
599 "kind": "agentMessage",
600 "turnId": "turn-1",
601 "revision": 1,
602 "status": "completed",
603 "text": "hello"
604 }))?;
605 handle.push_item(item.clone());
606
607 let mut first = handle.turn().items();
608 let mut second = handle.turn().items();
609 assert_eq!(ready(first.recv()), Some(Ok(Some(item.clone()))));
610 assert_eq!(ready(second.recv()), Some(Ok(Some(item))));
611 Ok(())
612 }
613
614 #[test]
615 fn readers_can_join_late_and_replay_after_completion() -> Result<(), serde_json::Error> {
616 let handle = TurnHandle::new("turn-1");
617 let first: Item = serde_json::from_value(serde_json::json!({
618 "itemId": "item-1",
619 "kind": "agentMessage",
620 "turnId": "turn-1",
621 "revision": 1,
622 "status": "completed",
623 "text": "first"
624 }))?;
625 let second: Item = serde_json::from_value(serde_json::json!({
626 "itemId": "item-2",
627 "kind": "agentMessage",
628 "turnId": "turn-1",
629 "revision": 1,
630 "status": "completed",
631 "text": "second"
632 }))?;
633 handle.push_item(first.clone());
634 let mut early = handle.turn().items();
635 handle.push_item(second.clone());
636 handle.fail(TurnWaitError::EventSourceFailed {
637 message: "terminal test failure".to_owned(),
638 });
639
640 assert_eq!(ready(early.recv()), Some(Ok(Some(first.clone()))));
641 assert_eq!(ready(early.recv()), Some(Ok(Some(second.clone()))));
642 assert!(matches!(
643 ready(early.recv()),
644 Some(Err(TurnStreamError::Turn(
645 TurnWaitError::EventSourceFailed { .. }
646 )))
647 ));
648
649 let mut late = handle.turn().items();
650 assert_eq!(ready(late.recv()), Some(Ok(Some(first))));
651 assert_eq!(ready(late.recv()), Some(Ok(Some(second))));
652 assert!(matches!(
653 ready(late.recv()),
654 Some(Err(TurnStreamError::Turn(
655 TurnWaitError::EventSourceFailed { .. }
656 )))
657 ));
658 Ok(())
659 }
660}