reef_core/runtime/run.rs
1//! Public Run handle and the sole tracked runtime owner.
2
3use std::error::Error;
4use std::fmt;
5use std::future::Future;
6use std::panic::{AssertUnwindSafe, catch_unwind, resume_unwind};
7use std::pin::Pin;
8use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
9use std::sync::{Arc, Mutex};
10use std::task::{Context, Poll};
11
12use tokio::sync::Notify;
13
14use super::agent::start::AdmittedStart;
15use super::agent::{AttachmentGuard, Revision};
16use super::driver;
17use super::event::{Event, EventSource, InputOrigin, Producer, StopCode, Stream, ValidationError};
18use super::hook::{
19 self, BufferedFact, CallbackResources, FactBuffer, LifecycleEnd, LifecycleObservation,
20 TranscriptSource,
21};
22use super::model::Client;
23use super::session::{Lease, Session};
24use crate::audit;
25use crate::state::run_state::{self};
26use crate::state::session_state::{self};
27use crate::transcript::controller::TranscriptController;
28use crate::transcript::item::Item;
29use crate::value::lifecycle::{Coord, Outcome};
30
31/// A cloneable handle to one admitted Run.
32///
33/// A Run exists only because [`Agent::start`](super::agent::Agent::start) or
34/// [`Agent::start_observed`](super::agent::Agent::start_observed) admitted a Session,
35/// so by the time you hold one the input is already recorded and the loop is already
36/// running. Ask it to stop and wait for its [`Terminal`]. When `start` constructed it,
37/// also take its [`take_events`](Run::take_events) once; `start_observed` instead returns
38/// that unique stream atomically beside the Run.
39///
40/// Every clone shares one Run. Dropping the last handle asks it to stop; it does
41/// not wait, and it does not abandon work already in flight.
42///
43/// ```
44/// # use std::future::{Future, ready};
45/// # use std::pin::Pin;
46/// # use std::sync::Arc;
47/// # use std::task::{Context, Poll};
48/// # use reef_core::futures_core::Stream;
49/// # use reef_core::runtime::model;
50/// # #[derive(Debug)] struct Offline;
51/// # #[derive(Debug)] struct Once(bool);
52/// # impl Stream for Once {
53/// # type Item = Result<model::Event, std::convert::Infallible>;
54/// # fn poll_next(mut self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<Option<Self::Item>> {
55/// # if std::mem::replace(&mut self.0, true) { return Poll::Ready(None); }
56/// # Poll::Ready(Some(Ok(model::Event::Done { finish_reason: "stop".to_owned(), completion: model::CompletionReason::Stop, provider_data: Vec::new() })))
57/// # }
58/// # }
59/// # impl model::Client for Offline {
60/// # type Error = std::convert::Infallible;
61/// # type Stream = Once;
62/// # fn info(&self) -> model::Info { model::Info::new("offline".to_owned(), 8_192) }
63/// # fn generate(&self, _: model::Request) -> impl Future<Output = Result<Self::Stream, Self::Error>> + Send { ready(Ok(Once(false))) }
64/// # }
65/// use reef_core::cancel;
66/// use reef_core::runtime::{agent, hook, run, session, tool};
67/// use reef_core::value::{content, lifecycle};
68///
69/// # #[tokio::main(flavor = "current_thread")]
70/// # async fn main() {
71/// # let config = agent::Config::new(Arc::new(Offline), tool::RegistryBuilder::new().build().unwrap(), hook::RegistryBuilder::new());
72/// # let agent = agent::Agent::new(config).unwrap();
73/// let session = session::Session::new("conversation-1".to_owned(), None);
74/// let (started, mut events) = agent
75/// .start_observed(session.clone(), vec![content::Part::text("Hi.".to_owned())], cancel::Token::new())
76/// .await
77/// .unwrap();
78///
79/// // Events are single-consumer: the atomic handoff means the Run and every
80/// // clone report that its stream was already taken.
81/// assert!(started.clone().take_events().is_err());
82/// while let Some(delivery) = events.recv().await {
83/// if delivery.decision_point().is_some() {
84/// delivery.accept().unwrap();
85/// }
86/// }
87///
88/// // Every waiter gets the same terminal, and waiting again is free.
89/// let terminal = started.wait().await;
90/// assert_eq!(terminal.outcome(), lifecycle::Outcome::Completed);
91/// assert_eq!(started.wait().await.outcome(), terminal.outcome());
92/// # }
93/// ```
94pub struct Run<M: Client> {
95 inner: Arc<RunInner<M>>,
96}
97
98impl<M: Client> fmt::Debug for Run<M> {
99 /// Locates the Run by session and revision, and counts live handles.
100 ///
101 /// There is no `M: Debug` bound: a Run would otherwise be unprintable for
102 /// every provider that has not opted in, and the model is not what
103 /// distinguishes one Run from another anyway. The three facts here are all
104 /// lock-free, so formatting a Run can never stall the Run.
105 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
106 formatter
107 .debug_struct("Run")
108 .field("session", &self.inner.session.id())
109 .field("revision", &self.inner.revision.id())
110 .field(
111 "handles",
112 &self.inner.public_handles.load(Ordering::Relaxed),
113 )
114 .finish()
115 }
116}
117
118impl<M: Client> Clone for Run<M> {
119 fn clone(&self) -> Self {
120 self.inner.public_handles.fetch_add(1, Ordering::Relaxed);
121 Self {
122 inner: Arc::clone(&self.inner),
123 }
124 }
125}
126
127impl<M: Client> Drop for Run<M> {
128 fn drop(&mut self) {
129 if self.inner.public_handles.fetch_sub(1, Ordering::AcqRel) == 1 {
130 self.inner.source.request_stop();
131 }
132 }
133}
134
135struct RunInner<M: Client> {
136 revision: Revision<M>,
137 session: Session,
138 source: Arc<EventSource>,
139 events: Mutex<Option<Stream>>,
140 terminal: Mutex<Option<Arc<Terminal>>>,
141 done: Notify,
142 public_handles: AtomicUsize,
143 task: Mutex<Option<tokio::task::JoinHandle<()>>>,
144}
145
146struct TaskExitState {
147 entered_task_body: AtomicBool,
148 task_panicked: AtomicBool,
149 fallback_armed: AtomicBool,
150}
151
152/// Publishes the abnormal terminal after the inner future has released resources.
153struct TaskFallback<M: Client> {
154 inner: Arc<RunInner<M>>,
155 identity: Coord,
156 state: Arc<TaskExitState>,
157}
158
159impl<M: Client> Drop for TaskFallback<M> {
160 fn drop(&mut self) {
161 if !self.state.fallback_armed.swap(false, Ordering::AcqRel) {
162 return;
163 }
164 let failure: Arc<dyn Error + Send + Sync> =
165 if !self.state.entered_task_body.load(Ordering::Acquire) {
166 Arc::new(TaskNotStartedError)
167 } else if self.state.task_panicked.load(Ordering::Acquire) {
168 Arc::new(TaskPanickedError)
169 } else {
170 Arc::new(TaskAbortedError)
171 };
172 self.inner.publish(Arc::new(Terminal::failed(
173 self.identity.clone(),
174 FailureKind::Internal,
175 failure,
176 )));
177 }
178}
179
180/// Marks only panics raised while polling the Run's inner task body.
181struct PanicMarkerFuture<F, M: Client> {
182 // Fields drop in declaration order: inner cleanup must precede fallback
183 // publication on every abnormal exit.
184 inner: Pin<Box<F>>,
185 fallback: TaskFallback<M>,
186}
187
188impl<F, M: Client> PanicMarkerFuture<F, M> {
189 fn new(inner: F, fallback: TaskFallback<M>) -> Self {
190 Self {
191 inner: Box::pin(inner),
192 fallback,
193 }
194 }
195}
196
197impl<F: Future, M: Client> Future for PanicMarkerFuture<F, M> {
198 type Output = F::Output;
199
200 fn poll(self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll<Self::Output> {
201 let this = self.get_mut();
202 match catch_unwind(AssertUnwindSafe(|| this.inner.as_mut().poll(context))) {
203 Ok(poll) => poll,
204 Err(payload) => {
205 this.fallback
206 .state
207 .task_panicked
208 .store(true, Ordering::Release);
209 resume_unwind(payload);
210 }
211 }
212 }
213}
214
215/// Linear owner of the post-admission cleanup chain.
216///
217/// Options make every step idempotent if an individual destructor itself unwinds.
218struct TaskCleanup<M: Client> {
219 inner: Arc<RunInner<M>>,
220 lease: Option<Lease>,
221 attachment: Option<AttachmentGuard<M>>,
222 producer: Option<Arc<Producer>>,
223 state: Arc<TaskExitState>,
224 armed: bool,
225}
226
227impl<M: Client> TaskCleanup<M> {
228 const fn new(
229 inner: Arc<RunInner<M>>,
230 lease: Lease,
231 attachment: AttachmentGuard<M>,
232 producer: Arc<Producer>,
233 state: Arc<TaskExitState>,
234 ) -> Self {
235 Self {
236 inner,
237 lease: Some(lease),
238 attachment: Some(attachment),
239 producer: Some(producer),
240 state,
241 armed: true,
242 }
243 }
244
245 fn enter_task_body(&mut self) {
246 self.state.entered_task_body.store(true, Ordering::Release);
247 }
248
249 const fn lease(&self) -> &Lease {
250 self.lease
251 .as_ref()
252 .expect("the Run Lease remains owned until terminal cleanup")
253 }
254
255 const fn producer(&self) -> &Arc<Producer> {
256 self.producer
257 .as_ref()
258 .expect("the event Producer remains open through RunEnd")
259 }
260
261 fn cleanup(&mut self) {
262 // Cancellation is intentionally after terminal freeze on the normal path.
263 // EventSource owns the Run-local child token, so this does not cancel the
264 // caller's parent token.
265 self.inner.source.request_stop();
266 // Lease::drop performs the same idempotent synchronous release.
267 drop(self.lease.take());
268 drop(self.attachment.take());
269 drop(self.producer.take());
270 self.inner.close_events();
271 }
272
273 fn complete(mut self, terminal: Terminal) {
274 self.cleanup();
275 self.armed = false;
276 self.state.fallback_armed.store(false, Ordering::Release);
277 self.inner.publish(Arc::new(terminal));
278 }
279}
280
281impl<M: Client> Drop for TaskCleanup<M> {
282 fn drop(&mut self) {
283 if !self.armed {
284 return;
285 }
286 self.cleanup();
287 self.armed = false;
288 }
289}
290
291impl<M: Client + 'static> Run<M> {
292 pub(crate) fn spawn(admitted: AdmittedStart<M>) -> Self {
293 Self::spawn_inner(admitted, |events| (Some(events), ())).0
294 }
295
296 pub(crate) fn spawn_observed(admitted: AdmittedStart<M>) -> (Self, Stream) {
297 Self::spawn_inner(admitted, |events| (None, events))
298 }
299
300 fn spawn_inner<R>(
301 admitted: AdmittedStart<M>,
302 place_stream: impl FnOnce(Stream) -> (Option<Stream>, R),
303 ) -> (Self, R) {
304 // Parsed from the `AdmittedStart` owner produced by either public Agent
305 // start entry point through their one shared admission path.
306 let AdmittedStart {
307 lease,
308 attachment,
309 revision,
310 session,
311 input,
312 admitted_items,
313 plan,
314 identity,
315 cancellation,
316 producer,
317 mut source,
318 hooks,
319 reservation_resources,
320 run_states,
321 session_state,
322 fact_buffer,
323 } = admitted;
324 let events = source
325 .take_stream()
326 .expect("a freshly admitted Run owns its event stream");
327 // Select the receiver's owner before spawning: terminal cleanup can only
328 // reach the `Option` stored in `RunInner`, never the separately returned value.
329 let (run_events, returned) = place_stream(events);
330 let source = Arc::new(source);
331 let inner = Arc::new(RunInner {
332 revision: revision.clone(),
333 session: session.clone(),
334 source: Arc::clone(&source),
335 events: Mutex::new(run_events),
336 terminal: Mutex::new(None),
337 done: Notify::new(),
338 public_handles: AtomicUsize::new(1),
339 task: Mutex::new(None),
340 });
341 // Establish both cleanup and its trailing terminal fallback before spawning.
342 // Tokio may discard an unpolled task during runtime shutdown, so constructing
343 // either owner inside the async body would lose the pre-poll terminal path.
344 let task_state = Arc::new(TaskExitState {
345 entered_task_body: AtomicBool::new(false),
346 task_panicked: AtomicBool::new(false),
347 fallback_armed: AtomicBool::new(true),
348 });
349 let fallback = TaskFallback {
350 inner: Arc::clone(&inner),
351 identity: identity.clone(),
352 state: Arc::clone(&task_state),
353 };
354 let mut cleanup =
355 TaskCleanup::new(Arc::clone(&inner), lease, attachment, producer, task_state);
356
357 let task_body = async move {
358 // This is deliberately the task body's first operation: cleanup can then
359 // distinguish an executor dropping an unpolled future from a panic after
360 // the task started running.
361 cleanup.enter_task_body();
362 // The admitted input is already a transcript record (admission commits
363 // it in runtime/agent/start.rs), so it reaches the model the same way every
364 // other record does. `ephemeral` keeps its actual meaning: content
365 // that belongs to one request and to no history.
366 let lifecycle = LifecycleFanout {
367 hooks: &hooks,
368 run_states: &run_states,
369 identity: &identity,
370 session_state: &session_state,
371 session_state_writer: cleanup.lease().state(),
372 transcript: cleanup.lease().transcript_reader(),
373 emitter: &fact_buffer,
374 };
375 lifecycle.observe(&LifecycleObservation::RunStart {
376 coord: identity.clone(),
377 input,
378 });
379
380 // Admission and `RunStart` both ran synchronous callbacks into the
381 // buffer, and no Producer existed then. Draining before Driver work
382 // preserves their producer FIFO; the loop then owns every drain in
383 // between, and the second drain below picks up `RunEnd`.
384 let report = driver::Report::default();
385 let admission_failure = publish_admission(
386 cleanup.producer(),
387 &fact_buffer,
388 &lifecycle,
389 &admitted_items,
390 )
391 .await
392 .err();
393 let driver_terminal = if let Some(error) = admission_failure {
394 driver::Terminal::Failed(driver::Error::Event(error))
395 } else {
396 let callback_resources = run_states
397 .iter()
398 .map(|run_state| {
399 CallbackResources::new(
400 &identity,
401 run_state,
402 &session_state,
403 Some(cleanup.lease().state()),
404 &fact_buffer,
405 )
406 })
407 .collect::<Vec<_>>();
408 let controller = TranscriptController::new(
409 cleanup.lease(),
410 &hooks,
411 &reservation_resources,
412 &identity,
413 &fact_buffer,
414 );
415 let tool_phase = driver::ToolPhase::new(
416 driver::Outward::new(cleanup.producer(), &fact_buffer),
417 &hooks,
418 &callback_resources,
419 &controller,
420 &identity,
421 Some(session.id().to_owned()),
422 );
423 let inbox = session.input_mailbox();
424 let engine = driver::Driver::new(
425 revision.model().as_ref(),
426 cleanup.lease(),
427 &inbox,
428 revision.tools(),
429 &cancellation,
430 )
431 .with_tool_phase(tool_phase)
432 .with_turn_trigger(admitted_items)
433 .with_report(&report);
434 engine.run(plan, &source).await
435 };
436
437 // `Driver::run` is the sole inner join barrier.
438 // Construct one immutable pre-fan-out candidate and give that exact
439 // observation to every registration while the Lease remains held.
440 let stop_code = report.stop_code();
441 let candidate =
442 terminal_from_driver(&driver_terminal, identity.clone(), stop_code.clone());
443 let observation = LifecycleObservation::RunEnd {
444 coord: identity.clone(),
445 end: lifecycle_end(&candidate),
446 turns: report.turns(),
447 };
448 lifecycle.observe(&observation);
449 // RunEnd audits/panic facts append to the same FactBuffer, and the
450 // loop has already published everything queued before it. This is the
451 // last drain there is, so it is also what carries any fact the loop
452 // queued after its own final publication point.
453 let run_end_failure =
454 publish_buffered_facts(cleanup.producer(), &fact_buffer, Some(&lifecycle))
455 .await
456 .err();
457
458 // Re-observe only validated control/fatal state after peer fan-out.
459 // TerminalLatch preserves the first nonfatal and first fatal rules.
460 let mut terminal = driver::TerminalLatch::default();
461 terminal.record(driver_terminal);
462 if let Some(error) = run_end_failure {
463 terminal.record(driver::Terminal::Failed(driver::Error::Event(error)));
464 }
465 let terminal = terminal_from_driver(&terminal.finish(&source), identity, stop_code);
466
467 // Freeze is complete before cancellation. Cleanup then performs the
468 // exact cancel -> release -> detach -> event close -> publish order.
469 cleanup.complete(terminal);
470 };
471 let task = tokio::spawn(PanicMarkerFuture::new(task_body, fallback));
472 *inner.task.lock().expect("Run task lock poisoned") = Some(task);
473
474 (Self { inner }, returned)
475 }
476}
477
478impl<M: Client> Run<M> {
479 /// Returns the immutable Agent revision frozen for this Run.
480 #[must_use]
481 pub fn revision(&self) -> Revision<M> {
482 self.inner.revision.clone()
483 }
484
485 /// Returns the Session owned by this Run.
486 #[must_use]
487 pub fn session(&self) -> Session {
488 self.inner.session.clone()
489 }
490
491 /// Takes the unique outward event consumer.
492 ///
493 /// # Errors
494 ///
495 /// [`EventsTakenError`] if the [`Stream`] was returned atomically by
496 /// [`Agent::start_observed`](super::agent::Agent::start_observed), or has already
497 /// been taken by an earlier call through this handle or any clone. Events are
498 /// single-consumer, so this is not contention that clears: the one consumer
499 /// already exists, and whoever holds it is the only party that can observe the
500 /// Run's events.
501 ///
502 /// # Panics
503 ///
504 /// If the lock guarding the not-yet-taken [`Stream`] is poisoned, per the
505 /// crate-level [lock-poisoning contract](crate#lock-poisoning). It is held
506 /// only for the take itself, so nothing a caller or a hook does runs under
507 /// it.
508 pub fn take_events(&self) -> Result<Stream, EventsTakenError> {
509 self.inner
510 .events
511 .lock()
512 .expect("Run event lock poisoned")
513 .take()
514 .ok_or(EventsTakenError)
515 }
516
517 /// Requests a normal stop without waiting or spawning cleanup work.
518 ///
519 /// # Panics
520 ///
521 /// If an event-control lock is poisoned, per the crate-level
522 /// [lock-poisoning contract](crate#lock-poisoning).
523 pub fn request_stop(&self) {
524 self.inner.source.request_stop();
525 }
526
527 /// Requests a normal stop and waits for complete cleanup.
528 ///
529 /// # Panics
530 ///
531 /// If an event-control or Run terminal lock is poisoned, per the crate-level
532 /// [lock-poisoning contract](crate#lock-poisoning).
533 pub async fn stop(&self) -> Arc<Terminal> {
534 self.request_stop();
535 self.wait().await
536 }
537
538 /// Waits for the shared terminal without requesting a stop.
539 ///
540 /// Every waiter, on this handle or any clone, receives the same
541 /// [`Terminal`], and waiting again after it has resolved returns
542 /// immediately. If its [`outcome`](Terminal::outcome) is
543 /// [`Failed`](Outcome::Failed), read [`Terminal::failure_kind`] and
544 /// [`Terminal::failure`] rather than reporting the outcome alone.
545 ///
546 /// # Panics
547 ///
548 /// If the Run terminal lock is poisoned, per the crate-level
549 /// [lock-poisoning contract](crate#lock-poisoning).
550 pub async fn wait(&self) -> Arc<Terminal> {
551 loop {
552 let notified = self.inner.done.notified();
553 tokio::pin!(notified);
554 notified.as_mut().enable();
555 if let Some(terminal) = self.inner.terminal() {
556 return terminal;
557 }
558 notified.await;
559 }
560 }
561
562 /// Abandons an event consumer still owned by this Run, requests stop, and waits.
563 ///
564 /// A stream returned by [`Agent::start_observed`](super::agent::Agent::start_observed)
565 /// is caller-owned, so this method closes the producer but cannot discard that
566 /// stream or its already queued events; the caller may drain them to EOF.
567 ///
568 /// # Panics
569 ///
570 /// If a Run or event-control lock is poisoned, per the crate-level
571 /// [lock-poisoning contract](crate#lock-poisoning).
572 pub async fn close(&self) -> Arc<Terminal> {
573 self.inner.close_events();
574 self.stop().await
575 }
576
577 #[cfg(test)]
578 pub(super) fn terminal_ready(&self) -> bool {
579 self.inner.terminal().is_some()
580 }
581
582 #[cfg(test)]
583 pub(super) async fn join_tracked_task(&self) -> Result<(), tokio::task::JoinError> {
584 let task = self
585 .inner
586 .task
587 .lock()
588 .expect("Run task lock poisoned")
589 .take()
590 .expect("the sole tracked Run task is joined at most once");
591 task.await
592 }
593}
594
595impl<M: Client> RunInner<M> {
596 fn close_events(&self) {
597 drop(self.events.lock().expect("Run event lock poisoned").take());
598 }
599
600 fn terminal(&self) -> Option<Arc<Terminal>> {
601 self.terminal
602 .lock()
603 .expect("Run terminal lock poisoned")
604 .clone()
605 }
606
607 fn publish(&self, terminal: Arc<Terminal>) {
608 let mut slot = self.terminal.lock().expect("Run terminal lock poisoned");
609 debug_assert!(slot.is_none(), "the sole Run task publishes once");
610 *slot = Some(terminal);
611 drop(slot);
612 self.done.notify_waiters();
613 }
614}
615
616/// The Run-scoped half of the observer lifecycle fan-out.
617///
618/// The driver owns every Turn/Step/Generation/ToolCall observation because it
619/// owns those boundaries; `RunStart` and `RunEnd` are the two the Run task owns,
620/// and both need the same borrowed resource set. Binding it once keeps the two
621/// fan-outs from drifting apart.
622pub(super) struct LifecycleFanout<'a> {
623 hooks: &'a hook::RuntimeRegistry,
624 run_states: &'a [run_state::Store],
625 identity: &'a Coord,
626 session_state: &'a session_state::Reader,
627 session_state_writer: &'a session_state::Writer,
628 transcript: crate::transcript::store::SnapshotReader,
629 emitter: &'a FactBuffer,
630}
631
632impl LifecycleFanout<'_> {
633 fn observe_audit(&self, event: &audit::Event) {
634 for adapter in self.hooks.observer_hooks() {
635 let run_state = self
636 .run_states
637 .get(adapter.index())
638 .expect("Run resources cover every hook registration");
639 let resources = CallbackResources::new(
640 self.identity,
641 run_state,
642 self.session_state,
643 Some(self.session_state_writer),
644 self.emitter,
645 );
646 adapter.invoke_audit_event(resources, &self.transcript, event);
647 }
648 }
649
650 fn observe(&self, observation: &LifecycleObservation) {
651 for adapter in self.hooks.observer_hooks() {
652 let run_state = self
653 .run_states
654 .get(adapter.index())
655 .expect("Run resources cover every hook registration");
656 let resources = CallbackResources::new(
657 self.identity,
658 run_state,
659 self.session_state,
660 Some(self.session_state_writer),
661 self.emitter,
662 );
663 adapter.invoke_lifecycle(
664 resources,
665 TranscriptSource::Reader(&self.transcript),
666 None,
667 observation,
668 );
669 }
670 }
671}
672
673/// Attempts to publish admission's outward projections before the loop begins.
674///
675/// Two facts became true in order. The queued callback facts come from
676/// [`RunHooks::prepare`](super::hook::RunHooks::prepare) and
677/// [`ObserverHooks::run_admitted`](super::hook::ObserverHooks::run_admitted), which ran with no Producer in existence;
678/// the admitted record then opens the Run's first Turn, and its best-effort
679/// `InputAdmitted` projection lets an uninterrupted renderer explain input that
680/// `prepare` may have rewritten before commit. The authoritative record is already
681/// in the Transcript: task destruction before this future is polled, or any other
682/// latched interruption, suppresses publication under C18/C20 without undoing it.
683///
684/// Nothing is published when admission recorded nothing, which happens when the
685/// caller submitted no parts and no registration composed any.
686async fn publish_admission(
687 producer: &Producer,
688 buffer: &FactBuffer,
689 lifecycle: &LifecycleFanout<'_>,
690 admitted: &[Item],
691) -> Result<(), ValidationError> {
692 publish_buffered_facts(producer, buffer, Some(lifecycle)).await?;
693 if admitted.is_empty() {
694 return Ok(());
695 }
696 producer
697 .emit_input_admitted(
698 InputOrigin::Start,
699 admitted.to_vec(),
700 lifecycle.identity.clone(),
701 )
702 .await?;
703 Ok(())
704}
705
706/// Publishes the Run-owner half of the buffered-fact drain.
707///
708/// The loop owns the drains taken between `RunStart` and `RunEnd`; this one
709/// covers the two edges the loop cannot reach, where the only coordinate that
710/// applies is the Run itself and so no event carries one.
711pub(super) async fn publish_buffered_facts(
712 producer: &Producer,
713 buffer: &FactBuffer,
714 observers: Option<&LifecycleFanout<'_>>,
715) -> Result<(), ValidationError> {
716 for fact in buffer.take_facts() {
717 let event = match fact {
718 BufferedFact::Audit(audit) => {
719 // `OnAuditEvent` observes the fact before it is published, so an
720 // observer and the outward consumer see the same value and the
721 // observer cannot answer an event the consumer already saw.
722 if let Some(observers) = observers {
723 observers.observe_audit(&audit);
724 }
725 Event::from_audit(audit)?
726 }
727 BufferedFact::Error(error) => Event::from_error(error),
728 };
729 producer
730 .emit(event, None)
731 .await
732 .expect("a successfully constructed ordinary fact remains valid");
733 }
734 Ok(())
735}
736
737fn terminal_from_driver(
738 terminal: &driver::Terminal,
739 coord: Coord,
740 stop_code: Option<StopCode>,
741) -> Terminal {
742 match terminal {
743 driver::Terminal::Completed => Terminal::new(Outcome::Completed, coord, None),
744 // Only a `RunHooks::decide_stop` stop carries a code; a consumer-requested stop
745 // has nothing to name itself with.
746 driver::Terminal::Stopped => Terminal::new(Outcome::Stopped, coord, stop_code),
747 driver::Terminal::Cancelled => Terminal::new(Outcome::Cancelled, coord, None),
748 driver::Terminal::Failed(error) => {
749 let failure = DriverError::from_loop(error);
750 Terminal::failed(coord, failure.kind, Arc::new(failure))
751 }
752 }
753}
754
755fn lifecycle_end(terminal: &Terminal) -> LifecycleEnd {
756 match terminal.outcome {
757 Outcome::Completed => LifecycleEnd::Completed,
758 Outcome::Stopped => LifecycleEnd::Stopped {
759 code: terminal.stop_code.clone(),
760 },
761 Outcome::Cancelled => LifecycleEnd::Cancelled {
762 error: Arc::new(CanceledError),
763 },
764 Outcome::Failed => LifecycleEnd::Failed {
765 error: terminal
766 .failure
767 .as_ref()
768 .expect("a failed terminal always retains failure detail")
769 .clone(),
770 },
771 }
772}
773
774/// Immutable terminal fact shared by every waiter.
775///
776/// [`outcome`](Terminal::outcome) is four words wide, so on its own it does not
777/// say much: **if it is [`Outcome::Failed`], the reason is one call away.**
778/// [`failure_kind`](Terminal::failure_kind) is the matchable classification and
779/// [`failure`](Terminal::failure) is the error itself, whose
780/// [`source`](Error::source) chain reaches the concrete provider or hook error
781/// underneath.
782///
783/// ```
784/// # use reef_core::runtime::run;
785/// # use reef_core::value::lifecycle;
786/// # fn report(terminal: &run::Terminal) -> String {
787/// if terminal.outcome() == lifecycle::Outcome::Failed {
788/// // Never report `Failed` on its own: these two say what happened.
789/// format!(
790/// "failed ({:?}): {}",
791/// terminal.failure_kind(),
792/// terminal.failure().map_or_else(|| "no detail".to_owned(), ToString::to_string),
793/// )
794/// } else {
795/// format!("{:?}", terminal.outcome())
796/// }
797/// # }
798/// ```
799pub struct Terminal {
800 outcome: Outcome,
801 coord: Coord,
802 stop_code: Option<StopCode>,
803 failure: Option<Arc<dyn Error + Send + Sync>>,
804 failure_kind: Option<FailureKind>,
805}
806
807impl fmt::Debug for Terminal {
808 /// States how the Run ended, rendering the failure through `Display`.
809 ///
810 /// The failure is an arbitrary third-party error whose `Debug` this crate
811 /// neither writes nor reviews; `Display` is the operator diagnostic the type
812 /// is contracted to produce, so that is what a terminal reports.
813 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
814 formatter
815 .debug_struct("Terminal")
816 .field("outcome", &self.outcome)
817 .field("coord", &self.coord)
818 .field("stop_code", &self.stop_code)
819 .field(
820 "failure",
821 &self.failure.as_ref().map(|failure| failure.to_string()),
822 )
823 .field("failure_kind", &self.failure_kind)
824 .finish()
825 }
826}
827
828impl Terminal {
829 fn new(outcome: Outcome, coord: Coord, stop_code: Option<StopCode>) -> Self {
830 Self {
831 outcome,
832 coord,
833 stop_code,
834 failure: None,
835 failure_kind: None,
836 }
837 }
838
839 fn failed(coord: Coord, kind: FailureKind, failure: Arc<dyn Error + Send + Sync>) -> Self {
840 Self {
841 outcome: Outcome::Failed,
842 coord,
843 stop_code: None,
844 failure: Some(failure),
845 failure_kind: Some(kind),
846 }
847 }
848
849 /// Returns the authoritative portable outcome.
850 #[must_use]
851 pub const fn outcome(&self) -> Outcome {
852 self.outcome
853 }
854
855 /// Returns the Run coordinate.
856 #[must_use]
857 pub const fn coord(&self) -> &Coord {
858 &self.coord
859 }
860
861 /// Returns an explicit hook stop code when one ended the Run.
862 #[must_use]
863 pub const fn stop_code(&self) -> Option<&StopCode> {
864 self.stop_code.as_ref()
865 }
866
867 /// Returns execution failure detail for a failed Run.
868 ///
869 /// `Some` exactly when [`outcome`](Self::outcome) is
870 /// [`Failed`](Outcome::Failed). Its [`Display`](fmt::Display) is already the
871 /// complete rendered reason, stage and payload together — reef renders each
872 /// failure where it is latched, because the crate-private stage error cannot
873 /// outlive the Run and a classification alone does not diagnose anything.
874 /// [`source`](Error::source) is therefore for *inspecting* the failure rather
875 /// than for re-rendering it: it reaches an underlying model-client, hook,
876 /// transcript, or other error that caused the Run to fail. A nonfatal
877 /// [`tool::ProviderError`](super::tool::ProviderError) does not appear here;
878 /// obtain it from the outward event's
879 /// [`as_error`](super::event::Event::as_error) accessor. A chain walk that
880 /// concatenates every link will see the tail of the first message repeated,
881 /// which is why this is worth stating.
882 #[must_use]
883 pub fn failure(&self) -> Option<&(dyn Error + Send + Sync + 'static)> {
884 self.failure.as_deref()
885 }
886
887 /// Returns the portable classification of a failed Run.
888 #[must_use]
889 pub const fn failure_kind(&self) -> Option<FailureKind> {
890 self.failure_kind
891 }
892}
893
894/// The unique event consumer was already taken or abandoned.
895#[derive(Clone, Copy, Debug, Eq, PartialEq)]
896pub struct EventsTakenError;
897
898impl fmt::Display for EventsTakenError {
899 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
900 formatter.write_str("Run events were already consumed")
901 }
902}
903
904impl Error for EventsTakenError {}
905
906/// A spawned Run task was dropped by its executor after entering its task body.
907///
908/// Unlike [`TaskPanickedError`], no panic escaped from a poll of the Run task body.
909/// Runtime shutdown is one way this can happen, including shutdown caused by an
910/// unrelated panic in a thread hosting the runtime. The Run's terminal is failed,
911/// and its Session lease and Agent attachment are released synchronously.
912#[derive(Clone, Copy, Debug, Eq, PartialEq)]
913pub struct TaskAbortedError;
914
915impl fmt::Display for TaskAbortedError {
916 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
917 formatter.write_str("Run task was aborted after it started")
918 }
919}
920
921impl Error for TaskAbortedError {}
922
923/// A spawned Run task was dropped by its executor before entering its task body.
924///
925/// This can happen when an async runtime shuts down after accepting the spawn but
926/// before polling the task. The Run has been admitted, so its terminal is failed
927/// and its Session lease and Agent attachment are released synchronously.
928#[derive(Clone, Copy, Debug, Eq, PartialEq)]
929pub struct TaskNotStartedError;
930
931impl fmt::Display for TaskNotStartedError {
932 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
933 formatter.write_str("Run task was dropped before it started")
934 }
935}
936
937impl Error for TaskNotStartedError {}
938
939/// A Run task panicked after entering its task body.
940///
941/// Its synchronous cleanup publishes this error because the panic bypassed the
942/// driver's normal terminal classification. Reef resumes the original panic to
943/// the executor after recording this terminal classification.
944#[derive(Clone, Copy, Debug, Eq, PartialEq)]
945pub struct TaskPanickedError;
946
947impl fmt::Display for TaskPanickedError {
948 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
949 formatter.write_str("Run task panicked")
950 }
951}
952
953impl Error for TaskPanickedError {}
954
955#[derive(Debug)]
956struct CanceledError;
957
958impl fmt::Display for CanceledError {
959 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
960 formatter.write_str("Run canceled")
961 }
962}
963
964impl Error for CanceledError {}
965
966/// The portable classification of a failed Run.
967///
968/// A Run can fail for reasons that call for very different handling — a
969/// provider outage, a malformed provider stream, a hook defect, a transcript
970/// conflict — and the concrete failure types belong to crate-private stages that
971/// a consumer could not downcast to even if they were exposed. This enum is the
972/// stable, matchable summary; [`Terminal::failure`] carries the exact text.
973#[derive(Clone, Copy, Debug, Eq, PartialEq)]
974#[non_exhaustive]
975pub enum FailureKind {
976 /// A transcript snapshot or mutation failed.
977 Transcript,
978 /// Resolving the Step's tools failed, usually a name collision.
979 ToolResolution,
980 /// The transcript could not be frozen into a valid model request.
981 Request,
982 /// The provider adapter itself failed.
983 Model,
984 /// The provider's normalized event stream violated the grammar.
985 Stream,
986 /// An outward event could not be constructed or delivered.
987 Event,
988 /// A hook callback failed.
989 Hook,
990 /// Routing or executing a tool batch failed.
991 ToolExecution,
992 /// Committing a tool batch failed.
993 ToolCommit,
994 /// A claimed input batch produced an invalid admission decision.
995 Input,
996 /// A runtime invariant produced a failure with no external cause.
997 ///
998 /// This is a defect in Reef, not in the caller or the provider.
999 Internal,
1000}
1001
1002/// The retained reason a Run driver failed.
1003///
1004/// The message is rendered from the concrete stage failure at the moment it is
1005/// latched, because the crate-private staged error itself is not `Clone` and
1006/// cannot outlive the driver. Rendering eagerly keeps the whole reason — stage,
1007/// payload, and any hook attribution — instead of the classification alone.
1008///
1009/// `cause` keeps the same failure as a *value*. Reading a message is enough to
1010/// diagnose a Run; deciding what to do about one is not, so the concrete stage
1011/// failure is retained and returned from [`Error::source`]. A provider error is
1012/// the consumer's own type, so `source()` is downcastable back to it.
1013#[derive(Debug)]
1014struct DriverError {
1015 kind: FailureKind,
1016 message: String,
1017 cause: Option<Arc<dyn Error + Send + Sync>>,
1018}
1019
1020impl DriverError {
1021 fn from_loop(error: &driver::Error) -> Self {
1022 Self {
1023 kind: error.classify(),
1024 message: error.to_string(),
1025 cause: error.cause(),
1026 }
1027 }
1028}
1029
1030impl fmt::Display for DriverError {
1031 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
1032 formatter.write_str(&self.message)
1033 }
1034}
1035
1036impl Error for DriverError {
1037 fn source(&self) -> Option<&(dyn Error + 'static)> {
1038 self.cause
1039 .as_ref()
1040 .map(|cause| cause.as_ref() as &(dyn Error + 'static))
1041 }
1042}