use std::any::Any;
use std::error;
use std::fmt;
use std::future::{Future, poll_fn};
use std::num::NonZeroU64;
use std::panic::{AssertUnwindSafe, catch_unwind, panic_any, resume_unwind};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex, mpsc};
use crate::cancel::Token;
use futures_core::Stream;
use super::event::{
Event, EventSource, InputOrigin, Kind, Producer, StopCode, StrongDecision, TerminalIntent,
ToolEmitterGate, ValidationError, tool_emitter,
};
use super::hook::{
self, AfterModelDecision, AfterStepDecision, BeforeModelDecision, BufferedFact,
CallbackResources, ContinuationDecision, DispatchError, Emitter, FactBuffer,
InputAdmitDecision, LifecycleEnd, LifecycleKind, LifecycleObservation, OwnedCallbackResources,
OwnedToolRegistrationAdapter, StagedTranscript, StepSnapshot, StopDecision,
ToolCallEndObservation, ToolCallProgress, TranscriptSource,
};
use super::model::{
self, Client, CompletionReason, ContentKind, Info, Request, RequestError, RequestPlan,
StreamError, StreamGrammar,
};
use super::run::FailureKind;
use super::session::{Claim, InputBatch, InputMailbox, InputTarget, Lease};
use super::tool::{self, CallDisposition, ExecutionError, ProviderError, StepResolutionError};
use super::tool_batch_commit::{ToolBatchCommit, ToolBatchCommitError, commit_tool_batch};
use super::tool_batch_prepare::{ToolBatchPreparePlan, prepare_tool_calls};
use super::tool_batch_route::{
RoutedToolBatch, ToolBatchRouteError, ToolRouteBinder, ToolRouteBinding, route_and_execute,
};
use crate::audit;
use crate::state::session_state::CallbackWritePermit;
use crate::transcript::controller::{ControllerError, TranscriptController};
use crate::transcript::item::{Assistant, Item, ModelVisibility, User};
use crate::transcript::store::{self, Snapshot, Store, Version};
use crate::value::content::{Part, Segment};
use crate::value::lifecycle::{Coord, CoordError, Outcome, StepIndex};
use crate::value::tool_call::{ToolCall, ToolResult};
#[cfg_attr(
not(test),
expect(dead_code, reason = "both payloads are read only by the crate's tests")
)]
pub(crate) enum Advance {
Input(InputBatch),
Generation(Generation),
}
pub(crate) enum InputAdmitOutcome {
Admitted(InputBatch),
Rejected(String),
}
pub(crate) struct Generation {
snapshot: Arc<StepSnapshot>,
assistant: Assistant,
finish_reason: String,
completion: CompletionReason,
}
impl Generation {
pub(crate) const fn snapshot(&self) -> &Arc<StepSnapshot> {
&self.snapshot
}
pub(crate) const fn assistant(&self) -> &Assistant {
&self.assistant
}
pub(crate) fn finish_reason(&self) -> &str {
&self.finish_reason
}
pub(crate) const fn completion(&self) -> CompletionReason {
self.completion
}
pub(crate) fn has_tool_calls(&self) -> bool {
self.assistant
.content()
.iter()
.any(|segment| matches!(segment, Segment::ToolCall { .. }))
}
fn tool_calls(&self) -> Vec<ToolCall> {
self.assistant
.content()
.iter()
.filter_map(|segment| match segment {
Segment::ToolCall { call, .. } => Some(call.clone()),
Segment::Text { .. } | Segment::Thinking { .. } => None,
})
.collect()
}
}
#[derive(Clone, Copy)]
pub(crate) struct Outward<'a> {
producer: &'a Arc<Producer>,
facts: &'a FactBuffer,
}
impl<'a> Outward<'a> {
pub(crate) const fn new(producer: &'a Arc<Producer>, facts: &'a FactBuffer) -> Self {
Self { producer, facts }
}
}
pub(super) fn lifecycle_transcript_source<'a>(
kind: LifecycleKind,
snapshot: Option<&'a Arc<StepSnapshot>>,
live: &'a store::SnapshotReader,
) -> TranscriptSource<'a> {
match kind {
LifecycleKind::StepStart
| LifecycleKind::GenerationStart
| LifecycleKind::ToolCallStart => TranscriptSource::Snapshot(
snapshot
.expect("a Step-bound start observation carries its snapshot")
.tools()
.transcript(),
),
LifecycleKind::RunStart
| LifecycleKind::RunEnd
| LifecycleKind::TurnStart
| LifecycleKind::TurnEnd
| LifecycleKind::StepEnd
| LifecycleKind::GenerationEnd
| LifecycleKind::ToolCallEnd => TranscriptSource::Reader(live),
}
}
pub(crate) struct ToolPhase<'a> {
outward: Outward<'a>,
hooks: &'a hook::RuntimeRegistry,
callback_resources: &'a [CallbackResources<'a>],
controller: &'a TranscriptController<'a>,
session_coord_id: String,
run_id: String,
session_id: Option<String>,
open: Mutex<OpenBoundaries>,
#[cfg(test)]
test_emitter_gates: Vec<ToolEmitterGate>,
#[cfg(test)]
test_callback_abort_order: Option<Arc<Mutex<Vec<CallbackAbortStep>>>>,
#[cfg(test)]
test_callback_ack_probe: Option<tokio::sync::watch::Sender<usize>>,
}
#[derive(Debug, Default)]
struct OpenBoundaries {
turn: Option<OpenTurn>,
step: Option<OpenScope>,
generation: Option<OpenScope>,
calls: Vec<OpenToolCall>,
}
#[derive(Debug)]
struct OpenTurn {
coord: Coord,
steps: usize,
}
#[derive(Debug)]
struct OpenScope {
coord: Coord,
snapshot: Arc<StepSnapshot>,
}
#[derive(Debug)]
struct OpenToolCall {
coord: Coord,
snapshot: Arc<StepSnapshot>,
original: ToolCall,
effective: ToolCall,
progress: CallProgress,
}
impl OpenToolCall {
fn announced(coord: Coord, snapshot: Arc<StepSnapshot>, call: ToolCall) -> Self {
Self {
coord,
snapshot,
original: call.clone(),
effective: call,
progress: CallProgress::settle(
CallDisposition::NotExecuted,
ToolResult::no_output(),
ResultStage::Routed,
),
}
}
fn end(self, interrupted: &LifecycleEnd) -> ToolCallEndObservation {
let (end, milestone) = self.progress.terminal(interrupted);
ToolCallEndObservation::try_new(self.coord, end, self.original, self.effective, milestone)
.expect("the loop reports each call at a milestone its disposition reaches")
}
}
#[derive(Clone, Copy, Debug)]
pub(super) enum ResultStage {
Routed,
Committed,
Closed,
}
#[derive(Debug)]
pub(super) enum CallProgress {
NoResult(CallDisposition),
Routed(CallDisposition, ToolResult),
Committed(CallDisposition, ToolResult),
Closed(CallDisposition, ToolResult),
}
impl CallProgress {
pub(super) fn settle(
disposition: CallDisposition,
result: ToolResult,
stage: ResultStage,
) -> Self {
match disposition {
CallDisposition::Truncated | CallDisposition::NotExecuted => {
Self::NoResult(disposition)
}
CallDisposition::Executed
| CallDisposition::ShortCircuited
| CallDisposition::Blocked
| CallDisposition::Rejected => match stage {
ResultStage::Routed => Self::Routed(disposition, result),
ResultStage::Committed => Self::Committed(disposition, result),
ResultStage::Closed => Self::Closed(disposition, result),
},
}
}
pub(super) fn terminal(self, interrupted: &LifecycleEnd) -> (LifecycleEnd, ToolCallProgress) {
match self {
Self::NoResult(disposition) => {
(interrupted.clone(), ToolCallProgress::observed(disposition))
}
Self::Routed(disposition, result) => (
interrupted.clone(),
ToolCallProgress::result_available(disposition, result),
),
Self::Committed(disposition, result) => (
interrupted.clone(),
ToolCallProgress::result_committed(disposition, result),
),
Self::Closed(disposition, result) => (
LifecycleEnd::Completed,
ToolCallProgress::completed_post_hooks(disposition, result),
),
}
}
}
enum AfterToolFact {
Audit(audit::Event),
Error(Arc<dyn error::Error + Send + Sync>),
}
impl AfterToolFact {
fn into_event(self) -> Result<Event, ValidationError> {
match self {
Self::Audit(event) => Event::from_audit(event),
Self::Error(error) => Ok(Event::from_error(error)),
}
}
}
#[derive(Clone, Copy, Eq, PartialEq)]
enum CallbackAck {
Accepted,
Abort,
}
#[cfg(test)]
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(super) enum CallbackAbortStep {
Revoked,
AbortAckSent,
}
#[cfg(test)]
macro_rules! record_callback_abort_step {
($owner:expr, $step:expr) => {
if let Some(order) = &$owner.callback_abort_order {
order
.lock()
.expect("callback abort order lock remains available")
.push($step);
}
};
}
#[cfg(not(test))]
macro_rules! record_callback_abort_step {
($owner:expr, $step:expr) => {};
}
struct AfterToolRequest {
fact: AfterToolFact,
ack: mpsc::Sender<CallbackAck>,
}
struct CallbackAbort;
#[cfg_attr(
not(test),
expect(dead_code, reason = "both fields are read only by the crate's tests")
)]
pub(super) struct AfterToolHookPanic {
hook: String,
payload: Box<dyn Any + Send>,
}
#[cfg(test)]
impl AfterToolHookPanic {
pub(super) fn hook(&self) -> &str {
&self.hook
}
pub(super) fn payload_is<T: Any>(&self) -> bool {
self.payload.is::<T>()
}
}
struct AfterToolCallbackOutput {
staged: Option<(Version, Snapshot)>,
returned: Vec<crate::transcript::item::Item>,
}
enum AfterToolCallbackExit {
Completed(Result<AfterToolCallbackOutput, store::Error>),
Panicked(Box<dyn Any + Send>),
}
struct ManagedCallbackOwner {
requests: Option<tokio::sync::mpsc::Receiver<AfterToolRequest>>,
pending_ack: Option<mpsc::Sender<CallbackAck>>,
callback_write_permit: Option<CallbackWritePermit>,
handle: Option<tokio::task::JoinHandle<AfterToolCallbackExit>>,
completed: Option<mpsc::Receiver<()>>,
#[cfg(test)]
callback_abort_order: Option<Arc<Mutex<Vec<CallbackAbortStep>>>>,
#[cfg(test)]
callback_ack_probe: Option<tokio::sync::watch::Sender<usize>>,
#[cfg(test)]
join_drop_handle_probe: Option<Arc<AtomicUsize>>,
}
impl ManagedCallbackOwner {
async fn recv(&mut self) -> Option<AfterToolRequest> {
self.requests
.as_mut()
.expect("callback request lane remains owned before join")
.recv()
.await
}
fn arm_ack(&mut self, ack: &mpsc::Sender<CallbackAck>) {
self.pending_ack = Some(ack.clone());
}
fn acknowledge_accept(&mut self, ack: mpsc::Sender<CallbackAck>) {
let _callback_is_waiting = ack.send(CallbackAck::Accepted);
self.pending_ack = None;
#[cfg(test)]
if let Some(probe) = &self.callback_ack_probe {
probe.send_modify(|count| *count += 1);
}
}
fn revoke_callback_writes_and_abort_pending_ack(&mut self) {
if let Some(permit) = self.callback_write_permit.take() {
permit.revoke();
record_callback_abort_step!(self, CallbackAbortStep::Revoked);
}
if let Some(ack) = self.pending_ack.take() {
let _callback_is_waiting = ack.send(CallbackAck::Abort);
record_callback_abort_step!(self, CallbackAbortStep::AbortAckSent);
}
}
async fn join(mut self) -> AfterToolCallbackExit {
self.requests.take();
self.pending_ack = None;
let exit = self
.handle
.as_mut()
.expect("managed callback handle is joined once")
.await
.expect("managed after-tool callback owner joins");
self.handle.take();
let _completed = self.completed.take();
exit
}
}
impl Drop for ManagedCallbackOwner {
fn drop(&mut self) {
self.revoke_callback_writes_and_abort_pending_ack();
self.requests.take();
#[cfg(test)]
if let Some(probe) = &self.join_drop_handle_probe {
probe.store(usize::from(self.handle.is_some()), Ordering::SeqCst);
}
if self.handle.is_some() {
self.completed
.take()
.expect("live callback owner retains completion receiver")
.recv()
.expect_err("managed blocking callback closes completion lane");
self.handle.take();
}
}
}
#[cfg(test)]
pub(super) async fn managed_callback_join_retains_drop_authority() -> bool {
let (_requests, request_receiver) = tokio::sync::mpsc::channel(1);
let (completed, completion) = mpsc::channel();
let (entered, entered_receiver) = mpsc::channel();
let release = Arc::new(std::sync::Barrier::new(2));
let worker_release = Arc::clone(&release);
let handle = tokio::task::spawn_blocking(move || {
let _completion = completed;
entered.send(()).expect("join test remains live");
worker_release.wait();
AfterToolCallbackExit::Completed(Ok(AfterToolCallbackOutput {
staged: None,
returned: Vec::new(),
}))
});
entered_receiver
.recv()
.expect("managed callback join test worker starts");
let observed = Arc::new(AtomicUsize::new(0));
let owner = ManagedCallbackOwner {
requests: Some(request_receiver),
pending_ack: None,
callback_write_permit: None,
handle: Some(handle),
completed: Some(completion),
callback_abort_order: None,
callback_ack_probe: None,
join_drop_handle_probe: Some(Arc::clone(&observed)),
};
let mut joining = Box::pin(owner.join());
poll_fn(|context| {
assert!(
joining.as_mut().poll(context).is_pending(),
"blocked callback join completed early"
);
std::task::Poll::Ready(())
})
.await;
release.wait();
drop(joining);
observed.load(Ordering::SeqCst) == 1
}
struct RendezvousHookEmitter {
requests: tokio::sync::mpsc::Sender<AfterToolRequest>,
}
impl RendezvousHookEmitter {
fn emit(&self, fact: AfterToolFact) {
let (ack, accepted) = mpsc::channel();
let _request_sent = self.requests.blocking_send(AfterToolRequest { fact, ack });
if accepted.recv() != Ok(CallbackAck::Accepted) {
panic_any(CallbackAbort);
}
}
}
impl Emitter for RendezvousHookEmitter {
fn emit_audit(&self, event: audit::Event) {
self.emit(AfterToolFact::Audit(event));
}
fn emit_error(&self, error: Arc<dyn error::Error + Send + Sync>) {
self.emit(AfterToolFact::Error(error));
}
}
fn propagate_after_tool_panic(hook: String, payload: Box<dyn Any + Send>) -> ! {
resume_unwind(Box::new(AfterToolHookPanic { hook, payload }));
}
impl<'a> ToolPhase<'a> {
pub(crate) fn new(
outward: Outward<'a>,
hooks: &'a hook::RuntimeRegistry,
callback_resources: &'a [CallbackResources<'a>],
controller: &'a TranscriptController<'a>,
coord_identity: &Coord,
session_id: Option<String>,
) -> Self {
Self {
outward,
hooks,
callback_resources,
controller,
session_coord_id: coord_identity.session_id().to_owned(),
run_id: coord_identity.run_id().to_owned(),
session_id,
open: Mutex::new(OpenBoundaries::default()),
#[cfg(test)]
test_emitter_gates: Vec::new(),
#[cfg(test)]
test_callback_abort_order: None,
#[cfg(test)]
test_callback_ack_probe: None,
}
}
#[cfg(test)]
pub(super) fn with_test_emitter_gate(mut self, gate: ToolEmitterGate) -> Self {
self.test_emitter_gates.push(gate);
self
}
#[cfg(test)]
pub(super) fn with_test_callback_abort_order(
mut self,
order: Arc<Mutex<Vec<CallbackAbortStep>>>,
) -> Self {
self.test_callback_abort_order = Some(order);
self
}
#[cfg(test)]
pub(super) fn with_test_callback_ack_probe(
mut self,
probe: tokio::sync::watch::Sender<usize>,
) -> Self {
self.test_callback_ack_probe = Some(probe);
self
}
}
pub(crate) enum Error {
Cancelled,
Transcript(store::Error),
Tools(StepResolutionError),
Request(RequestError),
Model(Arc<dyn error::Error + Send + Sync>),
Stream(StreamError),
MissingToolPhase,
Coordinate(CoordError),
Event(ValidationError),
ToolCallRejected,
Prepare(DispatchError),
Route(ToolBatchRouteError),
Commit(ToolBatchCommitError),
ToolResultRejected,
AfterToolExecuteRejected,
AfterToolExecuteTranscript(ControllerError),
Input(InputAdmitError),
InputAdmitTranscript(ControllerError),
GenerationTranscript(ControllerError),
Hook(DispatchError),
BoundaryTranscript(ControllerError),
ModelAborted(String),
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(crate) enum InputAdmitError {
EmptyModify,
EmptyRejectReason,
}
impl fmt::Display for InputAdmitError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str(describe_input(self))
}
}
impl error::Error for InputAdmitError {}
impl Error {
pub(crate) const fn classify(&self) -> FailureKind {
match self {
Self::Transcript(_)
| Self::AfterToolExecuteTranscript(_)
| Self::InputAdmitTranscript(_)
| Self::GenerationTranscript(_)
| Self::BoundaryTranscript(_) => FailureKind::Transcript,
Self::Tools(_) => FailureKind::ToolResolution,
Self::Request(_) => FailureKind::Request,
Self::Model(_) => FailureKind::Model,
Self::Stream(_) => FailureKind::Stream,
Self::Event(_) => FailureKind::Event,
Self::Prepare(_) | Self::Hook(_) | Self::ModelAborted(_) => FailureKind::Hook,
Self::Route(_) => FailureKind::ToolExecution,
Self::Commit(_) => FailureKind::ToolCommit,
Self::Input(_) => FailureKind::Input,
Self::Cancelled
| Self::MissingToolPhase
| Self::Coordinate(_)
| Self::ToolCallRejected
| Self::ToolResultRejected
| Self::AfterToolExecuteRejected => FailureKind::Internal,
}
}
pub(crate) fn cause(&self) -> Option<Arc<dyn error::Error + Send + Sync>> {
match self {
Self::Transcript(error) => Some(Arc::new(*error)),
Self::Tools(error) => Some(Arc::new(error.clone())),
Self::Request(error) => Some(Arc::new(error.clone())),
Self::Model(error) => Some(Arc::clone(error)),
Self::Stream(error) => Some(Arc::new(error.clone())),
Self::Coordinate(error) => Some(Arc::new(*error)),
Self::Event(error) => Some(Arc::new(*error)),
Self::Prepare(error) => Some(Arc::new(error.clone())),
Self::Route(error) => Some(Arc::new(error.execution().clone())),
Self::Commit(ToolBatchCommitError::Rewrite { error, .. }) => {
Some(Arc::new(error.clone()))
}
Self::Commit(ToolBatchCommitError::Controller(error))
| Self::AfterToolExecuteTranscript(error)
| Self::InputAdmitTranscript(error)
| Self::GenerationTranscript(error) => controller_cause(error),
Self::Input(error) => Some(Arc::new(*error)),
Self::Hook(error) => Some(Arc::new(error.clone())),
Self::BoundaryTranscript(error) => controller_cause(error),
Self::Cancelled
| Self::MissingToolPhase
| Self::ToolCallRejected
| Self::ToolResultRejected
| Self::AfterToolExecuteRejected
| Self::ModelAborted(_)
| Self::Commit(ToolBatchCommitError::Cancelled) => None,
}
}
const fn is_rejection(&self) -> bool {
matches!(
self,
Self::ToolCallRejected | Self::ToolResultRejected | Self::AfterToolExecuteRejected
)
}
fn is_cancellation(&self) -> bool {
match self {
Self::Cancelled | Self::Prepare(DispatchError::Cancelled) => true,
Self::Route(error) => error.execution().execution() == &ExecutionError::Cancelled,
Self::Commit(ToolBatchCommitError::Cancelled)
| Self::Commit(ToolBatchCommitError::Rewrite {
error: DispatchError::Cancelled,
..
})
| Self::Commit(ToolBatchCommitError::Controller(ControllerError::Dispatch(
DispatchError::Cancelled,
)))
| Self::AfterToolExecuteTranscript(ControllerError::Dispatch(
DispatchError::Cancelled,
))
| Self::InputAdmitTranscript(ControllerError::Dispatch(DispatchError::Cancelled)) => {
true
}
Self::Transcript(_)
| Self::Tools(_)
| Self::Request(_)
| Self::Model(_)
| Self::Stream(_)
| Self::MissingToolPhase
| Self::Coordinate(_)
| Self::Event(_)
| Self::ToolCallRejected
| Self::Prepare(DispatchError::Callback(_))
| Self::Commit(_)
| Self::ToolResultRejected
| Self::AfterToolExecuteRejected
| Self::AfterToolExecuteTranscript(_)
| Self::Input(_)
| Self::InputAdmitTranscript(_)
| Self::GenerationTranscript(_)
| Self::Hook(_)
| Self::BoundaryTranscript(_)
| Self::ModelAborted(_) => false,
}
}
}
fn boundary_hook(error: DispatchError) -> Error {
match error {
DispatchError::Cancelled => Error::Cancelled,
other => Error::Hook(other),
}
}
fn boundary_transcript(error: ControllerError) -> Error {
match error {
ControllerError::Dispatch(DispatchError::Cancelled) => Error::Cancelled,
other => Error::BoundaryTranscript(other),
}
}
fn describe_route(error: &ToolBatchRouteError) -> String {
format!(
"executing the batch failed: {}",
error.execution().execution()
)
}
fn describe_commit(error: &ToolBatchCommitError) -> String {
match error {
ToolBatchCommitError::Cancelled => "cancelled before any rewrite ran".to_owned(),
ToolBatchCommitError::Rewrite {
source_index,
error,
} => format!("the rewrite at source index {source_index} failed: {error}"),
ToolBatchCommitError::Controller(error) => {
format!("the atomic append failed: {error}")
}
}
}
fn controller_cause(error: &ControllerError) -> Option<Arc<dyn error::Error + Send + Sync>> {
match error {
ControllerError::Dispatch(error) => Some(Arc::new(error.clone())),
ControllerError::Lease(error) => Some(Arc::new(error.clone())),
ControllerError::InvalidTransition { .. } => None,
}
}
const fn describe_input(error: &InputAdmitError) -> &'static str {
match error {
InputAdmitError::EmptyModify => "a registration rewrote the batch to nothing",
InputAdmitError::EmptyRejectReason => "a registration rejected without a reason",
}
}
impl fmt::Display for Error {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Cancelled => formatter.write_str("the run was cancelled"),
Self::Transcript(error) => write!(formatter, "the transcript failed: {error}"),
Self::Tools(error) => write!(formatter, "step tool resolution failed: {error}"),
Self::Request(error) => write!(formatter, "the model request was rejected: {error}"),
Self::Model(error) => write!(formatter, "the model provider failed: {error}"),
Self::Stream(error) => write!(formatter, "the model stream was invalid: {error}"),
Self::MissingToolPhase => {
formatter.write_str("a generation produced tool calls with no tool phase attached")
}
Self::Coordinate(error) => {
write!(formatter, "a tool-call coordinate was invalid: {error}")
}
Self::Event(error) => write!(formatter, "an outward event was invalid: {error}"),
Self::ToolCallRejected => {
formatter.write_str("the consumer rejected a tool call delivery")
}
Self::Prepare(error) => write!(formatter, "tool-call preparation failed: {error}"),
Self::Route(error) => write!(
formatter,
"tool batch routing failed: {}",
describe_route(error)
),
Self::Commit(error) => write!(
formatter,
"the tool batch commit failed: {}",
describe_commit(error)
),
Self::ToolResultRejected => {
formatter.write_str("the consumer rejected a tool result delivery")
}
Self::AfterToolExecuteRejected => {
formatter.write_str("the consumer rejected an after-tool-execute delivery")
}
Self::AfterToolExecuteTranscript(error) => write!(
formatter,
"an after-tool-execute transcript mutation failed: {error}"
),
Self::Input(error) => {
write!(
formatter,
"an input admission decision was invalid: {error}"
)
}
Self::InputAdmitTranscript(error) => {
write!(formatter, "admitting an input batch failed: {error}")
}
Self::GenerationTranscript(error) => {
write!(formatter, "committing the generation failed: {error}")
}
Self::Hook(error) => write!(formatter, "a boundary callback failed: {error}"),
Self::BoundaryTranscript(error) => {
write!(formatter, "committing a boundary batch failed: {error}")
}
Self::ModelAborted(reason) => {
write!(formatter, "a hook aborted the model call: {reason}")
}
}
}
}
pub(crate) enum Terminal {
Completed,
Stopped,
Cancelled,
Failed(Error),
}
impl Terminal {
#[cfg(test)]
pub(crate) const fn outcome(&self) -> crate::value::lifecycle::Outcome {
match self {
Self::Completed => crate::value::lifecycle::Outcome::Completed,
Self::Stopped => crate::value::lifecycle::Outcome::Stopped,
Self::Cancelled => crate::value::lifecycle::Outcome::Cancelled,
Self::Failed(_) => crate::value::lifecycle::Outcome::Failed,
}
}
#[cfg(test)]
pub(crate) const fn failure(&self) -> Option<&Error> {
match self {
Self::Failed(error) => Some(error),
Self::Completed | Self::Stopped | Self::Cancelled => None,
}
}
const fn priority(&self) -> u8 {
match self {
Self::Completed => 0,
Self::Cancelled | Self::Stopped => 1,
Self::Failed(_) => 2,
}
}
}
fn boundary_end(terminal: &Terminal, stop_code: Option<StopCode>) -> LifecycleEnd {
match terminal {
Terminal::Completed => LifecycleEnd::Completed,
Terminal::Stopped => LifecycleEnd::Stopped { code: stop_code },
Terminal::Cancelled => LifecycleEnd::Cancelled {
error: Arc::new(BoundaryInterruption::canceled()),
},
Terminal::Failed(error) => LifecycleEnd::Failed {
error: Arc::new(BoundaryInterruption::failed(error)),
},
}
}
struct BoundaryInterruption {
message: String,
cause: Option<Arc<dyn error::Error + Send + Sync>>,
}
impl fmt::Debug for BoundaryInterruption {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("BoundaryInterruption")
.field("message", &self.message)
.field("cause", &self.cause.as_ref().map(|cause| cause.to_string()))
.finish()
}
}
impl BoundaryInterruption {
fn canceled() -> Self {
Self {
message: Error::Cancelled.to_string(),
cause: None,
}
}
fn failed(error: &Error) -> Self {
Self {
message: error.to_string(),
cause: error.cause(),
}
}
}
#[cfg(test)]
pub(super) fn boundary_interruption(error: &Error) -> Arc<dyn error::Error + Send + Sync> {
Arc::new(BoundaryInterruption::failed(error))
}
impl fmt::Display for BoundaryInterruption {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str(&self.message)
}
}
impl error::Error for BoundaryInterruption {
fn source(&self) -> Option<&(dyn error::Error + 'static)> {
self.cause
.as_ref()
.map(|cause| cause.as_ref() as &(dyn error::Error + 'static))
}
}
#[derive(Default)]
pub(super) struct TerminalLatch {
candidate: Option<Terminal>,
}
impl TerminalLatch {
pub(super) fn record(&mut self, candidate: Terminal) {
let replace = self
.candidate
.as_ref()
.is_none_or(|current| candidate.priority() > current.priority());
if replace {
self.candidate = Some(candidate);
}
}
fn observe_control(&mut self, source: &EventSource) {
match source.terminal_intent() {
TerminalIntent::Running => {}
TerminalIntent::Stopped => self.record(Terminal::Stopped),
TerminalIntent::Cancelled => self.record(Terminal::Cancelled),
TerminalIntent::Failed => {
if let Some(error) = source.latch_failure() {
self.record(Terminal::Failed(Error::Event(error)));
}
}
}
}
pub(super) fn record_error(&mut self, source: &EventSource, error: Error) {
if error.is_rejection() {
source.request_stop();
self.observe_control(source);
} else if error.is_cancellation() {
self.observe_control(source);
if self.candidate.is_none() {
self.record(Terminal::Cancelled);
}
} else {
let error = source.latch_failure().map_or(error, Error::Event);
self.record(Terminal::Failed(error));
}
}
const fn is_terminal(&self) -> bool {
self.candidate.is_some()
}
pub(super) fn finish(mut self, source: &EventSource) -> Terminal {
self.observe_control(source);
self.candidate.unwrap_or(Terminal::Completed)
}
}
struct Context<'a, M> {
model: &'a M,
lease: &'a Lease,
#[cfg(test)]
transcript: Option<crate::transcript::store::SnapshotReader>,
inbox: &'a InputMailbox,
tools: &'a tool::Registry,
cancellation: &'a Token,
}
impl<M> Context<'_, M> {
fn transcript_snapshot(&self) -> Result<crate::transcript::store::Snapshot, store::Error> {
#[cfg(test)]
if let Some(transcript) = &self.transcript {
return transcript.snapshot();
}
self.lease.transcript_snapshot()
}
}
#[derive(Clone, Copy)]
enum Phase {
Generation,
StepBoundary { generated_tool_calls: bool },
TurnBoundary,
Done,
}
struct GenerationPosition {
turn: u64,
step: u64,
step_in_run: u64,
generation: u64,
}
impl GenerationPosition {
const fn first() -> Self {
Self {
turn: 1,
step: 1,
step_in_run: 1,
generation: 1,
}
}
fn step_index(&self) -> StepIndex {
let turn = NonZeroU64::new(self.turn).expect("runtime turns are one-based");
let step = NonZeroU64::new(self.step).expect("runtime steps are one-based");
let step_in_run =
NonZeroU64::new(self.step_in_run).expect("runtime run-wide steps are one-based");
StepIndex::new(turn, step, step_in_run)
.expect("the driver maintains a valid step hierarchy")
}
fn turn_coord(&self, session_id: &str, run_id: &str) -> Coord {
let turn = NonZeroU64::new(self.turn).expect("runtime turns are one-based");
Coord::at_turn(session_id.to_owned(), run_id.to_owned(), turn)
.expect("the attached tool phase carries non-empty run identities")
}
fn step_coord(&self, session_id: &str, run_id: &str) -> Coord {
Coord::at_step(session_id.to_owned(), run_id.to_owned(), self.step_index())
.expect("the attached tool phase carries non-empty run identities")
}
fn coord(&self, session_id: &str, run_id: &str) -> Coord {
let generation =
NonZeroU64::new(self.generation).expect("runtime generations are one-based");
Coord::at_generation(
session_id.to_owned(),
run_id.to_owned(),
self.step_index(),
generation,
)
.expect("the attached tool phase carries non-empty run identities")
}
fn coords(&self, session_id: &str, run_id: &str) -> GenerationCoords {
GenerationCoords {
turn: self.turn_coord(session_id, run_id),
step: self.step_coord(session_id, run_id),
generation: self.coord(session_id, run_id),
}
}
const fn next_step(&mut self) {
self.step += 1;
self.step_in_run += 1;
self.generation += 1;
}
const fn next_turn(&mut self) {
self.turn += 1;
self.step = 1;
self.step_in_run += 1;
self.generation += 1;
}
}
struct GenerationCoords {
turn: Coord,
step: Coord,
generation: Coord,
}
#[derive(Default)]
pub(crate) struct Report {
turns: AtomicUsize,
stop_code: Mutex<Option<StopCode>>,
}
impl Report {
pub(crate) fn turns(&self) -> usize {
self.turns.load(Ordering::Relaxed)
}
pub(crate) fn stop_code(&self) -> Option<StopCode> {
self.stop_code
.lock()
.expect("driver report lock poisoned")
.clone()
}
}
pub(crate) struct Driver<'a, M> {
context: Context<'a, M>,
tool_phase: Option<ToolPhase<'a>>,
phase: Phase,
generation_position: GenerationPosition,
turn_open: bool,
turn_trigger: Vec<Item>,
step_snapshot: Option<Arc<StepSnapshot>>,
turns: usize,
continuations: usize,
report: Option<&'a Report>,
}
impl<'a, M: Client> Driver<'a, M> {
pub(crate) const fn new(
model: &'a M,
lease: &'a Lease,
inbox: &'a InputMailbox,
tools: &'a tool::Registry,
cancellation: &'a Token,
) -> Self {
Self {
context: Context {
model,
lease,
#[cfg(test)]
transcript: None,
inbox,
tools,
cancellation,
},
tool_phase: None,
phase: Phase::Generation,
generation_position: GenerationPosition::first(),
turn_open: false,
turn_trigger: Vec::new(),
step_snapshot: None,
turns: 0,
continuations: 0,
report: None,
}
}
pub(crate) fn with_tool_phase(mut self, tool_phase: ToolPhase<'a>) -> Self {
self.tool_phase = Some(tool_phase);
self
}
pub(crate) fn with_turn_trigger(mut self, trigger: Vec<Item>) -> Self {
self.turn_trigger = trigger;
self
}
pub(crate) const fn with_report(mut self, report: &'a Report) -> Self {
self.report = Some(report);
self
}
#[cfg(test)]
pub(super) fn with_transcript_reader(
model: &'a M,
lease: &'a Lease,
transcript: crate::transcript::store::SnapshotReader,
inbox: &'a InputMailbox,
tools: &'a tool::Registry,
cancellation: &'a Token,
) -> Self {
let mut driver = Self::new(model, lease, inbox, tools, cancellation);
driver.context.transcript = Some(transcript);
driver
}
pub(crate) async fn run(mut self, plan: RequestPlan, source: &EventSource) -> Terminal {
let mut terminal = TerminalLatch::default();
loop {
terminal.observe_control(source);
if terminal.is_terminal() {
return self.close_boundaries(terminal.finish(source));
}
enum Drive<T> {
Advanced(T),
Interrupted,
}
let driven = {
let advance = self.advance(plan.clone());
tokio::pin!(advance);
tokio::select! {
biased;
result = &mut advance => Drive::Advanced(result),
() = source.interruption() => Drive::Interrupted,
}
};
match driven {
Drive::Advanced(Ok(Some(_))) => {}
Drive::Advanced(Ok(None)) => {
return self.close_boundaries(terminal.finish(source));
}
Drive::Advanced(Err(error)) => {
terminal.record_error(source, error);
return self.close_boundaries(terminal.finish(source));
}
Drive::Interrupted => {
terminal.observe_control(source);
return self.close_boundaries(terminal.finish(source));
}
}
}
}
fn close_boundaries(&self, terminal: Terminal) -> Terminal {
if let Some(tool_phase) = &self.tool_phase {
let end = boundary_end(&terminal, self.report.and_then(Report::stop_code));
tool_phase.close_open_observations(&end);
}
terminal
}
pub(crate) async fn advance(&mut self, plan: RequestPlan) -> Result<Option<Advance>, Error> {
if self.context.cancellation.is_cancelled() {
return Err(Error::Cancelled);
}
loop {
match self.phase {
Phase::Generation => {
let coords = self.tool_phase.as_ref().map(|tool_phase| {
self.generation_position
.coords(&tool_phase.session_coord_id, &tool_phase.run_id)
});
self.open_turn(coords.as_ref()).await?;
self.emit_step_boundary(Kind::StepStart, None).await?;
let generation = {
let phase = self.tool_phase.as_ref();
Step {
context: &self.context,
phase,
coords: coords.as_ref(),
}
.advance(plan)
.await?
};
self.step_snapshot = Some(Arc::clone(generation.snapshot()));
let generated_tool_calls = generation.has_tool_calls();
if let Err(error) = self
.publish_generation(&generation, coords.as_ref(), generated_tool_calls)
.await
{
self.phase = Phase::Done;
return Err(error);
}
self.phase = Phase::StepBoundary {
generated_tool_calls,
};
return Ok(Some(Advance::Generation(generation)));
}
Phase::StepBoundary {
generated_tool_calls,
} => {
let end_turn = self.close_step().await?;
if !end_turn {
if let Some(mut claim) =
self.context.inbox.claim_conserved(InputTarget::NextStep)
{
let outcome = self.admit_input(&mut claim);
drop(claim);
match outcome? {
InputAdmitOutcome::Admitted(input) => {
self.publish_input_admitted(
InputOrigin::Mailbox,
input.items().to_vec(),
)
.await?;
self.generation_position.next_step();
self.phase = Phase::Generation;
return Ok(Some(Advance::Input(input)));
}
InputAdmitOutcome::Rejected(reason) => {
self.tool_phase
.as_ref()
.expect("only an attached tool phase refuses a batch")
.publish_input_rejected(reason)
.await?;
}
}
}
}
self.phase = if generated_tool_calls && !end_turn {
self.generation_position.next_step();
Phase::Generation
} else {
Phase::TurnBoundary
};
}
Phase::TurnBoundary => {
self.close_turn().await?;
if self.publish_requested_stop().await? {
self.phase = Phase::Done;
continue;
}
if let Some(mut claim) =
self.context.inbox.claim_conserved(InputTarget::NextTurn)
{
let outcome = self.admit_input(&mut claim);
drop(claim);
match outcome? {
InputAdmitOutcome::Admitted(input) => {
self.turn_trigger = input.items().to_vec();
self.publish_input_admitted(
InputOrigin::Mailbox,
self.turn_trigger.clone(),
)
.await?;
self.generation_position.next_turn();
self.phase = Phase::Generation;
return Ok(Some(Advance::Input(input)));
}
InputAdmitOutcome::Rejected(reason) => {
self.tool_phase
.as_ref()
.expect("only an attached tool phase refuses a batch")
.publish_input_rejected(reason)
.await?;
}
}
}
if let Some(continuation) = self.compose_continuation()? {
self.turn_trigger = continuation.items().to_vec();
self.publish_input_admitted(
InputOrigin::Continuation,
self.turn_trigger.clone(),
)
.await?;
self.generation_position.next_turn();
self.phase = Phase::Generation;
return Ok(Some(Advance::Input(continuation)));
} else {
self.phase = Phase::Done;
}
}
Phase::Done => return Ok(None),
}
}
}
async fn publish_generation(
&mut self,
generation: &Generation,
coords: Option<&GenerationCoords>,
generated_tool_calls: bool,
) -> Result<(), Error> {
let cancellation = self.context.cancellation;
let Some(tool_phase) = &mut self.tool_phase else {
return if generated_tool_calls {
Err(Error::MissingToolPhase)
} else {
Ok(())
};
};
let coord = &coords
.expect("an attached tool phase mints generation coordinates")
.generation;
tool_phase
.emit_generation_content(generation, coord)
.await?;
if generated_tool_calls {
tool_phase.process(generation, coord, cancellation).await?;
} else {
tool_phase.commit_assistant(generation, coord, cancellation)?;
}
tool_phase
.emit_lifecycle_event(Kind::GenerationEnd, Some(Outcome::Completed), coord.clone())
.await?;
tool_phase.observe_lifecycle(
coord,
Some(generation.snapshot()),
&LifecycleObservation::GenerationEnd {
coord: coord.clone(),
end: LifecycleEnd::Completed,
finish_reason: generation.finish_reason().to_owned(),
completion: generation.completion(),
},
);
tool_phase.closed_generation();
Ok(())
}
async fn open_turn(&mut self, coords: Option<&GenerationCoords>) -> Result<(), Error> {
if self.turn_open {
return Ok(());
}
if let Some(tool_phase) = &self.tool_phase {
let coord = &coords
.expect("an attached tool phase mints turn coordinates")
.turn;
let trigger = &self.turn_trigger;
tool_phase.staged_boundary(
coord,
self.context.cancellation,
tool_phase.has_turn_step_hooks(),
|staged| tool_phase.before_turn(staged, trigger, self.context.cancellation),
)?;
tool_phase
.emit_lifecycle_event(Kind::TurnStart, None, coord.clone())
.await?;
tool_phase.observe_lifecycle(
coord,
None,
&LifecycleObservation::TurnStart {
coord: coord.clone(),
trigger: trigger.clone(),
},
);
tool_phase.opened_turn(coord);
}
self.turn_open = true;
self.turns += 1;
if let Some(report) = self.report {
report.turns.store(self.turns, Ordering::Relaxed);
}
Ok(())
}
async fn close_turn(&mut self) -> Result<(), Error> {
let steps = self.generation_position.step;
if let Some(tool_phase) = &self.tool_phase {
let coord = self
.generation_position
.turn_coord(&tool_phase.session_coord_id, &tool_phase.run_id);
let trigger = &self.turn_trigger;
tool_phase.staged_boundary(
&coord,
self.context.cancellation,
tool_phase.has_turn_step_hooks(),
|staged| tool_phase.after_turn(staged, trigger, self.context.cancellation),
)?;
tool_phase
.emit_lifecycle_event(Kind::TurnEnd, Some(Outcome::Completed), coord.clone())
.await?;
tool_phase.observe_lifecycle(
&coord,
None,
&LifecycleObservation::TurnEnd {
coord: coord.clone(),
end: LifecycleEnd::Completed,
steps: steps as usize,
},
);
tool_phase.closed_turn();
}
self.turn_open = false;
Ok(())
}
async fn close_step(&mut self) -> Result<bool, Error> {
let snapshot = self
.step_snapshot
.take()
.expect("a step boundary always follows a frozen generation");
let mut end_turn = false;
if let Some(tool_phase) = &self.tool_phase {
let cancellation = self.context.cancellation;
let coord = self
.generation_position
.step_coord(&tool_phase.session_coord_id, &tool_phase.run_id);
tool_phase.staged_boundary(
&coord,
cancellation,
tool_phase.has_turn_step_hooks(),
|staged| {
let (ends_turn, items) =
tool_phase.after_step(staged, &snapshot, cancellation)?;
end_turn = ends_turn;
Ok(items)
},
)?;
tool_phase
.emit_lifecycle_event(Kind::StepEnd, Some(Outcome::Completed), coord.clone())
.await?;
tool_phase.observe_lifecycle(
&coord,
Some(&snapshot),
&LifecycleObservation::StepEnd {
coord: coord.clone(),
end: LifecycleEnd::Completed,
snapshot: Arc::clone(&snapshot),
},
);
tool_phase.closed_step();
}
Ok(end_turn)
}
async fn publish_requested_stop(&mut self) -> Result<bool, Error> {
let Some(tool_phase) = &self.tool_phase else {
return Ok(false);
};
let Some(code) = tool_phase.decide_stop(self.context.cancellation)? else {
return Ok(false);
};
let coord = tool_phase.run_coord();
tool_phase.drain_hook_facts(&coord).await?;
tool_phase
.outward
.producer
.emit_stop(code.clone(), coord)
.await
.map_err(Error::Event)?;
if let Some(report) = self.report {
*report
.stop_code
.lock()
.expect("driver report lock poisoned") = Some(code);
}
Ok(true)
}
fn compose_continuation(&mut self) -> Result<Option<InputBatch>, Error> {
let Some(tool_phase) = &self.tool_phase else {
return Ok(None);
};
let cancellation = self.context.cancellation;
let coord = tool_phase.run_coord();
let turns = self.turns;
let continuations = self.continuations;
let mut continuation = Vec::new();
tool_phase.staged_boundary(&coord, cancellation, tool_phase.has_run_hooks(), |staged| {
let parts =
tool_phase.compose_continuation(staged, continuations, turns, cancellation)?;
if parts.is_empty() {
return Ok(Vec::new());
}
continuation = vec![Item::User(User::new(parts, ModelVisibility::Visible))];
Ok(continuation.clone())
})?;
if continuation.is_empty() {
return Ok(None);
}
self.continuations += 1;
Ok(Some(
InputBatch::new(InputTarget::NextTurn, continuation)
.expect("a non-empty continuation is a valid batch"),
))
}
async fn emit_step_boundary(&self, kind: Kind, outcome: Option<Outcome>) -> Result<(), Error> {
if let Some(tool_phase) = &self.tool_phase {
let coord = self
.generation_position
.step_coord(&tool_phase.session_coord_id, &tool_phase.run_id);
tool_phase.entered_step(self.generation_position.step as usize);
tool_phase
.emit_lifecycle_event(kind, outcome, coord)
.await?;
}
Ok(())
}
async fn publish_input_admitted(
&self,
origin: InputOrigin,
items: Vec<Item>,
) -> Result<(), Error> {
if let Some(tool_phase) = &self.tool_phase {
let coord = tool_phase.run_coord();
tool_phase.drain_hook_facts(&coord).await?;
tool_phase
.outward
.producer
.emit_input_admitted(origin, items, coord)
.await
.map_err(Error::Event)?;
}
Ok(())
}
fn admit_input(&mut self, claim: &mut Claim<'_>) -> Result<InputAdmitOutcome, Error> {
let Some(tool_phase) = &mut self.tool_phase else {
return Ok(InputAdmitOutcome::Admitted(claim.settle()));
};
tool_phase.admit_input(claim, self.context.cancellation)
}
}
impl ToolPhase<'_> {
pub(crate) fn admit_input(
&self,
claim: &mut Claim<'_>,
cancellation: &Token,
) -> Result<InputAdmitOutcome, Error> {
let target = claim.batch().target();
let mut items = claim.batch().items().to_vec();
let transcript = self.controller.snapshot_reader();
for adapter in self.hooks.input_hooks() {
let registration_resources = self
.callback_resources
.get(adapter.index())
.copied()
.expect("input-admit resources cover every registry index");
let decision = adapter
.invoke_before_input_admit(
registration_resources,
&transcript,
target,
&items,
cancellation,
)
.map_err(|_| Error::Cancelled)?;
match decision {
InputAdmitDecision::Admit => {}
InputAdmitDecision::Modify(replacement) => {
if replacement.is_empty() {
return Err(Error::Input(InputAdmitError::EmptyModify));
}
items = replacement;
}
InputAdmitDecision::Reject(reason) => {
if reason.is_empty() {
return Err(Error::Input(InputAdmitError::EmptyRejectReason));
}
let _refused = claim.settle();
return Ok(InputAdmitOutcome::Rejected(reason));
}
}
}
self.controller
.append_batch_settling(items.clone(), cancellation, &mut || {
let _committed = claim.settle();
})
.map_err(|error| match error {
ControllerError::Dispatch(DispatchError::Cancelled) => Error::Cancelled,
other => Error::InputAdmitTranscript(other),
})?;
let batch = InputBatch::new(target, items)
.expect("admitted input stays non-empty after validated hooks");
Ok(InputAdmitOutcome::Admitted(batch))
}
pub(crate) fn commit_assistant(
&self,
generation: &Generation,
generation_coord: &Coord,
cancellation: &Token,
) -> Result<(), Error> {
let assistant = generation.assistant();
let records_nothing =
assistant.content().is_empty() && assistant.provider_data().is_none_or(<[_]>::is_empty);
let items = if records_nothing {
Vec::new()
} else {
vec![Item::Assistant(assistant.clone())]
};
self.controller
.with_coord(generation_coord)
.append_batch(items, cancellation)
.map_err(|error| match error {
ControllerError::Dispatch(DispatchError::Cancelled) => Error::Cancelled,
other => Error::GenerationTranscript(other),
})?;
Ok(())
}
pub(crate) async fn emit_generation_content(
&self,
generation: &Generation,
generation_coord: &Coord,
) -> Result<(), Error> {
self.drain_hook_facts(generation_coord).await?;
for segment in generation.assistant().content() {
let (kind, text, extensions) = match segment {
Segment::Text { text, extensions } => (Kind::Text, text, extensions),
Segment::Thinking { text, extensions } => (Kind::Thinking, text, extensions),
Segment::ToolCall { .. } => continue,
};
self.outward
.producer
.emit_text(
kind,
text.clone(),
generation_coord.clone(),
extensions.item_id().map(str::to_owned),
)
.await
.map_err(Error::Event)?;
}
Ok(())
}
fn resources<'scope>(
&'scope self,
index: usize,
coord: &'scope Coord,
) -> CallbackResources<'scope> {
self.callback_resources
.get(index)
.copied()
.expect("boundary resources cover every registry index")
.with_coord(coord)
}
fn run_coord(&self) -> Coord {
Coord::at_run(self.session_coord_id.clone(), self.run_id.clone())
.expect("the attached tool phase carries non-empty run identities")
}
fn observe_lifecycle(
&self,
coord: &Coord,
snapshot: Option<&Arc<StepSnapshot>>,
observation: &LifecycleObservation,
) {
let reader = self.controller.snapshot_reader();
let transcript = lifecycle_transcript_source(observation.kind(), snapshot, &reader);
for adapter in self.hooks.observer_hooks() {
let resources = self.resources(adapter.index(), coord);
adapter.invoke_lifecycle(resources, transcript, snapshot, observation);
}
}
fn opened_turn(&self, coord: &Coord) {
self.open().turn = Some(OpenTurn {
coord: coord.clone(),
steps: 0,
});
}
fn entered_step(&self, steps: usize) {
self.open()
.turn
.as_mut()
.expect("a Step is entered only inside a Turn this phase opened")
.steps = steps;
}
fn opened_step(&self, coord: &Coord, snapshot: &Arc<StepSnapshot>) {
self.open().step = Some(OpenScope {
coord: coord.clone(),
snapshot: Arc::clone(snapshot),
});
}
fn opened_generation(&self, coord: &Coord, snapshot: &Arc<StepSnapshot>) {
self.open().generation = Some(OpenScope {
coord: coord.clone(),
snapshot: Arc::clone(snapshot),
});
}
fn closed_turn(&self) {
self.open().turn.take();
}
fn closed_step(&self) {
self.open().step.take();
}
fn closed_generation(&self) {
self.open().generation.take();
}
fn open(&self) -> std::sync::MutexGuard<'_, OpenBoundaries> {
self.open.lock().expect("driver boundary lock poisoned")
}
fn close_open_observations(&self, end: &LifecycleEnd) {
let open = std::mem::take(&mut *self.open());
self.close_tool_calls(open.calls, end);
if let Some(generation) = open.generation {
self.observe_lifecycle(
&generation.coord,
Some(&generation.snapshot),
&LifecycleObservation::GenerationEnd {
coord: generation.coord.clone(),
end: end.clone(),
finish_reason: String::new(),
completion: CompletionReason::Unknown,
},
);
}
if let Some(step) = open.step {
self.observe_lifecycle(
&step.coord,
Some(&step.snapshot),
&LifecycleObservation::StepEnd {
coord: step.coord.clone(),
end: end.clone(),
snapshot: Arc::clone(&step.snapshot),
},
);
}
if let Some(turn) = open.turn {
self.observe_lifecycle(
&turn.coord,
None,
&LifecycleObservation::TurnEnd {
coord: turn.coord.clone(),
end: end.clone(),
steps: turn.steps,
},
);
}
}
fn close_tool_calls(&self, calls: Vec<OpenToolCall>, end: &LifecycleEnd) {
for call in calls {
let coord = call.coord.clone();
let snapshot = Arc::clone(&call.snapshot);
let observation = LifecycleObservation::ToolCallEnd(call.end(end));
self.observe_lifecycle(&coord, Some(&snapshot), &observation);
}
}
fn observe_model_event(
&self,
coord: &Coord,
snapshot: &Arc<StepSnapshot>,
event: &model::Event,
) {
for adapter in self.hooks.observer_hooks() {
let resources = self.resources(adapter.index(), coord);
adapter.invoke_model_event(
resources,
snapshot.tools().transcript(),
Some(snapshot),
event,
);
}
}
async fn publish_input_rejected(&self, reason: String) -> Result<(), Error> {
let coord = self.run_coord();
self.drain_hook_facts(&coord).await?;
self.outward
.producer
.emit_input_rejected(reason, coord)
.await
.map_err(Error::Event)?;
Ok(())
}
fn observe_audit(&self, coord: &Coord, event: &audit::Event) {
let transcript = self.controller.snapshot_reader();
for adapter in self.hooks.observer_hooks() {
let resources = self.resources(adapter.index(), coord);
adapter.invoke_audit_event(resources, &transcript, event);
}
}
async fn drain_hook_facts(&self, coord: &Coord) -> Result<(), Error> {
for fact in self.outward.facts.take_facts() {
let event = match fact {
BufferedFact::Audit(audit) => {
self.observe_audit(coord, &audit);
Event::from_audit(audit).map_err(Error::Event)?
}
BufferedFact::Error(error) => Event::from_error(error),
};
self.outward
.producer
.emit_extension(event, coord.clone())
.await
.expect("a successfully constructed ordinary fact remains valid");
}
Ok(())
}
fn has_turn_step_hooks(&self) -> bool {
self.hooks.turn_step_hooks().next().is_some()
}
fn has_run_hooks(&self) -> bool {
self.hooks.run_hooks().next().is_some()
}
fn staged_boundary<F>(
&self,
coord: &Coord,
cancellation: &Token,
registered: bool,
dispatch: F,
) -> Result<(), Error>
where
F: FnOnce(&StagedTranscript) -> Result<Vec<Item>, Error>,
{
if !registered {
return Ok(());
}
let controller = self.controller.with_coord(coord);
let (base_version, returned, staged_store) = {
let store = controller.staging_store().map_err(Error::Transcript)?;
let base_version = store.version().map_err(Error::Transcript)?;
let staged = StagedTranscript::from_store(&store);
let returned = dispatch(&staged)?;
drop(staged);
(base_version, returned, store)
};
if staged_store.version().map_err(Error::Transcript)? != base_version {
let snapshot = staged_store
.into_snapshot_with_suffix(returned)
.map_err(Error::Transcript)?;
controller
.replace_snapshot(snapshot, base_version, cancellation)
.map_err(boundary_transcript)?;
} else {
controller
.append_batch(returned, cancellation)
.map_err(boundary_transcript)?;
}
Ok(())
}
fn before_turn(
&self,
staged: &StagedTranscript,
trigger: &[Item],
cancellation: &Token,
) -> Result<Vec<Item>, Error> {
let coord = self.run_coord();
let mut returned = Vec::new();
for adapter in self.hooks.turn_step_hooks() {
let resources = self.resources(adapter.index(), &coord);
returned.extend(
adapter
.invoke_before_turn(resources, staged, trigger, cancellation)
.map_err(boundary_hook)?,
);
}
Ok(returned)
}
fn after_turn(
&self,
staged: &StagedTranscript,
trigger: &[Item],
cancellation: &Token,
) -> Result<Vec<Item>, Error> {
let coord = self.run_coord();
let mut returned = Vec::new();
for adapter in self.hooks.turn_step_hooks() {
let resources = self.resources(adapter.index(), &coord);
returned.extend(
adapter
.invoke_after_turn(resources, staged, trigger, cancellation)
.map_err(boundary_hook)?,
);
}
Ok(returned)
}
fn before_step(
&self,
staged: &StagedTranscript,
model: &Info,
plan: &mut RequestPlan,
cancellation: &Token,
) -> Result<Vec<Item>, Error> {
let coord = self.run_coord();
let mut returned = Vec::new();
for adapter in self.hooks.turn_step_hooks() {
let resources = self.resources(adapter.index(), &coord);
returned.extend(
adapter
.invoke_before_step(resources, staged, model, plan, cancellation)
.map_err(boundary_hook)?,
);
}
Ok(returned)
}
fn after_step(
&self,
staged: &StagedTranscript,
snapshot: &Arc<StepSnapshot>,
cancellation: &Token,
) -> Result<(bool, Vec<Item>), Error> {
let coord = self.run_coord();
let mut end_turn = false;
let mut returned = Vec::new();
for adapter in self.hooks.turn_step_hooks() {
let resources = self.resources(adapter.index(), &coord);
let decision = adapter
.invoke_after_step(resources, staged, snapshot, cancellation)
.map_err(boundary_hook)?;
match decision {
AfterStepDecision::Continue(items) => returned.extend(items),
AfterStepDecision::EndTurn(items) => {
end_turn = true;
returned.extend(items);
}
}
}
Ok((end_turn, returned))
}
fn compose_continuation(
&self,
staged: &StagedTranscript,
continuations: usize,
turns: usize,
cancellation: &Token,
) -> Result<Vec<Part>, Error> {
let coord = self.run_coord();
let mut continuation = Vec::new();
for adapter in self.hooks.run_hooks() {
let resources = self.resources(adapter.index(), &coord);
let decision = adapter
.invoke_compose_continuation(
resources,
staged,
&continuation,
continuations,
turns,
cancellation,
)
.map_err(boundary_hook)?;
match decision {
ContinuationDecision::Keep => {}
ContinuationDecision::Replace(replacement) => continuation = replacement,
}
}
Ok(continuation)
}
fn decide_stop(&self, cancellation: &Token) -> Result<Option<StopCode>, Error> {
let coord = self.run_coord();
let transcript = self.controller.snapshot_reader();
for adapter in self.hooks.run_hooks() {
let resources = self.resources(adapter.index(), &coord);
match adapter
.invoke_decide_stop(resources, &transcript, cancellation)
.map_err(boundary_hook)?
{
StopDecision::Continue => {}
StopDecision::Stop(code) => return Ok(Some(code)),
}
}
Ok(None)
}
fn before_model(
&self,
coord: &Coord,
snapshot: &Arc<StepSnapshot>,
request: Request,
cancellation: &Token,
) -> Result<ModelDecision, Error> {
let mut request = request;
for adapter in self.hooks.model_hooks() {
let resources = self.resources(adapter.index(), coord);
let decision = adapter
.invoke_before_model(resources, snapshot, &request, cancellation)
.map_err(boundary_hook)?;
match decision {
BeforeModelDecision::Continue => {}
BeforeModelDecision::Modify(replacement) => request = replacement,
BeforeModelDecision::ShortCircuit(output) => {
return Ok(ModelDecision::ShortCircuit(output));
}
BeforeModelDecision::Abort(reason) => {
return Err(Error::ModelAborted(reason));
}
}
}
Ok(ModelDecision::Generate(request))
}
#[expect(
clippy::too_many_arguments,
reason = "OnAfterModel observes five independent frozen model facts, matching its scope"
)]
fn after_model(
&self,
coord: &Coord,
snapshot: &Arc<StepSnapshot>,
output: Assistant,
generated: bool,
finish_reason: &str,
completion: CompletionReason,
cancellation: &Token,
) -> Result<Assistant, Error> {
let mut output = output;
for adapter in self.hooks.model_hooks() {
let resources = self.resources(adapter.index(), coord);
let decision = adapter
.invoke_after_model(
resources,
snapshot,
&output,
generated,
finish_reason,
completion,
cancellation,
)
.map_err(boundary_hook)?;
match decision {
AfterModelDecision::Continue => {}
AfterModelDecision::Modify(replacement) => output = replacement,
}
}
Ok(output)
}
pub(crate) async fn emit_lifecycle_event(
&self,
kind: Kind,
outcome: Option<Outcome>,
coord: Coord,
) -> Result<(), Error> {
self.drain_hook_facts(&coord).await?;
self.outward
.producer
.emit_lifecycle(kind, outcome, coord)
.await
.map_err(Error::Event)?;
Ok(())
}
async fn publish_provider_degradations(
&self,
coord: &Coord,
errors: Vec<ProviderError>,
) -> Result<(), Error> {
if errors.is_empty() {
return Ok(());
}
self.drain_hook_facts(coord).await?;
for error in errors {
self.outward
.producer
.emit_extension(Event::from_error(Arc::new(error)), coord.clone())
.await
.expect("a successfully constructed ordinary fact remains valid");
}
Ok(())
}
async fn invoke_after_tool_callback(
&self,
adapter: OwnedToolRegistrationAdapter,
resources: OwnedCallbackResources,
coord: Coord,
staged_store: Store,
snapshot: Arc<StepSnapshot>,
result: ToolResult,
) -> Result<AfterToolCallbackOutput, Error> {
self.drain_hook_facts(&coord).await?;
let callback_hook = adapter.name().to_owned();
let callback_coord = coord.clone();
let callback_write_permit = resources.callback_write_permit();
let (requests, request_receiver) = tokio::sync::mpsc::channel(1);
let (completed, completion) = mpsc::channel();
let handle = tokio::task::spawn_blocking(move || {
let _completion = completed;
match catch_unwind(AssertUnwindSafe(|| {
let emitter = RendezvousHookEmitter { requests };
let base_version = staged_store.version()?;
let staged = StagedTranscript::from_store(&staged_store);
let resources = resources.bind(&callback_coord, &emitter);
let returned =
adapter.invoke_after_tool_commit(resources, &staged, &snapshot, &result);
drop(staged);
let staged = if staged_store.version()? == base_version {
None
} else {
Some((base_version, staged_store.into_snapshot()?))
};
Ok(AfterToolCallbackOutput { staged, returned })
})) {
Ok(output) => AfterToolCallbackExit::Completed(output),
Err(payload) => AfterToolCallbackExit::Panicked(payload),
}
});
let mut owner = ManagedCallbackOwner {
requests: Some(request_receiver),
pending_ack: None,
callback_write_permit,
handle: Some(handle),
completed: Some(completion),
#[cfg(test)]
callback_abort_order: self.test_callback_abort_order.clone(),
#[cfg(test)]
callback_ack_probe: self.test_callback_ack_probe.clone(),
#[cfg(test)]
join_drop_handle_probe: None,
};
let mut stopped = None;
while let Some(request) = owner.recv().await {
let AfterToolRequest { fact, ack } = request;
owner.arm_ack(&ack);
if let AfterToolFact::Audit(audit) = &fact {
self.observe_audit(&coord, audit);
}
let event = match fact.into_event() {
Ok(event) => event,
Err(error) => {
owner.revoke_callback_writes_and_abort_pending_ack();
stopped = Some(Error::Event(error));
break;
}
};
let outcome = self
.outward
.producer
.emit_after_tool_commit(event, coord.clone())
.await
.expect("validated after-tool Audit/Error event");
match outcome {
StrongDecision::Accepted => owner.acknowledge_accept(ack),
StrongDecision::Rejected => {
owner.revoke_callback_writes_and_abort_pending_ack();
stopped = Some(Error::AfterToolExecuteRejected);
break;
}
StrongDecision::Cancelled => {
owner.revoke_callback_writes_and_abort_pending_ack();
stopped = Some(Error::Cancelled);
break;
}
}
}
let exit = owner.join().await;
if let Some(error) = stopped {
return Err(error);
}
match exit {
AfterToolCallbackExit::Completed(output) => output.map_err(Error::Transcript),
AfterToolCallbackExit::Panicked(payload) => {
propagate_after_tool_panic(callback_hook, payload);
}
}
}
async fn process(
&mut self,
generation: &Generation,
generation_coord: &Coord,
cancellation: &Token,
) -> Result<(), Error> {
let calls = generation.tool_calls();
for call in &calls {
let coord = tool_call_coord(generation_coord, call, call.name().as_str())
.map_err(Error::Coordinate)?;
self.announce_tool_call(&coord, generation.snapshot(), call);
self.drain_hook_facts(&coord).await?;
let outcome = self
.outward
.producer
.emit_tool_call(call.clone(), coord)
.await
.map_err(Error::Event)?;
match outcome {
StrongDecision::Accepted => {}
StrongDecision::Rejected => return Err(Error::ToolCallRejected),
StrongDecision::Cancelled => return Err(Error::Cancelled),
}
}
let execution_snapshot = Arc::clone(generation.snapshot().shared_tools());
let plan = prepare_tool_calls(
calls,
self.hooks,
self.callback_resources,
Arc::clone(generation.snapshot()),
cancellation,
)
.map_err(Error::Prepare)?;
self.record_effective_calls(&plan);
let mut emitter_gates = Vec::new();
let routed = {
let mut route_binder = |effective_call: &ToolCall| {
let coord = prepared_tool_call_coord(generation_coord, effective_call);
let (emitter, emitter_gate) = tool_emitter(self.outward.producer, coord);
emitter_gates.push(emitter_gate);
ToolRouteBinding::new(emitter)
};
route_and_execute(
plan,
execution_snapshot,
self.session_id.clone(),
&mut route_binder as &mut ToolRouteBinder<'_>,
cancellation,
)
.await
};
#[cfg(test)]
emitter_gates.append(&mut self.test_emitter_gates);
close_tool_emitters(emitter_gates).await;
let routed = routed.map_err(Error::Route)?;
self.record_routed_results(&routed);
let generation_controller = self.controller.with_coord(generation_coord);
let committed = commit_tool_batch(
routed,
generation.assistant().clone(),
generation_coord,
self.hooks,
self.callback_resources,
Arc::clone(generation.snapshot()),
&generation_controller,
cancellation,
)
.map_err(Error::Commit)?;
self.record_commit(&committed, ResultStage::Committed);
for finalized in committed.finalized() {
let coord = tool_call_coord(
generation_coord,
finalized.original_call(),
finalized.effective_call().name().as_str(),
)
.map_err(Error::Coordinate)?;
self.drain_hook_facts(&coord).await?;
let outcome = self
.outward
.producer
.emit_tool_result(finalized.item().clone(), coord)
.await
.map_err(Error::Event)?;
match outcome {
StrongDecision::Accepted => {}
StrongDecision::Rejected => return Err(Error::ToolResultRejected),
StrongDecision::Cancelled => return Err(Error::Cancelled),
}
}
for finalized in committed
.finalized()
.iter()
.filter(|finalized| routed_disposition(finalized.disposition()))
{
let coord = tool_call_coord(
generation_coord,
finalized.original_call(),
finalized.effective_call().name().as_str(),
)
.map_err(Error::Coordinate)?;
for adapter in self.hooks.owned_tool_hooks() {
let registration_resources = self
.callback_resources
.get(adapter.index())
.copied()
.expect("after-tool resources cover every registry index")
.owned_handles();
let staged_store = generation_controller
.staging_store()
.map_err(Error::Transcript)?;
let callback = self
.invoke_after_tool_callback(
adapter,
registration_resources,
coord.clone(),
staged_store,
Arc::clone(generation.snapshot()),
finalized.item().result().clone(),
)
.await?;
if let Some((base_version, snapshot)) = callback.staged {
generation_controller
.replace_snapshot(snapshot, base_version, &Token::new())
.map_err(Error::AfterToolExecuteTranscript)?;
}
generation_controller
.append_batch(callback.returned, &Token::new())
.map_err(Error::AfterToolExecuteTranscript)?;
}
}
self.record_commit(&committed, ResultStage::Closed);
let closed = std::mem::take(&mut self.open().calls);
self.close_tool_calls(closed, &LifecycleEnd::Completed);
Ok(())
}
fn announce_tool_call(&self, coord: &Coord, snapshot: &Arc<StepSnapshot>, call: &ToolCall) {
self.observe_lifecycle(
coord,
Some(snapshot),
&LifecycleObservation::ToolCallStart {
coord: coord.clone(),
call: call.clone(),
},
);
self.open().calls.push(OpenToolCall::announced(
coord.clone(),
Arc::clone(snapshot),
call.clone(),
));
}
fn record_effective_calls(&self, plan: &ToolBatchPreparePlan) {
let mut open = self.open();
for (call, prepared) in open.calls.iter_mut().zip(plan.entries()) {
call.effective = prepared.effective().clone();
}
}
fn record_routed_results(&self, routed: &RoutedToolBatch) {
let mut open = self.open();
for (call, record) in open.calls.iter_mut().zip(routed.records()) {
let outcome = record.outcome();
call.progress = CallProgress::settle(
outcome.disposition(),
outcome.result().clone(),
ResultStage::Routed,
);
}
}
fn record_commit(&self, committed: &ToolBatchCommit, stage: ResultStage) {
let mut open = self.open();
for (call, finalized) in open.calls.iter_mut().zip(committed.finalized()) {
call.progress = CallProgress::settle(
finalized.disposition(),
finalized.item().result().clone(),
stage,
);
}
}
}
async fn close_tool_emitters(gates: Vec<ToolEmitterGate>) {
for gate in gates {
gate.close_and_wait().await;
}
}
const fn routed_disposition(disposition: CallDisposition) -> bool {
!matches!(
disposition,
CallDisposition::Blocked
| CallDisposition::ShortCircuited
| CallDisposition::Truncated
| CallDisposition::NotExecuted
)
}
pub(super) fn tool_call_coord(
generation: &Coord,
call: &ToolCall,
effective_name: &str,
) -> Result<Coord, CoordError> {
let turn = NonZeroU64::new(generation.turn()).expect("tool calls belong to a turn");
let step = NonZeroU64::new(generation.step_in_turn()).expect("tool calls belong to a step");
let step_in_run =
NonZeroU64::new(generation.step_in_run()).expect("tool calls carry a run-wide step");
let generation_index =
NonZeroU64::new(generation.generation()).expect("tool calls belong to a generation");
let step = StepIndex::new(turn, step, step_in_run)
.expect("the runtime generation coordinate has a valid hierarchy");
Coord::at_tool_call(
generation.session_id().to_owned(),
generation.run_id().to_owned(),
step,
generation_index,
call.id().as_str().to_owned(),
effective_name.to_owned(),
)
}
fn prepared_tool_call_coord(generation: &Coord, effective_call: &ToolCall) -> Coord {
tool_call_coord(generation, effective_call, effective_call.name().as_str())
.expect("prepared tool calls have valid coordinates")
}
enum ModelDecision {
Generate(Request),
ShortCircuit(Assistant),
}
struct Step<'a, 'phase, M> {
context: &'a Context<'a, M>,
phase: Option<&'a ToolPhase<'phase>>,
coords: Option<&'a GenerationCoords>,
}
impl<M: Client> Step<'_, '_, M> {
async fn advance(self, mut plan: RequestPlan) -> Result<Generation, Error> {
let model = self.context.model.info();
if let Some(phase) = self.phase {
let cancellation = self.context.cancellation;
let coord = &self
.coords
.expect("an attached tool phase mints step coordinates")
.step;
phase.staged_boundary(coord, cancellation, phase.has_turn_step_hooks(), |staged| {
phase.before_step(staged, &model, &mut plan, cancellation)
})?;
}
let transcript = freeze_transcript(self.context.transcript_snapshot())?;
let resolution = await_or_cancel(
self.context.cancellation,
self.context
.tools
.resolve_step(transcript, self.context.cancellation),
)
.await?;
let (tools, provider_errors) = resolution.map_err(Error::Tools)?.into_parts();
if let Some(phase) = self.phase {
let coord = &self
.coords
.expect("an attached tool phase mints step coordinates")
.step;
phase
.publish_provider_degradations(coord, provider_errors)
.await?;
}
tools.apply_to_plan(&mut plan);
let snapshot = StepSnapshot::freeze(model, plan.clone(), tools);
let request = plan
.try_into_request(snapshot.tools().transcript().items().to_vec())
.map_err(Error::Request)?;
if let Some(phase) = self.phase {
let coords = self
.coords
.expect("an attached tool phase mints step coordinates");
phase.observe_lifecycle(
&coords.step,
Some(&snapshot),
&LifecycleObservation::StepStart {
coord: coords.step.clone(),
snapshot: Arc::clone(&snapshot),
},
);
phase.opened_step(&coords.step, &snapshot);
}
GenerationDriver {
model: self.context.model,
snapshot,
cancellation: self.context.cancellation,
phase: self.phase,
coords: self.coords,
}
.run(request)
.await
}
}
struct GenerationDriver<'a, 'phase, M> {
model: &'a M,
snapshot: Arc<StepSnapshot>,
cancellation: &'a Token,
phase: Option<&'a ToolPhase<'phase>>,
coords: Option<&'a GenerationCoords>,
}
impl<M: Client> GenerationDriver<'_, '_, M> {
async fn emit_content_delta(
&self,
id: String,
kind: ContentKind,
delta: String,
) -> Result<(), Error> {
let Some((phase, coords)) = self.phase.zip(self.coords) else {
return Ok(());
};
let kind = match kind {
ContentKind::Text => Kind::TextDelta,
ContentKind::Thinking => Kind::ThinkingDelta,
};
phase.drain_hook_facts(&coords.generation).await?;
phase
.outward
.producer
.emit_text(kind, delta, coords.generation.clone(), Some(id))
.await
.map_err(Error::Event)?;
Ok(())
}
fn observe_model_event(&self, event: &model::Event) {
if let Some((phase, coords)) = self.phase.zip(self.coords) {
phase.observe_model_event(&coords.generation, &self.snapshot, event);
}
}
async fn run(self, request: Request) -> Result<Generation, Error> {
let request = match self.phase.zip(self.coords) {
Some((phase, coords)) => {
match phase.before_model(
&coords.generation,
&self.snapshot,
request,
self.cancellation,
)? {
ModelDecision::Generate(request) => request,
ModelDecision::ShortCircuit(output) => {
return self.short_circuit(output, coords, phase);
}
}
}
None => request,
};
if let Some((phase, coords)) = self.phase.zip(self.coords) {
phase.observe_lifecycle(
&coords.generation,
Some(&self.snapshot),
&LifecycleObservation::GenerationStart {
coord: coords.generation.clone(),
request: request.clone(),
},
);
phase.opened_generation(&coords.generation, &self.snapshot);
}
self.generate(request).await
}
fn short_circuit(
&self,
output: Assistant,
coords: &GenerationCoords,
phase: &ToolPhase<'_>,
) -> Result<Generation, Error> {
let assistant = phase.after_model(
&coords.generation,
&self.snapshot,
output,
false,
"",
CompletionReason::Unknown,
self.cancellation,
)?;
Ok(Generation {
snapshot: Arc::clone(&self.snapshot),
assistant,
finish_reason: String::new(),
completion: CompletionReason::Unknown,
})
}
async fn generate(self, request: Request) -> Result<Generation, Error> {
closing_cancel_gate(self.cancellation)?;
let events = await_or_cancel(self.cancellation, self.model.generate(request)).await?;
let events = events.map_err(|error| Error::Model(Arc::new(error)))?;
closing_cancel_gate(self.cancellation)?;
tokio::pin!(events);
let mut grammar = StreamGrammar::new();
let mut segments = Vec::new();
let mut terminal = None;
loop {
let event = await_or_cancel(
self.cancellation,
poll_fn(|context| events.as_mut().poll_next(context)),
)
.await?;
let Some(event) = event else {
closing_cancel_gate(self.cancellation)?;
break;
};
let event = event.map_err(|error| Error::Model(Arc::new(error)))?;
grammar.accept(&event).map_err(Error::Stream)?;
closing_cancel_gate(self.cancellation)?;
self.observe_model_event(&event);
match event {
model::Event::ContentFinal {
id,
kind,
content,
extensions,
} => {
let extensions = extensions.with_item_id(id);
segments.push(match kind {
ContentKind::Text => Segment::Text {
text: content,
extensions,
},
ContentKind::Thinking => Segment::Thinking {
text: content,
extensions,
},
});
}
model::Event::ToolCallFinal { call, extensions } => {
segments.push(Segment::ToolCall { call, extensions });
}
model::Event::Done {
finish_reason,
completion,
provider_data,
} => terminal = Some((finish_reason, completion, provider_data)),
model::Event::ContentDelta { id, kind, delta } => {
self.emit_content_delta(id, kind, delta).await?;
}
model::Event::ToolCallStart { .. }
| model::Event::ToolCallArgumentsDelta { .. }
| model::Event::Usage(_) => {}
}
}
grammar.finish().map_err(Error::Stream)?;
let (finish_reason, completion, provider_data) =
terminal.expect("StreamGrammar accepted a terminal event");
let assistant = Assistant::new(segments, Some(provider_data));
let assistant = match self.phase.zip(self.coords) {
Some((phase, coords)) => phase.after_model(
&coords.generation,
&self.snapshot,
assistant,
true,
&finish_reason,
completion,
self.cancellation,
)?,
None => assistant,
};
Ok(Generation {
snapshot: self.snapshot,
assistant,
finish_reason,
completion,
})
}
}
pub(super) fn freeze_transcript(
snapshot: Result<crate::transcript::store::Snapshot, store::Error>,
) -> Result<crate::transcript::store::Snapshot, Error> {
snapshot.map_err(Error::Transcript)
}
async fn await_or_cancel<T>(
cancellation: &Token,
future: impl Future<Output = T>,
) -> Result<T, Error> {
tokio::select! {
biased;
output = future => Ok(output),
() = cancellation.cancelled() => Err(Error::Cancelled),
}
}
fn closing_cancel_gate(cancellation: &Token) -> Result<(), Error> {
if cancellation.is_cancelled() {
Err(Error::Cancelled)
} else {
Ok(())
}
}