#![cfg(test)]
use std::collections::VecDeque;
use std::error;
use std::fmt;
use std::future::{Future, ready};
use std::num::NonZeroU64;
use std::panic::AssertUnwindSafe;
use std::pin::Pin;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use std::task::{Context, Poll, Waker};
use std::thread;
use crate::cancel::Token;
use tokio::sync::{Notify, watch};
use super::agent::{Agent, Config};
use super::driver::{
Advance, AfterToolHookPanic, CallbackAbortStep, Driver, Error, InputAdmitError,
InputAdmitOutcome, Outward, Terminal, TerminalLatch, ToolPhase, lifecycle_transcript_source,
tool_call_coord,
};
use super::event::testing::{
FreezePoint, InterruptionProbe, interruption_probe, pause_after_final_confirm,
pause_before_reject,
};
use super::event::{
DecisionError, DecisionPoint, Delivery, DeliveryOutcome, Kind, StopCode, ToolEmitError,
ValidationError, channel, tool_emitter,
};
use super::hook::{
self, AfterModelDecision, AfterModelScope, AfterStepDecision, AfterStepScope,
AfterToolCommitScope, AttributedError, BeforeModelDecision, BeforeModelScope, BeforeStepScope,
CallbackResources, ContinuationDecision, ContinuationScope, DispatchError, Emitter, FactBuffer,
HookError, HookResult, InputAdmitDecision, InputAdmitScope, InputHooks, LifecycleEnd,
LifecycleKind, LifecycleObservation, ModelHooks, ObserverHooks, ObserverScope, PrepareDecision,
PrepareScope, ReadScope, RunHooks, StagedTranscript, StepSnapshot, StopDecision,
StopDecisionScope, ToolCallDecision, ToolCallScope, ToolHooks, ToolResultDecision,
ToolResultScope, TranscriptHooks, TranscriptMutation, TranscriptReservation, TurnScope,
TurnStepHooks,
};
use super::model::{
self, Client, CompletionReason, ContentKind, Info, Request, RequestError, RequestPlan,
StreamError, TokenUsage,
};
use super::run::FailureKind;
use super::session::{InputBatch, InputMailbox, InputTarget, Lease, LeaseError, Session};
use super::tool::{
self, Descriptor, ExecutionError, Executor, ExecutorError, ExposeAll, Invocation, MappingError,
Provider, ProviderError, ProviderResolution, StepResolutionError,
};
use super::tool_batch_commit::ToolBatchCommitError;
use super::tool_batch_route::ToolBatchRouteError;
use crate::audit;
use crate::opaque::{Envelope, Payload};
use crate::state::run_state::{self};
use crate::state::session_state::{self, SnapshotValue, WriteError};
use crate::transcript::controller::{ControllerError, ReservationResources, TranscriptController};
use crate::transcript::item::{Assistant, Item, ModelVisibility, System, User};
use crate::transcript::store::{self, Snapshot, Store, Version, poison};
use crate::value::content::{Part, Segment, SegmentExtensions};
use crate::value::lifecycle::{Coord, CoordError, Outcome, StepIndex};
use crate::value::tool_call::{self, Id, ToolCall, ToolResult};
fn assert_pending_once<F: Future + ?Sized>(future: Pin<&mut F>, message: &str) {
let mut context = Context::from_waker(Waker::noop());
assert!(
matches!(future.poll(&mut context), Poll::Pending),
"{message}"
);
}
#[derive(Clone, Debug)]
struct FakeError(&'static str);
impl fmt::Display for FakeError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str(self.0)
}
}
impl error::Error for FakeError {}
struct FakeEvents {
events: VecDeque<Result<model::Event, FakeError>>,
pending: Option<Arc<Notify>>,
polls: usize,
cancel_at: Option<(usize, Token)>,
interrupt_at: Option<(usize, InterruptionProbe)>,
}
impl FakeEvents {
fn ready(events: Vec<Result<model::Event, FakeError>>) -> Self {
Self {
events: events.into(),
pending: None,
polls: 0,
cancel_at: None,
interrupt_at: None,
}
}
fn cancelling(
events: Vec<Result<model::Event, FakeError>>,
poll: usize,
cancellation: Token,
) -> Self {
Self {
events: events.into(),
pending: None,
polls: 0,
cancel_at: Some((poll, cancellation)),
interrupt_at: None,
}
}
fn interrupting(
events: Vec<Result<model::Event, FakeError>>,
poll: usize,
interruption: InterruptionProbe,
) -> Self {
Self {
events: events.into(),
pending: None,
polls: 0,
cancel_at: None,
interrupt_at: Some((poll, interruption)),
}
}
fn pending(notify: Arc<Notify>) -> Self {
Self {
events: VecDeque::new(),
pending: Some(notify),
polls: 0,
cancel_at: None,
interrupt_at: None,
}
}
}
impl futures_core::Stream for FakeEvents {
type Item = Result<model::Event, FakeError>;
fn poll_next(mut self: Pin<&mut Self>, _context: &mut Context<'_>) -> Poll<Option<Self::Item>> {
self.polls += 1;
if self
.cancel_at
.as_ref()
.is_some_and(|(poll, _)| *poll == self.polls)
{
self.cancel_at
.take()
.expect("matching cancellation action exists")
.1
.cancel();
}
if self
.interrupt_at
.as_ref()
.is_some_and(|(poll, _)| *poll == self.polls)
{
self.interrupt_at
.take()
.expect("matching interruption action exists")
.1
.request_stop();
}
if let Some(event) = self.events.pop_front() {
return Poll::Ready(Some(event));
}
if let Some(notify) = &self.pending {
notify.notify_one();
Poll::Pending
} else {
Poll::Ready(None)
}
}
}
enum Start {
Ready(Result<FakeEvents, FakeError>),
Wait {
entered: Arc<Notify>,
release: Arc<Notify>,
result: Result<FakeEvents, FakeError>,
},
}
struct FakeModel {
starts: Mutex<VecDeque<Start>>,
requests: Mutex<Vec<Request>>,
calls: AtomicUsize,
}
impl FakeModel {
fn ready(events: Vec<Result<model::Event, FakeError>>) -> Self {
Self::starting(Start::Ready(Ok(FakeEvents::ready(events))))
}
fn starting(start: Start) -> Self {
Self::scripted(vec![start])
}
fn scripted(starts: Vec<Start>) -> Self {
Self {
starts: Mutex::new(starts.into()),
requests: Mutex::new(Vec::new()),
calls: AtomicUsize::new(0),
}
}
fn calls(&self) -> usize {
self.calls.load(Ordering::SeqCst)
}
}
impl Client for FakeModel {
type Error = FakeError;
type Stream = FakeEvents;
fn info(&self) -> Info {
Info::new("fake".to_owned(), 8_192)
.with_tools(true)
.with_thinking(true)
}
fn generate(
&self,
request: Request,
) -> impl Future<Output = Result<Self::Stream, Self::Error>> + Send {
self.calls.fetch_add(1, Ordering::SeqCst);
self.requests.lock().unwrap().push(request);
let start = self
.starts
.lock()
.unwrap()
.pop_front()
.expect("one scripted model start per call");
async move {
match start {
Start::Ready(result) => result,
Start::Wait {
entered,
release,
result,
} => {
entered.notify_one();
release.notified().await;
result
}
}
}
}
}
#[derive(Default)]
struct ModelOwnerCounts {
future_active: AtomicUsize,
future_dropped: AtomicUsize,
future_completed: AtomicUsize,
stream_active: AtomicUsize,
stream_dropped: AtomicUsize,
stream_completed: AtomicUsize,
}
struct ModelFutureOwner(Arc<ModelOwnerCounts>);
impl Drop for ModelFutureOwner {
fn drop(&mut self) {
self.0.future_active.fetch_sub(1, Ordering::SeqCst);
self.0.future_dropped.fetch_add(1, Ordering::SeqCst);
}
}
struct ModelStreamOwner(Arc<ModelOwnerCounts>);
impl Drop for ModelStreamOwner {
fn drop(&mut self) {
self.0.stream_active.fetch_sub(1, Ordering::SeqCst);
self.0.stream_dropped.fetch_add(1, Ordering::SeqCst);
}
}
#[derive(Clone)]
enum OwnerModelMode {
Complete,
PendingFuture(Arc<Notify>),
PendingStream(Arc<Notify>),
}
struct OwnerModel {
mode: OwnerModelMode,
counts: Arc<ModelOwnerCounts>,
}
struct OwnerEvents {
mode: OwnerModelMode,
events: VecDeque<Result<model::Event, FakeError>>,
counts: Arc<ModelOwnerCounts>,
_owner: ModelStreamOwner,
}
impl futures_core::Stream for OwnerEvents {
type Item = Result<model::Event, FakeError>;
fn poll_next(mut self: Pin<&mut Self>, _context: &mut Context<'_>) -> Poll<Option<Self::Item>> {
if let Some(event) = self.events.pop_front() {
return Poll::Ready(Some(event));
}
match &self.mode {
OwnerModelMode::PendingStream(entered) => {
entered.notify_one();
Poll::Pending
}
OwnerModelMode::Complete | OwnerModelMode::PendingFuture(_) => {
self.counts.stream_completed.fetch_add(1, Ordering::SeqCst);
Poll::Ready(None)
}
}
}
}
impl Client for OwnerModel {
type Error = FakeError;
type Stream = OwnerEvents;
fn info(&self) -> Info {
Info::new("owned".to_owned(), 8_192)
}
fn generate(
&self,
_request: Request,
) -> impl Future<Output = Result<Self::Stream, Self::Error>> + Send {
self.counts.future_active.fetch_add(1, Ordering::SeqCst);
let owner = ModelFutureOwner(Arc::clone(&self.counts));
let counts = Arc::clone(&self.counts);
let mode = self.mode.clone();
async move {
if let OwnerModelMode::PendingFuture(entered) = &mode {
entered.notify_one();
std::future::pending::<()>().await;
}
counts.future_completed.fetch_add(1, Ordering::SeqCst);
counts.stream_active.fetch_add(1, Ordering::SeqCst);
let events = if matches!(mode, OwnerModelMode::Complete) {
vec![Ok(done())].into()
} else {
VecDeque::new()
};
let stream = OwnerEvents {
mode,
events,
counts: Arc::clone(&counts),
_owner: ModelStreamOwner(counts),
};
drop(owner);
Ok(stream)
}
}
}
fn done() -> model::Event {
model::Event::Done {
finish_reason: "stop".to_owned(),
completion: CompletionReason::Stop,
provider_data: Vec::new(),
}
}
fn extensions() -> SegmentExtensions {
SegmentExtensions::new(String::new(), None, None)
}
fn user(text: &str) -> Item {
Item::User(User::new(
vec![Part::text(text.to_owned())],
ModelVisibility::Visible,
))
}
fn system(text: &str) -> Item {
Item::System(System::new(
"after-tool".to_owned(),
text.to_owned(),
ModelVisibility::Visible,
))
}
fn lease(seed: Vec<Item>) -> (Session, Lease) {
let session = Session::new(
"session".to_owned(),
Some(Snapshot::new(Version::ZERO, seed)),
);
let lease = session.try_acquire().unwrap();
(session, lease)
}
fn release_and_read_successor(session: &Session, lease: &Lease) -> Snapshot {
assert!(
session.try_acquire().is_none(),
"the caller still owns the current lease before release"
);
lease.release();
let successor = session
.try_acquire()
.expect("run join barrier makes the session immediately reusable");
let snapshot = successor
.transcript_snapshot()
.expect("successor reads one atomic transcript truth");
successor.release();
snapshot
}
fn empty_tools() -> tool::Registry {
tool::RegistryBuilder::new().build().unwrap()
}
fn hosted(name: &str) -> tool::Registration {
let name = tool::Name::local(name.to_owned()).unwrap();
tool::Registration::hosted(
Descriptor::new(name, "hosted".to_owned(), Payload::copy_from_slice(b"{}")),
Envelope::try_new(
"provider".to_owned(),
"hosted".to_owned(),
Payload::copy_from_slice(b"{}"),
)
.unwrap(),
)
}
async fn advance<M: Client>(
model: &M,
lease: &Lease,
inbox: &InputMailbox,
tools: &tool::Registry,
plan: RequestPlan,
cancellation: &Token,
) -> Result<Option<Advance>, Error> {
Driver::new(model, lease, inbox, tools, cancellation)
.advance(plan)
.await
}
fn expect_advance(result: Result<Option<Advance>, Error>) -> Option<Advance> {
result.unwrap_or_else(|_| panic!("expected successful advance"))
}
fn expect_generation(result: Result<Option<Advance>, Error>) -> super::driver::Generation {
match result
.unwrap_or_else(|_| panic!("expected generation"))
.unwrap_or_else(|| panic!("driver ended before generation"))
{
Advance::Generation(generation) => generation,
Advance::Input(_) => panic!("unexpected inbox claim"),
}
}
fn input_text(advance: Option<Advance>) -> (InputTarget, String) {
let Some(Advance::Input(input)) = advance else {
panic!("expected inbox claim");
};
let Item::User(item) = &input.items()[0] else {
panic!("expected user input");
};
(input.target(), item.parts()[0].fields().text().to_owned())
}
#[tokio::test]
async fn persistent_driver_claims_mixed_inputs_only_after_their_completed_boundaries() {
let (_session, lease) = lease(Vec::new());
let inbox = InputMailbox::new();
for (target, text) in [
(InputTarget::NextTurn, "turn-1"),
(InputTarget::NextStep, "step-1"),
(InputTarget::NextTurn, "turn-2"),
(InputTarget::NextStep, "step-2"),
] {
inbox.push(InputBatch::new(target, vec![user(text)]).unwrap());
}
let model = FakeModel::scripted(
(0..5)
.map(|_| Start::Ready(Ok(FakeEvents::ready(vec![Ok(done())]))))
.collect(),
);
let cancellation = Token::new();
let tools = empty_tools();
let mut driver = Driver::new(&model, &lease, &inbox, &tools, &cancellation);
expect_generation(driver.advance(RequestPlan::new()).await);
assert_eq!(model.calls(), 1, "the first boundary must be a generation");
assert_eq!(inbox.len(), 4);
for (target, text, calls) in [
(InputTarget::NextStep, "step-1", 1),
(InputTarget::NextStep, "step-2", 2),
(InputTarget::NextTurn, "turn-1", 3),
(InputTarget::NextTurn, "turn-2", 4),
] {
assert_eq!(
input_text(expect_advance(driver.advance(RequestPlan::new()).await)),
(target, text.to_owned())
);
assert_eq!(model.calls(), calls, "a claim must not invoke the model");
expect_generation(driver.advance(RequestPlan::new()).await);
assert_eq!(model.calls(), calls + 1);
}
assert!(expect_advance(driver.advance(RequestPlan::new()).await).is_none());
assert!(expect_advance(driver.advance(RequestPlan::new()).await).is_none());
assert!(inbox.is_empty());
}
#[tokio::test]
async fn owner_run_drives_all_boundaries_to_natural_completion() {
let (_session, lease) = lease(Vec::new());
let inbox = InputMailbox::new();
inbox.push(InputBatch::new(InputTarget::NextStep, vec![user("step")]).unwrap());
inbox.push(InputBatch::new(InputTarget::NextTurn, vec![user("turn")]).unwrap());
let model = FakeModel::scripted(
(0..3)
.map(|_| Start::Ready(Ok(FakeEvents::ready(vec![Ok(done())]))))
.collect(),
);
let tools = empty_tools();
let cancellation = Token::new();
let (_producer, source) = channel(&cancellation);
let driver = Driver::new(&model, &lease, &inbox, &tools, &cancellation);
let terminal = driver.run(RequestPlan::new(), &source).await;
assert!(matches!(terminal, Terminal::Completed));
assert_eq!(
terminal.outcome(),
crate::value::lifecycle::Outcome::Completed
);
assert!(terminal.failure().is_none());
assert_eq!(model.calls(), 3);
assert!(inbox.is_empty());
assert!(!super::event::testing::has_pending_strong_decision(&source));
}
#[tokio::test]
async fn owner_run_return_is_a_model_future_and_stream_join_barrier() {
let (session, lease) = lease(Vec::new());
let counts = Arc::new(ModelOwnerCounts::default());
let model = OwnerModel {
mode: OwnerModelMode::Complete,
counts: Arc::clone(&counts),
};
let inbox = InputMailbox::new();
let tools = empty_tools();
let cancellation = Token::new();
let (_producer, source) = channel(&cancellation);
let driver = Driver::new(&model, &lease, &inbox, &tools, &cancellation);
let terminal = driver.run(RequestPlan::new(), &source).await;
assert!(matches!(terminal, Terminal::Completed));
assert_eq!(counts.future_active.load(Ordering::SeqCst), 0);
assert_eq!(counts.future_completed.load(Ordering::SeqCst), 1);
assert_eq!(counts.future_dropped.load(Ordering::SeqCst), 1);
assert_eq!(counts.stream_active.load(Ordering::SeqCst), 0);
assert_eq!(counts.stream_completed.load(Ordering::SeqCst), 1);
assert_eq!(counts.stream_dropped.load(Ordering::SeqCst), 1);
assert!(!super::event::testing::has_pending_strong_decision(&source));
let successor = release_and_read_successor(&session, &lease);
assert_eq!(successor.version(), Version::ZERO);
assert!(successor.items().is_empty());
}
#[tokio::test]
async fn dropping_run_at_model_future_or_stream_seam_drops_the_owned_model_work() {
for pending_future in [true, false] {
let (session, lease) = lease(Vec::new());
let entered = Arc::new(Notify::new());
let counts = Arc::new(ModelOwnerCounts::default());
let mode = if pending_future {
OwnerModelMode::PendingFuture(Arc::clone(&entered))
} else {
OwnerModelMode::PendingStream(Arc::clone(&entered))
};
let model = OwnerModel {
mode,
counts: Arc::clone(&counts),
};
let inbox = InputMailbox::new();
let tools = empty_tools();
let cancellation = Token::new();
let (_producer, source) = channel(&cancellation);
let driver = Driver::new(&model, &lease, &inbox, &tools, &cancellation);
let mut run = Box::pin(driver.run(RequestPlan::new(), &source));
tokio::select! {
() = entered.notified() => {}
terminal = &mut run => panic!("run ended before the model seam: {:?}", terminal.outcome()),
}
assert_eq!(
counts.future_active.load(Ordering::SeqCst),
usize::from(pending_future)
);
assert_eq!(
counts.stream_active.load(Ordering::SeqCst),
usize::from(!pending_future)
);
drop(run);
assert_eq!(counts.future_active.load(Ordering::SeqCst), 0);
assert_eq!(counts.future_dropped.load(Ordering::SeqCst), 1);
assert_eq!(
counts.future_completed.load(Ordering::SeqCst),
usize::from(!pending_future)
);
assert_eq!(counts.stream_active.load(Ordering::SeqCst), 0);
assert_eq!(
counts.stream_dropped.load(Ordering::SeqCst),
usize::from(!pending_future)
);
assert_eq!(counts.stream_completed.load(Ordering::SeqCst), 0);
assert!(!super::event::testing::has_pending_strong_decision(&source));
let successor = release_and_read_successor(&session, &lease);
assert_eq!(successor.version(), Version::ZERO);
assert!(successor.items().is_empty());
}
}
#[tokio::test]
async fn consumer_drop_interrupts_pending_model_future_and_stream_before_release() {
for pending_future in [true, false] {
let (session, lease) = lease(Vec::new());
let entered = Arc::new(Notify::new());
let counts = Arc::new(ModelOwnerCounts::default());
let mode = if pending_future {
OwnerModelMode::PendingFuture(Arc::clone(&entered))
} else {
OwnerModelMode::PendingStream(Arc::clone(&entered))
};
let model = OwnerModel {
mode,
counts: Arc::clone(&counts),
};
let inbox = InputMailbox::new();
let tools = empty_tools();
let cancellation = Token::new();
let (_producer, mut source) = channel(&cancellation);
let stream = source.take_stream().unwrap();
let driver = Driver::new(&model, &lease, &inbox, &tools, &cancellation);
let mut run = Box::pin(driver.run(RequestPlan::new(), &source));
tokio::select! {
() = entered.notified() => {}
terminal = &mut run => panic!("run ended before pending model work: {:?}", terminal.outcome()),
}
drop(stream);
let terminal = tokio::time::timeout(std::time::Duration::from_secs(1), &mut run)
.await
.expect("consumer drop interrupts pending model work");
assert!(matches!(terminal, Terminal::Stopped));
assert_eq!(counts.future_active.load(Ordering::SeqCst), 0);
assert_eq!(counts.future_dropped.load(Ordering::SeqCst), 1);
assert_eq!(
counts.future_completed.load(Ordering::SeqCst),
usize::from(!pending_future)
);
assert_eq!(counts.stream_active.load(Ordering::SeqCst), 0);
assert_eq!(
counts.stream_dropped.load(Ordering::SeqCst),
usize::from(!pending_future)
);
assert_eq!(counts.stream_completed.load(Ordering::SeqCst), 0);
assert!(!super::event::testing::has_pending_strong_decision(&source));
let snapshot = lease.transcript_snapshot().unwrap();
assert_eq!(snapshot.version(), Version::ZERO);
assert!(snapshot.items().is_empty());
let successor = release_and_read_successor(&session, &lease);
assert_eq!(successor.version(), Version::ZERO);
assert!(successor.items().is_empty());
}
}
#[tokio::test]
async fn owner_run_maps_explicit_cancellation_without_calling_model() {
let (_session, lease) = lease(Vec::new());
let model = FakeModel::ready(vec![Ok(done())]);
let inbox = InputMailbox::new();
let tools = empty_tools();
let cancellation = Token::new();
let (_producer, source) = channel(&cancellation);
cancellation.cancel();
let driver = Driver::new(&model, &lease, &inbox, &tools, &cancellation);
let terminal = driver.run(RequestPlan::new(), &source).await;
assert!(matches!(terminal, Terminal::Cancelled));
assert_eq!(
terminal.outcome(),
crate::value::lifecycle::Outcome::Cancelled
);
assert!(terminal.failure().is_none());
assert_eq!(model.calls(), 0);
}
#[tokio::test]
async fn owner_preserves_first_nonfatal_cause_when_cancel_and_stop_are_ready_together() {
for cancel_first in [true, false] {
let (_session, lease) = lease(Vec::new());
let model = FakeModel::ready(vec![Ok(done())]);
let inbox = InputMailbox::new();
let tools = empty_tools();
let cancellation = Token::new();
let (_producer, source) = channel(&cancellation);
if cancel_first {
cancellation.cancel();
source.request_stop();
} else {
source.request_stop();
cancellation.cancel();
}
let driver = Driver::new(&model, &lease, &inbox, &tools, &cancellation);
let terminal = driver.run(RequestPlan::new(), &source).await;
assert!(if cancel_first {
matches!(terminal, Terminal::Cancelled)
} else {
matches!(terminal, Terminal::Stopped)
});
assert_eq!(model.calls(), 0);
}
}
#[tokio::test]
async fn owner_preserves_the_producer_owned_single_stop() {
let (_session, lease) = lease(Vec::new());
let model = FakeModel::ready(vec![Ok(done())]);
let inbox = InputMailbox::new();
let tools = empty_tools();
let cancellation = Token::new();
let (producer, mut source) = channel(&cancellation);
let mut stream = source.take_stream().unwrap();
let coord = generation_coord();
assert_eq!(
producer
.emit_stop(StopCode::new("policy".to_owned()).unwrap(), coord.clone())
.await
.unwrap(),
DeliveryOutcome::Delivered
);
assert_eq!(
producer
.emit_stop(StopCode::new("duplicate".to_owned()).unwrap(), coord)
.await
.unwrap(),
DeliveryOutcome::Rejected
);
let driver = Driver::new(&model, &lease, &inbox, &tools, &cancellation);
let terminal = driver.run(RequestPlan::new(), &source).await;
assert!(matches!(terminal, Terminal::Stopped));
assert_eq!(
terminal.outcome(),
crate::value::lifecycle::Outcome::Stopped
);
assert!(terminal.failure().is_none());
assert_eq!(model.calls(), 0);
let stop = stream.recv().await.expect("the producer's unique stop");
assert_eq!(stop.event().stop_code().unwrap().as_str(), "policy");
let next = stream.recv();
tokio::pin!(next);
assert_pending_once(next.as_mut(), "the owner must not emit a duplicate Stop");
}
#[tokio::test]
async fn event_validation_fatal_beats_late_cancellation_before_model_start() {
let (_session, lease) = lease(Vec::new());
let model = FakeModel::ready(vec![Ok(done())]);
let inbox = InputMailbox::new();
let tools = empty_tools();
let cancellation = Token::new();
let (producer, source) = channel(&cancellation);
assert_eq!(
producer
.emit(
super::event::testing::invalid_audit(generation_coord()),
None,
)
.await
.unwrap_err(),
super::event::ValidationError::EmptyAuditKind
);
let driver = Driver::new(&model, &lease, &inbox, &tools, &cancellation);
let terminal = driver.run(RequestPlan::new(), &source).await;
cancellation.cancel();
assert!(matches!(
terminal.failure(),
Some(Error::Event(super::event::ValidationError::EmptyAuditKind))
));
assert_eq!(terminal.outcome(), crate::value::lifecycle::Outcome::Failed);
assert_eq!(model.calls(), 0);
}
#[tokio::test]
async fn cancellation_observed_before_a_real_model_error_is_upgraded_to_failed() {
let (_session, lease) = lease(Vec::new());
let cancellation = Token::new();
let model = FakeModel::starting(Start::Ready(Ok(FakeEvents::cancelling(
vec![Err(FakeError("fatal-after-cancel"))],
1,
cancellation.clone(),
))));
let inbox = InputMailbox::new();
let tools = empty_tools();
let (_producer, source) = channel(&cancellation);
let driver = Driver::new(&model, &lease, &inbox, &tools, &cancellation);
let terminal = driver.run(RequestPlan::new(), &source).await;
assert!(
matches!(terminal.failure(), Some(Error::Model(error)) if error.to_string() == "fatal-after-cancel")
);
assert_eq!(terminal.outcome(), crate::value::lifecycle::Outcome::Failed);
}
#[tokio::test]
async fn real_model_fatal_wins_when_advance_and_source_interruption_ready_in_same_poll() {
let (_session, lease) = lease(Vec::new());
let cancellation = Token::new();
let (_producer, source) = channel(&cancellation);
let model = FakeModel::starting(Start::Ready(Ok(FakeEvents::interrupting(
vec![Err(FakeError("fatal-with-stop"))],
1,
interruption_probe(&source),
))));
let inbox = InputMailbox::new();
let tools = empty_tools();
let driver = Driver::new(&model, &lease, &inbox, &tools, &cancellation);
let terminal = driver.run(RequestPlan::new(), &source).await;
assert!(
matches!(terminal.failure(), Some(Error::Model(error)) if error.to_string() == "fatal-with-stop")
);
assert_eq!(terminal.outcome(), crate::value::lifecycle::Outcome::Failed);
}
#[test]
fn terminal_latch_keeps_first_nonfatal_and_first_fatal_while_fatal_still_upgrades() {
let cancellation = Token::new();
let (_producer, source) = channel(&cancellation);
let mut canceled_first = TerminalLatch::default();
canceled_first.record(Terminal::Cancelled);
canceled_first.record(Terminal::Stopped);
assert!(matches!(
canceled_first.finish(&source),
Terminal::Cancelled
));
let mut stopped_first = TerminalLatch::default();
stopped_first.record(Terminal::Stopped);
stopped_first.record(Terminal::Cancelled);
assert!(matches!(stopped_first.finish(&source), Terminal::Stopped));
let mut latch = TerminalLatch::default();
let completed = Terminal::Completed;
assert_eq!(
completed.outcome(),
crate::value::lifecycle::Outcome::Completed
);
assert!(completed.failure().is_none());
latch.record(completed);
latch.record(Terminal::Cancelled);
latch.record(Terminal::Stopped);
latch.record(Terminal::Failed(Error::Model(Arc::new(FakeError("first")))));
latch.record(Terminal::Failed(Error::Model(Arc::new(FakeError(
"second",
)))));
let terminal = latch.finish(&source);
assert!(
matches!(terminal.failure(), Some(Error::Model(error)) if error.to_string() == "first")
);
}
#[test]
fn owner_maps_every_nested_cancellation_surface_without_failure() {
let errors = [
Error::Cancelled,
Error::Prepare(DispatchError::Cancelled),
Error::Route(super::tool_batch_route::ToolBatchRouteError::new(
super::tool_batch::Error::cancelled_batch(),
)),
Error::Commit(super::tool_batch_commit::ToolBatchCommitError::Cancelled),
Error::Commit(super::tool_batch_commit::ToolBatchCommitError::Rewrite {
source_index: 0,
error: DispatchError::Cancelled,
}),
Error::Commit(super::tool_batch_commit::ToolBatchCommitError::Controller(
ControllerError::Dispatch(DispatchError::Cancelled),
)),
Error::AfterToolExecuteTranscript(ControllerError::Dispatch(DispatchError::Cancelled)),
];
for error in errors {
let cancellation = Token::new();
let (_producer, source) = channel(&cancellation);
let mut latch = TerminalLatch::default();
latch.record_error(&source, error);
let terminal = latch.finish(&source);
assert_eq!(
terminal.outcome(),
crate::value::lifecycle::Outcome::Cancelled
);
assert!(terminal.failure().is_none());
}
let cancellation = Token::new();
let (_producer, source) = channel(&cancellation);
let mut stopped = TerminalLatch::default();
stopped.record(Terminal::Stopped);
stopped.record_error(&source, Error::Cancelled);
assert!(matches!(stopped.finish(&source), Terminal::Stopped));
}
struct ObservingProvider {
versions: Arc<Mutex<Vec<Version>>>,
}
impl Provider for ObservingProvider {
fn resolve<'a>(
&'a self,
transcript: &'a Snapshot,
_cancellation: Token,
) -> Pin<Box<dyn Future<Output = ProviderResolution> + Send + 'a>> {
self.versions.lock().unwrap().push(transcript.version());
Box::pin(ready(ProviderResolution::success(Vec::new())))
}
}
#[test]
fn lifecycle_transcript_sources_match_start_and_end_phase_contracts() {
let live_store = Store::restore(Snapshot::new(Version::from_u64(9), vec![system("live")]));
let live = live_store.snapshot_reader();
let frozen = Snapshot::new(Version::from_u64(3), vec![system("frozen")]);
let step = StepSnapshot::freeze(
Info::new("fake".to_owned(), 1),
RequestPlan::new(),
empty_tools()
.resolve_step_without_providers(frozen.clone(), &Token::new(), &ExposeAll)
.unwrap()
.into_snapshot(),
);
for kind in [
LifecycleKind::StepStart,
LifecycleKind::GenerationStart,
LifecycleKind::ToolCallStart,
] {
let observed = lifecycle_transcript_source(kind, Some(&step), &live)
.snapshot()
.unwrap();
assert_eq!(observed.version(), Version::from_u64(3));
assert!(observed.shares_items_with(&frozen));
}
for kind in [
LifecycleKind::RunStart,
LifecycleKind::RunEnd,
LifecycleKind::TurnStart,
LifecycleKind::TurnEnd,
LifecycleKind::StepEnd,
LifecycleKind::GenerationEnd,
LifecycleKind::ToolCallEnd,
] {
let observed = lifecycle_transcript_source(kind, Some(&step), &live)
.snapshot()
.unwrap();
assert_eq!(observed.version(), Version::from_u64(9));
assert_eq!(observed.items()[0].to_plain_text(), "live");
}
}
#[tokio::test]
async fn step_freezes_transcript_tools_and_hook_snapshot_before_model() {
let (session, lease) = lease(vec![user("before")]);
let versions = Arc::new(Mutex::new(Vec::new()));
let mut builder = tool::RegistryBuilder::new();
builder
.register(hosted("web"))
.provider(Arc::new(ObservingProvider {
versions: Arc::clone(&versions),
}));
let tools = builder.build().unwrap();
let entered = Arc::new(Notify::new());
let release = Arc::new(Notify::new());
let model = FakeModel::starting(Start::Wait {
entered: Arc::clone(&entered),
release: Arc::clone(&release),
result: Ok(FakeEvents::ready(vec![
Ok(model::Event::ContentDelta {
id: "text".to_owned(),
kind: ContentKind::Text,
delta: "answer".to_owned(),
}),
Ok(model::Event::ContentFinal {
id: "text".to_owned(),
kind: ContentKind::Text,
content: "answer".to_owned(),
extensions: extensions(),
}),
Ok(model::Event::ContentDelta {
id: "thought".to_owned(),
kind: ContentKind::Thinking,
delta: "reason".to_owned(),
}),
Ok(model::Event::ContentFinal {
id: "thought".to_owned(),
kind: ContentKind::Thinking,
content: "reason".to_owned(),
extensions: extensions(),
}),
Ok(model::Event::Usage(TokenUsage::default())),
Ok(done()),
])),
});
let inbox = session.input_mailbox();
let cancellation = Token::new();
let running = advance(
&model,
&lease,
&inbox,
&tools,
RequestPlan::new(),
&cancellation,
);
let mutate = async {
entered.notified().await;
lease
.append_batch_if_version(vec![user("after")], Version::ZERO)
.unwrap();
release.notify_one();
};
let (result, ()) = tokio::join!(running, mutate);
let generation = expect_generation(result);
assert_eq!(versions.lock().unwrap().as_slice(), &[Version::ZERO]);
assert_eq!(
generation.snapshot().tools().transcript().version(),
Version::ZERO
);
assert_eq!(generation.snapshot().tools().transcript().items().len(), 1);
assert_eq!(generation.snapshot().model().id(), "fake");
assert_eq!(generation.snapshot().tools().exposure().len(), 1);
assert_eq!(generation.finish_reason(), "stop");
assert_eq!(generation.completion(), CompletionReason::Stop);
assert!(!generation.has_tool_calls());
assert!(
matches!(&generation.assistant().content()[0], Segment::Text { text, .. } if text == "answer")
);
assert!(
matches!(&generation.assistant().content()[1], Segment::Thinking { text, .. } if text == "reason")
);
assert!(
generation
.assistant()
.provider_data()
.is_some_and(<[_]>::is_empty)
);
let requests = model.requests.lock().unwrap();
assert_eq!(requests[0].messages().len(), 1);
assert_eq!(requests[0].extensions().len(), 1);
assert_eq!(lease.transcript_snapshot().unwrap().items().len(), 2);
}
#[tokio::test]
async fn generation_with_tool_calls_requires_tool_phase_and_stops_driver() {
let (_session, lease) = lease(Vec::new());
let call = ToolCall::new(
Id::new("call".to_owned()),
tool_call::Name::new("search".to_owned()),
Payload::copy_from_slice(br#"{"q":1}"#),
);
let model = FakeModel::scripted(vec![
Start::Ready(Ok(FakeEvents::ready(vec![
Ok(model::Event::ToolCallStart {
id: call.id().clone(),
name: call.name().clone(),
}),
Ok(model::Event::ToolCallArgumentsDelta {
id: call.id().clone(),
delta: Payload::copy_from_slice(br#"{"q":1}"#),
}),
Ok(model::Event::ToolCallFinal {
call,
extensions: extensions(),
}),
Ok(model::Event::Done {
finish_reason: "tools".to_owned(),
completion: CompletionReason::ToolCalls,
provider_data: Vec::new(),
}),
]))),
Start::Ready(Ok(FakeEvents::ready(vec![Ok(done())]))),
]);
let inbox = InputMailbox::new();
let tools = empty_tools();
let cancellation = Token::new();
let mut driver = Driver::new(&model, &lease, &inbox, &tools, &cancellation);
assert!(matches!(
driver.advance(RequestPlan::new()).await,
Err(Error::MissingToolPhase)
));
assert_eq!(model.calls(), 1);
assert!(expect_advance(driver.advance(RequestPlan::new()).await).is_none());
assert_eq!(
model.calls(),
1,
"a failed tool phase is terminal for this driver"
);
}
#[tokio::test]
async fn cancellation_before_step_prevents_model_invocation() {
let (_session, lease) = lease(Vec::new());
let model = FakeModel::ready(vec![Ok(done())]);
let cancellation = Token::new();
cancellation.cancel();
let inbox = InputMailbox::new();
inbox.push(InputBatch::new(InputTarget::NextTurn, vec![user("queued")]).unwrap());
let tools = empty_tools();
let result = advance(
&model,
&lease,
&inbox,
&tools,
RequestPlan::new(),
&cancellation,
)
.await;
assert!(matches!(result, Err(Error::Cancelled)));
assert_eq!(inbox.len(), 1);
assert_eq!(model.calls(), 0);
}
struct CancellingProvider;
impl Provider for CancellingProvider {
fn resolve<'a>(
&'a self,
_transcript: &'a Snapshot,
cancellation: Token,
) -> Pin<Box<dyn Future<Output = ProviderResolution> + Send + 'a>> {
cancellation.cancel();
Box::pin(ready(ProviderResolution::success(Vec::new())))
}
}
#[tokio::test]
async fn cancellation_during_step_freeze_prevents_model_invocation() {
let (_session, lease) = lease(Vec::new());
let model = FakeModel::ready(vec![Ok(done())]);
let cancellation = Token::new();
let mut builder = tool::RegistryBuilder::new();
builder.provider(Arc::new(CancellingProvider));
let tools = builder.build().unwrap();
let result = advance(
&model,
&lease,
&InputMailbox::new(),
&tools,
RequestPlan::new(),
&cancellation,
)
.await;
assert!(matches!(result, Err(Error::Cancelled)));
assert_eq!(model.calls(), 0);
}
struct PendingProvider {
starts: AtomicUsize,
started: Arc<Notify>,
drops: Arc<AtomicUsize>,
dropped: Arc<Notify>,
}
impl PendingProvider {
fn new() -> Arc<Self> {
Arc::new(Self {
starts: AtomicUsize::new(0),
started: Arc::new(Notify::new()),
drops: Arc::new(AtomicUsize::new(0)),
dropped: Arc::new(Notify::new()),
})
}
async fn wait_until_started(&self) {
loop {
let started = self.started.notified();
if self.starts.load(Ordering::Acquire) != 0 {
return;
}
started.await;
}
}
async fn wait_until_dropped(&self) {
loop {
let dropped = self.dropped.notified();
if self.drops.load(Ordering::Acquire) != 0 {
return;
}
dropped.await;
}
}
}
struct PendingProviderGuard {
drops: Arc<AtomicUsize>,
dropped: Arc<Notify>,
}
impl Drop for PendingProviderGuard {
fn drop(&mut self) {
self.drops.fetch_add(1, Ordering::Release);
self.dropped.notify_waiters();
}
}
impl Provider for PendingProvider {
fn resolve<'a>(
&'a self,
_transcript: &'a Snapshot,
_cancellation: Token,
) -> Pin<Box<dyn Future<Output = ProviderResolution> + Send + 'a>> {
Box::pin(async move {
let _guard = PendingProviderGuard {
drops: Arc::clone(&self.drops),
dropped: Arc::clone(&self.dropped),
};
self.starts.fetch_add(1, Ordering::Release);
self.started.notify_waiters();
std::future::pending().await
})
}
}
#[tokio::test]
async fn parent_cancellation_drops_a_pending_provider_and_returns_cancelled() {
let (session, lease) = lease(Vec::new());
let model = FakeModel::ready(vec![Ok(done())]);
let provider = PendingProvider::new();
let mut builder = tool::RegistryBuilder::new();
builder.provider(Arc::clone(&provider) as Arc<dyn Provider>);
let tools = builder.build().unwrap();
let inbox = InputMailbox::new();
let cancellation = Token::new();
let (_producer, source) = channel(&cancellation);
let driver = Driver::new(&model, &lease, &inbox, &tools, &cancellation);
let mut running = Box::pin(driver.run(RequestPlan::new(), &source));
tokio::select! {
biased;
terminal = &mut running => panic!("run ended before provider cancellation: {:?}", terminal.outcome()),
() = provider.wait_until_started() => {}
}
assert_eq!(provider.drops.load(Ordering::Acquire), 0);
cancellation.cancel();
let terminal = match running
.as_mut()
.poll(&mut Context::from_waker(std::task::Waker::noop()))
{
Poll::Ready(terminal) => terminal,
Poll::Pending => panic!("parent cancellation did not interrupt the pending provider"),
};
assert!(matches!(terminal, Terminal::Cancelled));
assert_eq!(provider.drops.load(Ordering::Acquire), 1);
assert_eq!(model.calls(), 0);
assert!(!super::event::testing::has_pending_strong_decision(&source));
drop(running);
let successor = release_and_read_successor(&session, &lease);
assert_eq!(successor.version(), Version::ZERO);
assert!(successor.items().is_empty());
}
#[tokio::test]
async fn run_parent_cancellation_releases_lease_and_agent_attachment_after_pending_provider() {
let provider = PendingProvider::new();
let mut builder = tool::RegistryBuilder::new();
builder.provider(Arc::clone(&provider) as Arc<dyn Provider>);
let model = Arc::new(FakeModel::ready(vec![Ok(done())]));
let agent = Agent::new(Config::new(
Arc::clone(&model),
builder.build().unwrap(),
hook::RegistryBuilder::new(),
))
.unwrap();
let session = Session::new("pending-provider".to_owned(), None);
let parent = Token::new();
let run = agent
.start(
session.clone(),
vec![Part::text("resolve tools".to_owned())],
parent.clone(),
)
.await
.unwrap();
provider.wait_until_started().await;
assert!(
session.try_acquire().is_none(),
"the Run still owns its lease"
);
parent.cancel();
provider.wait_until_dropped().await;
let terminal = run.wait().await;
assert_eq!(terminal.outcome(), Outcome::Cancelled);
assert!(terminal.failure().is_none());
assert_eq!(provider.drops.load(Ordering::Acquire), 1);
assert_eq!(model.calls(), 0);
drop(
session
.try_acquire()
.expect("Cancelled terminal is published after Session lease release"),
);
agent
.close()
.await
.expect("Cancelled terminal is published after Agent detachment");
}
#[tokio::test]
async fn cancellation_interrupts_pending_model_start() {
let (_session, lease) = lease(Vec::new());
let entered = Arc::new(Notify::new());
let model = FakeModel::starting(Start::Wait {
entered: Arc::clone(&entered),
release: Arc::new(Notify::new()),
result: Ok(FakeEvents::ready(vec![Ok(done())])),
});
let cancellation = Token::new();
let inbox = InputMailbox::new();
let tools = empty_tools();
let running = advance(
&model,
&lease,
&inbox,
&tools,
RequestPlan::new(),
&cancellation,
);
let cancel = async {
entered.notified().await;
cancellation.cancel();
};
let (result, ()) = tokio::join!(running, cancel);
assert!(matches!(result, Err(Error::Cancelled)));
assert_eq!(model.calls(), 1);
}
#[tokio::test]
async fn cancellation_interrupts_pending_model_stream() {
let (_session, lease) = lease(Vec::new());
let pending = Arc::new(Notify::new());
let model = FakeModel::starting(Start::Ready(Ok(FakeEvents::pending(Arc::clone(&pending)))));
let cancellation = Token::new();
let inbox = InputMailbox::new();
let tools = empty_tools();
let running = advance(
&model,
&lease,
&inbox,
&tools,
RequestPlan::new(),
&cancellation,
);
let cancel = async {
pending.notified().await;
cancellation.cancel();
};
let (result, ()) = tokio::join!(running, cancel);
assert!(matches!(result, Err(Error::Cancelled)));
}
#[tokio::test]
async fn ready_done_and_eof_cross_the_closing_cancel_gate() {
for events in [vec![Ok(done())], Vec::new()] {
let (_session, lease) = lease(Vec::new());
let cancellation = Token::new();
let model = FakeModel::starting(Start::Ready(Ok(FakeEvents::cancelling(
events,
1,
cancellation.clone(),
))));
let result = advance(
&model,
&lease,
&InputMailbox::new(),
&empty_tools(),
RequestPlan::new(),
&cancellation,
)
.await;
assert!(matches!(result, Err(Error::Cancelled)));
}
}
#[tokio::test]
async fn real_provider_and_grammar_errors_beat_same_poll_cancellation() {
let (_session, lease) = lease(Vec::new());
let cancellation = Token::new();
let provider_error = FakeModel::starting(Start::Ready(Ok(FakeEvents::cancelling(
vec![Err(FakeError("event"))],
1,
cancellation.clone(),
))));
let result = advance(
&provider_error,
&lease,
&InputMailbox::new(),
&empty_tools(),
RequestPlan::new(),
&cancellation,
)
.await;
assert!(matches!(result, Err(Error::Model(error)) if error.to_string() == "event"));
let cancellation = Token::new();
let grammar_error = FakeModel::starting(Start::Ready(Ok(FakeEvents::cancelling(
vec![Ok(done()), Ok(model::Event::Usage(TokenUsage::default()))],
2,
cancellation.clone(),
))));
let result = advance(
&grammar_error,
&lease,
&InputMailbox::new(),
&empty_tools(),
RequestPlan::new(),
&cancellation,
)
.await;
assert!(matches!(
result,
Err(Error::Stream(StreamError::EventAfterDone))
));
}
#[tokio::test]
async fn poisoned_transcript_stops_real_advance_without_side_effects() {
let (_session, lease) = lease(vec![user("healthy-session")]);
let transcript = Store::from_items(vec![user("poisoned-store")]);
let transcript_reader = transcript.snapshot_reader();
let inbox = InputMailbox::new();
inbox.push(InputBatch::new(InputTarget::NextStep, vec![user("step")]).unwrap());
inbox.push(InputBatch::new(InputTarget::NextTurn, vec![user("turn")]).unwrap());
let versions = Arc::new(Mutex::new(Vec::new()));
let mut builder = tool::RegistryBuilder::new();
builder.provider(Arc::new(ObservingProvider {
versions: Arc::clone(&versions),
}));
let tools = builder.build().unwrap();
let model = FakeModel::ready(vec![Ok(done())]);
let cancellation = Token::new();
let poisoner = transcript.clone();
assert!(thread::spawn(move || poison(&poisoner)).join().is_err());
let mut driver = Driver::with_transcript_reader(
&model,
&lease,
transcript_reader,
&inbox,
&tools,
&cancellation,
);
let result = driver.advance(RequestPlan::new()).await;
assert!(matches!(
result,
Err(Error::Transcript(store::Error::Poisoned))
));
assert_eq!(model.calls(), 0);
assert!(
versions.lock().unwrap().is_empty(),
"provider was not resolved"
);
assert_eq!(inbox.len(), 2, "failed snapshot must not claim Inbox");
}
struct CollisionProvider(tool::Registration);
impl Provider for CollisionProvider {
fn resolve<'a>(
&'a self,
_transcript: &'a Snapshot,
_cancellation: Token,
) -> Pin<Box<dyn Future<Output = ProviderResolution> + Send + 'a>> {
Box::pin(ready(ProviderResolution::success(vec![self.0.clone()])))
}
}
#[tokio::test]
async fn preparation_failures_stop_before_model() {
let (_session, lease) = lease(Vec::new());
let model = FakeModel::ready(vec![Ok(done())]);
let cancellation = Token::new();
let duplicate = hosted("duplicate");
let mut builder = tool::RegistryBuilder::new();
builder
.register(duplicate.clone())
.provider(Arc::new(CollisionProvider(duplicate)));
let result = advance(
&model,
&lease,
&InputMailbox::new(),
&builder.build().unwrap(),
RequestPlan::new(),
&cancellation,
)
.await;
assert!(matches!(
result,
Err(Error::Tools(StepResolutionError::Collision(name)))
if name.canonical() == "duplicate"
));
let mut invalid = RequestPlan::new();
invalid.push_system_instruction(Part::text(String::new()));
let result = advance(
&model,
&lease,
&InputMailbox::new(),
&empty_tools(),
invalid,
&cancellation,
)
.await;
assert!(matches!(result, Err(Error::Request(_))));
assert_eq!(model.calls(), 0);
let snapshot = Snapshot::new(Version::ZERO, Vec::new());
let resolved = empty_tools()
.resolve_step_without_providers(snapshot, &cancellation, &ExposeAll)
.unwrap();
assert!(resolved.provider_errors().is_empty());
}
#[tokio::test]
async fn model_and_stream_failures_remain_distinct() {
let (_session, lease) = lease(Vec::new());
let inbox = InputMailbox::new();
let tools = empty_tools();
let cancellation = Token::new();
let start_error = FakeModel::starting(Start::Ready(Err(FakeError("start"))));
let result = advance(
&start_error,
&lease,
&inbox,
&tools,
RequestPlan::new(),
&cancellation,
)
.await;
assert!(matches!(result, Err(Error::Model(error)) if error.to_string() == "start"));
let event_error = FakeModel::ready(vec![Err(FakeError("event"))]);
let result = advance(
&event_error,
&lease,
&inbox,
&tools,
RequestPlan::new(),
&cancellation,
)
.await;
assert!(matches!(result, Err(Error::Model(error)) if error.to_string() == "event"));
let invalid = FakeModel::ready(vec![
Ok(done()),
Ok(model::Event::Usage(TokenUsage::default())),
]);
let result = advance(
&invalid,
&lease,
&inbox,
&tools,
RequestPlan::new(),
&cancellation,
)
.await;
assert!(matches!(
result,
Err(Error::Stream(StreamError::EventAfterDone))
));
let missing = FakeModel::ready(Vec::new());
let result = advance(
&missing,
&lease,
&inbox,
&tools,
RequestPlan::new(),
&cancellation,
)
.await;
assert!(matches!(
result,
Err(Error::Stream(StreamError::MissingTerminal))
));
}
struct SilentEmitter;
impl Emitter for SilentEmitter {
fn emit_audit(&self, _event: audit::Event) {}
fn emit_error(&self, _error: Arc<dyn error::Error + Send + Sync>) {}
}
struct RecordedReservation {
generation: u64,
published: Arc<Mutex<Vec<u64>>>,
}
impl TranscriptReservation for RecordedReservation {
fn publish(self: Box<Self>) {
self.published.lock().unwrap().push(self.generation);
}
fn abort(self: Box<Self>) {}
}
struct GenerationRecordingHooks {
reserved: Arc<Mutex<Vec<u64>>>,
published: Arc<Mutex<Vec<u64>>>,
}
impl TranscriptHooks for GenerationRecordingHooks {
fn reserve(
&self,
scope: &ReadScope<'_>,
_mutation: &TranscriptMutation,
) -> HookResult<Box<dyn TranscriptReservation>> {
let generation = scope.coord().generation();
self.reserved.lock().unwrap().push(generation);
Ok(Box::new(RecordedReservation {
generation,
published: Arc::clone(&self.published),
}))
}
}
struct GenerationRecordingToolHooks {
before_results: Arc<Mutex<Vec<u64>>>,
}
impl ToolHooks for GenerationRecordingToolHooks {
fn before_tool_call(&self, _scope: &ToolCallScope<'_>) -> HookResult<ToolCallDecision> {
Ok(ToolCallDecision::Continue)
}
fn before_tool_result(&self, scope: &ToolResultScope<'_>) -> HookResult<ToolResultDecision> {
self.before_results
.lock()
.unwrap()
.push(scope.coord().generation());
Ok(ToolResultDecision::Continue)
}
fn after_tool_commit(&self, _scope: &AfterToolCommitScope<'_>) -> Vec<Item> {
Vec::new()
}
}
struct AfterToolHooks {
label: &'static str,
calls: Arc<Mutex<Vec<String>>>,
emit_facts: bool,
replace_staged: bool,
return_item: bool,
settle_named_calls: bool,
active: Option<Arc<AtomicUsize>>,
}
impl ToolHooks for AfterToolHooks {
fn before_tool_call(&self, scope: &ToolCallScope<'_>) -> HookResult<ToolCallDecision> {
if self.settle_named_calls {
match scope.call().name().as_str() {
"blocked" => return Ok(ToolCallDecision::Block("policy".to_owned())),
"short" => {
return Ok(ToolCallDecision::ShortCircuit(ToolResult::success_text(
"cached".to_owned(),
)));
}
_ => {}
}
}
Ok(ToolCallDecision::Continue)
}
fn before_tool_result(&self, _scope: &ToolResultScope<'_>) -> HookResult<ToolResultDecision> {
Ok(ToolResultDecision::Continue)
}
fn after_tool_commit(&self, scope: &AfterToolCommitScope<'_>) -> Vec<Item> {
let _active = self.active.as_ref().map(|active| {
active.fetch_add(1, Ordering::SeqCst);
ActiveCallback(Arc::clone(active))
});
let identity = format!("{}:{}", self.label, scope.coord().call_id());
self.calls.lock().unwrap().push(identity.clone());
if self.emit_facts {
scope.emit_audit(audit::Event::new(format!("{identity}:audit-1"), None));
scope.emit_error(Arc::new(FakeError("after-error")));
scope.emit_audit(audit::Event::new(format!("{identity}:audit-2"), None));
}
if self.replace_staged {
scope
.staged()
.replace(vec![system(&format!("{identity}:staged"))])
.unwrap();
}
if self.return_item {
vec![system(&format!("{identity}:returned"))]
} else {
Vec::new()
}
}
}
struct InvalidAfterFactHooks {
continued: Arc<AtomicUsize>,
}
impl ToolHooks for InvalidAfterFactHooks {
fn before_tool_call(&self, _scope: &ToolCallScope<'_>) -> HookResult<ToolCallDecision> {
Ok(ToolCallDecision::Continue)
}
fn before_tool_result(&self, _scope: &ToolResultScope<'_>) -> HookResult<ToolResultDecision> {
Ok(ToolResultDecision::Continue)
}
fn after_tool_commit(&self, scope: &AfterToolCommitScope<'_>) -> Vec<Item> {
scope.emit_audit(audit::Event::new(String::new(), None));
self.continued.fetch_add(1, Ordering::SeqCst);
Vec::new()
}
}
struct PerEventPauseHooks;
impl ToolHooks for PerEventPauseHooks {
fn before_tool_call(&self, _scope: &ToolCallScope<'_>) -> HookResult<ToolCallDecision> {
Ok(ToolCallDecision::Continue)
}
fn before_tool_result(&self, _scope: &ToolResultScope<'_>) -> HookResult<ToolResultDecision> {
Ok(ToolResultDecision::Continue)
}
fn after_tool_commit(&self, scope: &AfterToolCommitScope<'_>) -> Vec<Item> {
scope.emit_audit(audit::Event::new("pause-1".to_owned(), None));
scope.emit_audit(audit::Event::new("pause-2".to_owned(), None));
Vec::new()
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct AfterWriteSnapshotValue(u8);
impl SnapshotValue for AfterWriteSnapshotValue {}
struct WriteAfterEmitHooks {
key: session_state::Key<AfterWriteSnapshotValue>,
writes: Arc<AtomicUsize>,
}
impl ToolHooks for WriteAfterEmitHooks {
fn before_tool_call(&self, _scope: &ToolCallScope<'_>) -> HookResult<ToolCallDecision> {
Ok(ToolCallDecision::Continue)
}
fn before_tool_result(&self, _scope: &ToolResultScope<'_>) -> HookResult<ToolResultDecision> {
Ok(ToolResultDecision::Continue)
}
fn after_tool_commit(&self, scope: &AfterToolCommitScope<'_>) -> Vec<Item> {
scope.emit_audit(audit::Event::new("before-session-write".to_owned(), None));
scope
.session_state_writer()
.expect("registration declared Session state")
.set(&self.key, &AfterWriteSnapshotValue(7))
.unwrap();
self.writes.fetch_add(1, Ordering::SeqCst);
Vec::new()
}
}
#[derive(Clone, Copy, Debug)]
enum CatchAbortMode {
Accept,
Reject,
DropDelivery,
Invalid,
ChannelClose,
}
#[derive(Debug, Eq, PartialEq)]
struct CatchAbortObservation {
caught_abort: bool,
writes_succeeded: [bool; 3],
}
struct CatchAbortWriteHooks {
mode: CatchAbortMode,
set_key: session_state::Key<AfterWriteSnapshotValue>,
load_or_store_key: session_state::Key<AfterWriteSnapshotValue>,
delete_key: session_state::Key<AfterWriteSnapshotValue>,
observation: Arc<Mutex<Option<CatchAbortObservation>>>,
completed: Arc<Notify>,
}
impl ToolHooks for CatchAbortWriteHooks {
fn before_tool_call(&self, _scope: &ToolCallScope<'_>) -> HookResult<ToolCallDecision> {
Ok(ToolCallDecision::Continue)
}
fn before_tool_result(&self, _scope: &ToolResultScope<'_>) -> HookResult<ToolResultDecision> {
Ok(ToolResultDecision::Continue)
}
fn after_tool_commit(&self, scope: &AfterToolCommitScope<'_>) -> Vec<Item> {
let caught_abort = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
let kind = if matches!(self.mode, CatchAbortMode::Invalid) {
String::new()
} else {
"catch-abort".to_owned()
};
scope.emit_audit(audit::Event::new(kind, None));
}))
.is_err();
let writer = scope
.session_state_writer()
.expect("registration declared Session state");
let writes_succeeded = [
writer
.set(&self.set_key, &AfterWriteSnapshotValue(1))
.is_ok(),
writer
.load_or_store(&self.load_or_store_key, &AfterWriteSnapshotValue(2))
.is_ok(),
writer.delete(&self.delete_key).is_ok(),
];
*self.observation.lock().unwrap() = Some(CatchAbortObservation {
caught_abort,
writes_succeeded,
});
self.completed.notify_one();
Vec::new()
}
}
struct ActiveCallback(Arc<AtomicUsize>);
impl Drop for ActiveCallback {
fn drop(&mut self) {
self.0.fetch_sub(1, Ordering::SeqCst);
}
}
#[derive(Debug, Eq, PartialEq)]
struct DropOwnerAbortObservation {
caught_abort: bool,
write_errors: [WriteError; 3],
}
struct DropOwnerCatchAbortHooks {
set_key: session_state::Key<AfterWriteSnapshotValue>,
load_or_store_key: session_state::Key<AfterWriteSnapshotValue>,
delete_key: session_state::Key<AfterWriteSnapshotValue>,
observation: Arc<Mutex<Option<DropOwnerAbortObservation>>>,
active: Arc<AtomicUsize>,
}
impl ToolHooks for DropOwnerCatchAbortHooks {
fn before_tool_call(&self, _scope: &ToolCallScope<'_>) -> HookResult<ToolCallDecision> {
Ok(ToolCallDecision::Continue)
}
fn before_tool_result(&self, _scope: &ToolResultScope<'_>) -> HookResult<ToolResultDecision> {
Ok(ToolResultDecision::Continue)
}
fn after_tool_commit(&self, scope: &AfterToolCommitScope<'_>) -> Vec<Item> {
self.active.fetch_add(1, Ordering::SeqCst);
let _active = ActiveCallback(Arc::clone(&self.active));
let caught_abort = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
scope.emit_audit(audit::Event::new("pending-owner-drop".to_owned(), None));
}))
.is_err();
let writer = scope
.session_state_writer()
.expect("registration declared Session state");
let write_errors = [
writer
.set(&self.set_key, &AfterWriteSnapshotValue(1))
.expect_err("owner Drop revoked set"),
writer
.load_or_store(&self.load_or_store_key, &AfterWriteSnapshotValue(2))
.expect_err("owner Drop revoked load_or_store"),
writer
.delete(&self.delete_key)
.expect_err("owner Drop revoked delete"),
];
*self.observation.lock().unwrap() = Some(DropOwnerAbortObservation {
caught_abort,
write_errors,
});
Vec::new()
}
}
struct PanicAfterEmitHooks {
active: Arc<AtomicUsize>,
}
impl ToolHooks for PanicAfterEmitHooks {
fn before_tool_call(&self, _scope: &ToolCallScope<'_>) -> HookResult<ToolCallDecision> {
Ok(ToolCallDecision::Continue)
}
fn before_tool_result(&self, _scope: &ToolResultScope<'_>) -> HookResult<ToolResultDecision> {
Ok(ToolResultDecision::Continue)
}
fn after_tool_commit(&self, scope: &AfterToolCommitScope<'_>) -> Vec<Item> {
self.active.fetch_add(1, Ordering::SeqCst);
let _active = ActiveCallback(Arc::clone(&self.active));
scope.emit_audit(audit::Event::new("accepted-before-panic".to_owned(), None));
scope
.staged()
.replace(vec![system("must-not-commit")])
.unwrap();
panic!("real after-tool panic");
}
}
struct AuditingExecutor {
executions: Arc<AtomicUsize>,
}
impl Executor for AuditingExecutor {
type Output = String;
async fn execute(
&self,
invocation: &Invocation,
call: &ToolCall,
_cancellation: Token,
) -> Result<Self::Output, ExecutorError> {
self.executions.fetch_add(1, Ordering::SeqCst);
invocation
.emitter()
.emit_audit(audit::Event::new("generation-audit".to_owned(), None))
.await
.map_err(|error| ExecutorError::new(error.to_string()))?;
Ok(call.id().as_str().to_owned())
}
}
#[derive(Clone, Copy)]
enum ExecutorMode {
Success,
OrdinaryError,
}
struct LoopExecutor {
mode: ExecutorMode,
executions: Arc<AtomicUsize>,
accepted_calls: Arc<AtomicUsize>,
acceptance_at_execute: Arc<Mutex<Vec<usize>>>,
}
impl Executor for LoopExecutor {
type Output = String;
async fn execute(
&self,
_invocation: &Invocation,
call: &ToolCall,
_cancellation: Token,
) -> Result<Self::Output, ExecutorError> {
self.executions.fetch_add(1, Ordering::SeqCst);
self.acceptance_at_execute
.lock()
.unwrap()
.push(self.accepted_calls.load(Ordering::SeqCst));
if call.name().as_str() == "first" {
tokio::task::yield_now().await;
}
match self.mode {
ExecutorMode::Success => Ok(call.id().as_str().to_owned()),
ExecutorMode::OrdinaryError => Err(ExecutorError::new("ordinary".to_owned())),
}
}
}
#[derive(Default)]
struct PendingExecutorCounts {
active: AtomicUsize,
dropped: AtomicUsize,
}
struct PendingExecutorOwner(Arc<PendingExecutorCounts>);
impl Drop for PendingExecutorOwner {
fn drop(&mut self) {
self.0.active.fetch_sub(1, Ordering::SeqCst);
self.0.dropped.fetch_add(1, Ordering::SeqCst);
}
}
struct PendingExecutor {
entered: Arc<Notify>,
counts: Arc<PendingExecutorCounts>,
}
impl Executor for PendingExecutor {
type Output = String;
async fn execute(
&self,
_invocation: &Invocation,
_call: &ToolCall,
_cancellation: Token,
) -> Result<Self::Output, ExecutorError> {
self.counts.active.fetch_add(1, Ordering::SeqCst);
let _owner = PendingExecutorOwner(Arc::clone(&self.counts));
self.entered.notify_one();
std::future::pending().await
}
}
struct StatefulProvider {
resolves: Arc<AtomicUsize>,
registration: tool::Registration,
}
impl Provider for StatefulProvider {
fn resolve<'a>(
&'a self,
_transcript: &'a Snapshot,
_cancellation: Token,
) -> Pin<Box<dyn Future<Output = ProviderResolution> + Send + 'a>> {
self.resolves.fetch_add(1, Ordering::SeqCst);
Box::pin(ready(ProviderResolution::success(vec![
self.registration.clone(),
])))
}
}
fn client_tool(
name: &str,
mode: ExecutorMode,
fatal_mapping: bool,
executions: &Arc<AtomicUsize>,
accepted_calls: &Arc<AtomicUsize>,
acceptance_at_execute: &Arc<Mutex<Vec<usize>>>,
) -> tool::Registration {
let descriptor = Descriptor::new(
tool::Name::local(name.to_owned()).unwrap(),
"loop tool".to_owned(),
Payload::copy_from_slice(b"{}"),
);
let executor = Arc::new(LoopExecutor {
mode,
executions: Arc::clone(executions),
accepted_calls: Arc::clone(accepted_calls),
acceptance_at_execute: Arc::clone(acceptance_at_execute),
});
tool::Registration::client(
descriptor,
executor,
Arc::new(move |output: String| {
if fatal_mapping {
Err(MappingError::Protocol("fatal".to_owned()))
} else {
Ok(ToolResult::success_text(output))
}
}),
)
}
fn executable_tools(
names: &[&str],
mode: ExecutorMode,
fatal_mapping: bool,
executions: &Arc<AtomicUsize>,
accepted_calls: &Arc<AtomicUsize>,
acceptance_at_execute: &Arc<Mutex<Vec<usize>>>,
) -> tool::Registry {
let mut builder = tool::RegistryBuilder::new();
for name in names {
builder.register(client_tool(
name,
mode,
fatal_mapping,
executions,
accepted_calls,
acceptance_at_execute,
));
}
builder.build().unwrap()
}
fn pending_executable_tool(
entered: &Arc<Notify>,
counts: &Arc<PendingExecutorCounts>,
) -> tool::Registry {
let descriptor = Descriptor::new(
tool::Name::local("tool".to_owned()).unwrap(),
"pending loop tool".to_owned(),
Payload::copy_from_slice(b"{}"),
);
let registration = tool::Registration::client(
descriptor,
Arc::new(PendingExecutor {
entered: Arc::clone(entered),
counts: Arc::clone(counts),
}),
Arc::new(|output: String| Ok(ToolResult::success_text(output))),
);
let mut builder = tool::RegistryBuilder::new();
builder.register(registration);
builder.build().unwrap()
}
fn model_tool_call(id: &str, name: &str) -> ToolCall {
ToolCall::new(
Id::new(id.to_owned()),
tool_call::Name::new(name.to_owned()),
Payload::copy_from_slice(b"{}"),
)
}
fn tool_generation(calls: &[ToolCall]) -> Vec<Result<model::Event, FakeError>> {
let mut events = vec![
Ok(model::Event::ContentDelta {
id: "text".to_owned(),
kind: ContentKind::Text,
delta: "before tools".to_owned(),
}),
Ok(model::Event::ContentFinal {
id: "text".to_owned(),
kind: ContentKind::Text,
content: "before tools".to_owned(),
extensions: extensions(),
}),
];
for call in calls {
events.push(Ok(model::Event::ToolCallStart {
id: call.id().clone(),
name: call.name().clone(),
}));
events.push(Ok(model::Event::ToolCallArgumentsDelta {
id: call.id().clone(),
delta: Payload::copy_from_slice(b"{}"),
}));
events.push(Ok(model::Event::ToolCallFinal {
call: call.clone(),
extensions: extensions(),
}));
}
events.push(Ok(model::Event::Done {
finish_reason: "tools".to_owned(),
completion: CompletionReason::ToolCalls,
provider_data: Vec::new(),
}));
events
}
fn generation_coord() -> Coord {
let one = NonZeroU64::MIN;
Coord::at_generation(
"session".to_owned(),
"run".to_owned(),
StepIndex::new(one, one, one).unwrap(),
one,
)
.unwrap()
}
#[test]
fn tool_call_coordinates_fail_closed_for_external_empty_values() {
let generation = generation_coord();
assert!(matches!(
tool_call_coord(&generation, &model_tool_call("", "tool"), "tool"),
Err(CoordError::EmptyCallId)
));
assert!(matches!(
tool_call_coord(&generation, &model_tool_call("call", "tool"), ""),
Err(CoordError::EmptyToolName)
));
}
fn empty_hooks() -> hook::RuntimeRegistry {
hook::RegistryBuilder::new().build_runtime().unwrap()
}
fn event_identity(delivery: &Delivery) -> (DecisionPoint, String) {
let seam = delivery.decision_point().expect("tool event is strong");
let id = match delivery.event().kind() {
Kind::ToolCall => delivery
.event()
.tool_call()
.expect("tool call payload")
.id()
.as_str()
.to_owned(),
Kind::ToolResult => delivery
.event()
.tool_result()
.expect("tool result payload")
.call_id()
.as_str()
.to_owned(),
kind => panic!("unexpected event kind: {kind:?}"),
};
(seam, id)
}
async fn expect_kinds(stream: &mut super::event::Stream, expected: &[Kind]) {
for kind in expected {
let delivery = stream.recv().await.expect("expected outward event");
assert_eq!(delivery.event().kind(), *kind);
}
}
async fn expect_generation_prelude(stream: &mut super::event::Stream, opens_turn: bool) {
let mut expected = Vec::new();
if opens_turn {
expected.push(Kind::TurnStart);
}
expected.extend([Kind::StepStart, Kind::TextDelta, Kind::Text]);
expect_kinds(stream, &expected).await;
}
async fn accept_tool_exchange(stream: &mut super::event::Stream) -> (Coord, Coord, Coord) {
let call = stream.recv().await.expect("tool call delivery");
assert_eq!(event_identity(&call).0, DecisionPoint::ToolCall);
let call_coord = call.event().coord().expect("tool call coord").clone();
call.accept().unwrap();
let audit = stream.recv().await.expect("tool audit delivery");
assert_eq!(audit.event().kind(), Kind::Audit);
let audit_coord = audit.event().coord().expect("tool audit coord").clone();
let result = stream.recv().await.expect("tool result delivery");
assert_eq!(event_identity(&result).0, DecisionPoint::ToolResult);
let result_coord = result.event().coord().expect("tool result coord").clone();
result.accept().unwrap();
(call_coord, audit_coord, result_coord)
}
async fn run_test_owned_emission_case(fatal_mapping: bool) {
let (session, lease) = lease(Vec::new());
lease.admit().unwrap();
let call = model_tool_call("dynamic-call", "dynamic");
let model = FakeModel::scripted(vec![
Start::Ready(Ok(FakeEvents::ready(tool_generation(&[call])))),
Start::Ready(Ok(FakeEvents::ready(vec![Ok(done())]))),
]);
let resolves = Arc::new(AtomicUsize::new(0));
let executions = Arc::new(AtomicUsize::new(0));
let accepted_calls = Arc::new(AtomicUsize::new(0));
let acceptance_at_execute = Arc::new(Mutex::new(Vec::new()));
let (active_tx, active_rx) = std::sync::mpsc::channel();
let active_rx = Arc::new(Mutex::new(active_rx));
let mapper_active = Arc::clone(&active_rx);
let registration = tool::Registration::client(
Descriptor::new(
tool::Name::local("dynamic".to_owned()).unwrap(),
"stateful dynamic tool".to_owned(),
Payload::copy_from_slice(b"{}"),
),
Arc::new(LoopExecutor {
mode: ExecutorMode::Success,
executions: Arc::clone(&executions),
accepted_calls: Arc::clone(&accepted_calls),
acceptance_at_execute: Arc::clone(&acceptance_at_execute),
}),
Arc::new(move |output: String| {
mapper_active
.lock()
.unwrap()
.recv_timeout(std::time::Duration::from_secs(2))
.map_err(|_| MappingError::Protocol("emission never became active".to_owned()))?;
if fatal_mapping {
Err(MappingError::Protocol(
"fatal while emission active".to_owned(),
))
} else {
Ok(ToolResult::success_text(output))
}
}),
);
let mut builder = tool::RegistryBuilder::new();
builder.provider(Arc::new(StatefulProvider {
resolves: Arc::clone(&resolves),
registration,
}));
let tools = builder.build().unwrap();
let hooks = empty_hooks();
let reservations = ReservationResources::new(0, lease.state().reader());
let identity = generation_coord();
let emitter = SilentEmitter;
let controller = TranscriptController::new(&lease, &hooks, &reservations, &identity, &emitter);
let cancellation = Token::new();
let (producer, mut source) = channel(&cancellation);
let mut stream = source.take_stream().unwrap();
let (test_emitter, test_gate) = tool_emitter(&producer, identity.clone());
let emitter_probe = super::event::testing::emitter_probe(&test_emitter);
let emission_pause = super::event::testing::EmitPauseProbe::new(active_tx);
let owner_pause = emission_pause.clone();
let (start_tx, start_rx) = std::sync::mpsc::channel();
let (late_tx, late_rx) = std::sync::mpsc::channel();
let (late_result_tx, late_result_rx) = std::sync::mpsc::channel();
let emission_owner = thread::spawn(move || {
let runtime = tokio::runtime::Builder::new_current_thread()
.build()
.expect("test emission runtime");
start_rx.recv().expect("test starts emission");
runtime
.block_on(test_emitter.emit_audit(audit::Event::new("first".to_owned(), None)))
.expect("the open window delivers the first emission at once");
owner_pause.arm(&test_emitter);
let _held_result =
runtime.block_on(test_emitter.emit_audit(audit::Event::new("second".to_owned(), None)));
late_rx
.recv()
.expect("test requests the post-advance probe");
let result = runtime
.block_on(test_emitter.emit_audit(audit::Event::new("after-advance".to_owned(), None)));
late_result_tx
.send(result)
.expect("test receives the post-advance result");
});
let facts = FactBuffer::default();
let outward = Outward::new(&producer, &facts);
let phase = ToolPhase::new(
outward,
&hooks,
&[],
&controller,
&identity,
Some("session".to_owned()),
)
.with_test_emitter_gate(test_gate);
let inbox = InputMailbox::new();
let driver = Driver::new(&model, &lease, &inbox, &tools, &cancellation).with_tool_phase(phase);
let mut running = Box::pin(driver.run(RequestPlan::new(), &source));
tokio::select! {
() = expect_generation_prelude(&mut stream, true) => {}
terminal = &mut running => panic!("run returned before ToolCall acceptance: {:?}", terminal.outcome()),
}
let tool_call = tokio::select! {
delivery = stream.recv() => delivery.expect("tool call delivery"),
terminal = &mut running => panic!("run returned before ToolCall acceptance: {:?}", terminal.outcome()),
};
assert_eq!(event_identity(&tool_call).0, DecisionPoint::ToolCall);
accepted_calls.fetch_add(1, Ordering::SeqCst);
tool_call.accept().unwrap();
start_tx
.send(())
.expect("start the independent emission owner");
assert_pending_once(
running.as_mut(),
"advance must wait for the still-active independent emission",
);
assert_eq!(
executions.load(Ordering::SeqCst),
1,
"executor already returned"
);
assert_eq!(resolves.load(Ordering::SeqCst), 1, "provider resolves once");
assert_eq!(
lease.transcript_snapshot().unwrap().version(),
Version::ZERO,
"close/drain precedes commit"
);
assert_eq!(
emitter_probe.state(),
(true, 1),
"the window is closed to new emissions while the held one drains"
);
emission_pause.resume();
if fatal_mapping {
cancellation.cancel();
} else {
for expected in ["first", "second"] {
let delivery = stream.recv().await.expect("test-owned audit delivery");
assert_eq!(delivery.event().kind(), Kind::Audit);
assert_eq!(delivery.event().as_audit().unwrap().kind(), expected);
}
let tool_result = tokio::select! {
delivery = stream.recv() => delivery.expect("tool result delivery"),
terminal = &mut running => panic!("run returned before ToolResult acceptance: {:?}", terminal.outcome()),
};
assert_eq!(event_identity(&tool_result).0, DecisionPoint::ToolResult);
tool_result.accept().unwrap();
}
let terminal = tokio::time::timeout(std::time::Duration::from_secs(2), &mut running)
.await
.expect("close/drain eventually completes");
if fatal_mapping {
match terminal.failure() {
Some(failure @ Error::Route(error)) => {
assert!(matches!(
error.execution().execution(),
ExecutionError::Mapping(MappingError::Protocol(message))
if message == "fatal while emission active"
));
assert_eq!(
failure.to_string(),
concat!(
"tool batch routing failed: executing the batch failed: ",
"tool output mapping failed: tool mapping protocol failure: ",
"fatal while emission active"
)
);
assert_eq!(
failure
.cause()
.expect("a routed batch failure retains the batch failure")
.to_string(),
concat!(
"prepared tool batch failed at source index 0: ",
"tool output mapping failed: tool mapping protocol failure: ",
"fatal while emission active"
)
);
}
_ => panic!("late cancellation displaced the original mapping failure"),
}
assert_eq!(
lease.transcript_snapshot().unwrap().version(),
Version::ZERO
);
} else {
assert!(matches!(terminal, Terminal::Completed));
assert_eq!(model.calls(), 2);
assert_eq!(
lease.transcript_snapshot().unwrap().version(),
Version::from_u64(2)
);
}
assert_eq!(
emitter_probe.state(),
(true, 0),
"Driver::run returns only after every tool emitter is closed and idle"
);
assert!(!super::event::testing::has_pending_strong_decision(&source));
late_tx.send(()).expect("probe emitter after run");
assert_eq!(
late_result_rx
.recv_timeout(std::time::Duration::from_secs(2))
.expect("post-advance emission resolves"),
Err(ToolEmitError::Closed),
"run closes the emission window on every exit"
);
emission_owner
.join()
.expect("independent emission owner exits");
let successor = release_and_read_successor(&session, &lease);
assert_eq!(
successor.version(),
if fatal_mapping {
Version::ZERO
} else {
Version::from_u64(2)
}
);
}
#[tokio::test]
async fn step_resolves_provider_once_and_drains_test_owned_emission_on_success_and_error() {
run_test_owned_emission_case(false).await;
run_test_owned_emission_case(true).await;
}
#[tokio::test]
async fn accepted_calls_precede_execution_and_results_preserve_source_order_after_atomic_commit() {
let (session, lease) = lease(Vec::new());
lease.admit().unwrap();
let calls = [
model_tool_call("call-1", "first"),
model_tool_call("call-2", "second"),
];
let model = FakeModel::scripted(vec![
Start::Ready(Ok(FakeEvents::ready(tool_generation(&calls)))),
Start::Ready(Ok(FakeEvents::ready(vec![Ok(done())]))),
]);
let executions = Arc::new(AtomicUsize::new(0));
let accepted_calls = Arc::new(AtomicUsize::new(0));
let acceptance_at_execute = Arc::new(Mutex::new(Vec::new()));
let tools = executable_tools(
&["first", "second"],
ExecutorMode::Success,
false,
&executions,
&accepted_calls,
&acceptance_at_execute,
);
let hooks = empty_hooks();
let reservations = ReservationResources::new(0, lease.state().reader());
let coord = generation_coord();
let emitter = SilentEmitter;
let controller = TranscriptController::new(&lease, &hooks, &reservations, &coord, &emitter);
let cancellation = Token::new();
let (producer, mut source) = channel(&cancellation);
let mut stream = source.take_stream().unwrap();
let facts = FactBuffer::default();
let outward = Outward::new(&producer, &facts);
let phase = ToolPhase::new(
outward,
&hooks,
&[],
&controller,
&coord,
Some("session".to_owned()),
);
let inbox = InputMailbox::new();
let driver = Driver::new(&model, &lease, &inbox, &tools, &cancellation).with_tool_phase(phase);
let observed = Arc::new(Mutex::new(Vec::new()));
let consumer_log = Arc::clone(&observed);
let consumer_accepted = Arc::clone(&accepted_calls);
let consumer = async {
expect_generation_prelude(&mut stream, true).await;
for _ in 0..4 {
let delivery = stream.recv().await.expect("expected tool delivery");
let identity = event_identity(&delivery);
if identity.0 == DecisionPoint::ToolCall {
consumer_accepted.fetch_add(1, Ordering::SeqCst);
}
consumer_log.lock().unwrap().push(identity);
delivery.accept().unwrap();
}
};
let (terminal, ()) = tokio::join!(driver.run(RequestPlan::new(), &source), consumer);
assert!(matches!(terminal, Terminal::Completed));
assert_eq!(executions.load(Ordering::SeqCst), 2);
assert_eq!(
*acceptance_at_execute.lock().unwrap(),
[2, 2],
"tool execution cannot begin until every ToolCall was accepted"
);
assert_eq!(
*observed.lock().unwrap(),
[
(DecisionPoint::ToolCall, "call-1".to_owned()),
(DecisionPoint::ToolCall, "call-2".to_owned()),
(DecisionPoint::ToolResult, "call-1".to_owned()),
(DecisionPoint::ToolResult, "call-2".to_owned()),
]
);
let snapshot = lease.transcript_snapshot().unwrap();
assert_eq!(snapshot.version(), Version::from_u64(3));
assert!(matches!(snapshot.items()[0], Item::Assistant(_)));
assert!(
matches!(&snapshot.items()[1], Item::ToolResult(item) if item.call_id().as_str() == "call-1")
);
assert!(
matches!(&snapshot.items()[2], Item::ToolResult(item) if item.call_id().as_str() == "call-2")
);
assert_eq!(model.calls(), 2, "accepted tools continue the active Turn");
assert!(!super::event::testing::has_pending_strong_decision(&source));
let successor = release_and_read_successor(&session, &lease);
assert_eq!(successor.version(), Version::from_u64(3));
assert_eq!(successor.items().len(), 3);
assert!(matches!(successor.items()[0], Item::Assistant(_)));
assert!(
matches!(&successor.items()[1], Item::ToolResult(item) if item.call_id().as_str() == "call-1")
);
assert!(
matches!(&successor.items()[2], Item::ToolResult(item) if item.call_id().as_str() == "call-2")
);
}
#[tokio::test]
async fn consecutive_tool_generations_use_distinct_monotonic_runtime_coords() {
let (_session, lease) = lease(Vec::new());
lease.admit().unwrap();
let first_call = model_tool_call("call-1", "tool");
let second_call = model_tool_call("call-2", "tool");
let model = FakeModel::scripted(vec![
Start::Ready(Ok(FakeEvents::ready(tool_generation(&[first_call])))),
Start::Ready(Ok(FakeEvents::ready(tool_generation(&[second_call])))),
]);
let executions = Arc::new(AtomicUsize::new(0));
let registration = tool::Registration::client(
Descriptor::new(
tool::Name::local("tool".to_owned()).unwrap(),
"coordinate recorder".to_owned(),
Payload::copy_from_slice(b"{}"),
),
Arc::new(AuditingExecutor {
executions: Arc::clone(&executions),
}),
Arc::new(|output: String| Ok(ToolResult::success_text(output))),
);
let mut tool_builder = tool::RegistryBuilder::new();
tool_builder.register(registration);
let tools = tool_builder.build().unwrap();
let reserved_generations = Arc::new(Mutex::new(Vec::new()));
let published_generations = Arc::new(Mutex::new(Vec::new()));
let before_result_generations = Arc::new(Mutex::new(Vec::new()));
let mut hook_builder = hook::RegistryBuilder::new();
hook_builder.register(
hook::Registration::new("generation-recorder".to_owned())
.tool(Arc::new(GenerationRecordingToolHooks {
before_results: Arc::clone(&before_result_generations),
}))
.transcript(Arc::new(GenerationRecordingHooks {
reserved: Arc::clone(&reserved_generations),
published: Arc::clone(&published_generations),
})),
);
let hooks = hook_builder.build_runtime().unwrap();
let reservations = ReservationResources::new(hooks.len(), lease.state().reader());
let identity = generation_coord();
let emitter = SilentEmitter;
let controller = TranscriptController::new(&lease, &hooks, &reservations, &identity, &emitter);
let callback_state = run_state::Store::new();
let callback_reader = lease.state().reader();
let callback_resources = [CallbackResources::new(
&identity,
&callback_state,
&callback_reader,
None,
&emitter,
)];
let cancellation = Token::new();
let (producer, mut source) = channel(&cancellation);
let mut stream = source.take_stream().unwrap();
let facts = FactBuffer::default();
let outward = Outward::new(&producer, &facts);
let phase = ToolPhase::new(
outward,
&hooks,
&callback_resources,
&controller,
&identity,
Some("session".to_owned()),
);
let inbox = InputMailbox::new();
let mut driver =
Driver::new(&model, &lease, &inbox, &tools, &cancellation).with_tool_phase(phase);
let first_consumer = async {
expect_generation_prelude(&mut stream, true).await;
accept_tool_exchange(&mut stream).await
};
let (first_generation, first_coords) =
tokio::join!(driver.advance(RequestPlan::new()), first_consumer);
expect_generation(first_generation);
let second_consumer = async {
expect_kinds(&mut stream, &[Kind::GenerationEnd, Kind::StepEnd]).await;
expect_generation_prelude(&mut stream, false).await;
accept_tool_exchange(&mut stream).await
};
let (second_generation, second_coords) =
tokio::join!(driver.advance(RequestPlan::new()), second_consumer);
expect_generation(second_generation);
assert_eq!(first_coords.0, first_coords.1);
assert_eq!(first_coords.0, first_coords.2);
assert_eq!(second_coords.0, second_coords.1);
assert_eq!(second_coords.0, second_coords.2);
assert_ne!(first_coords.0, second_coords.0);
assert_eq!(first_coords.0.turn(), 1);
assert_eq!(second_coords.0.turn(), 1);
assert_eq!(first_coords.0.step_in_turn(), 1);
assert_eq!(second_coords.0.step_in_turn(), 2);
assert_eq!(first_coords.0.generation(), 1);
assert_eq!(second_coords.0.generation(), 2);
assert_eq!(
*reserved_generations.lock().unwrap(),
[1, 2],
"real transcript reservations must observe each current generation"
);
assert_eq!(
*published_generations.lock().unwrap(),
[1, 2],
"published reservations retain their originating generation"
);
assert_eq!(
*before_result_generations.lock().unwrap(),
[1, 2],
"before-result callbacks match ToolCall/audit/ToolResult generations"
);
assert_eq!(executions.load(Ordering::SeqCst), 2);
assert_eq!(
lease.transcript_snapshot().unwrap().version(),
Version::from_u64(4)
);
}
#[tokio::test]
async fn rejected_or_dropped_tool_call_executes_nothing_commits_nothing_and_stops() {
for (reject_index, drop_delivery) in [(0, false), (0, true), (1, false), (1, true)] {
let (session, lease) = lease(Vec::new());
lease.admit().unwrap();
let calls = [
model_tool_call("call-1", "first"),
model_tool_call("call-2", "second"),
];
let model = FakeModel::ready(tool_generation(&calls));
let executions = Arc::new(AtomicUsize::new(0));
let accepted_calls = Arc::new(AtomicUsize::new(0));
let acceptance_at_execute = Arc::new(Mutex::new(Vec::new()));
let tools = executable_tools(
&["first", "second"],
ExecutorMode::Success,
false,
&executions,
&accepted_calls,
&acceptance_at_execute,
);
let hooks = empty_hooks();
let reservations = ReservationResources::new(0, lease.state().reader());
let coord = generation_coord();
let emitter = SilentEmitter;
let controller = TranscriptController::new(&lease, &hooks, &reservations, &coord, &emitter);
let cancellation = Token::new();
let (producer, mut source) = channel(&cancellation);
let mut stream = source.take_stream().unwrap();
let facts = FactBuffer::default();
let outward = Outward::new(&producer, &facts);
let phase = ToolPhase::new(outward, &hooks, &[], &controller, &coord, None);
let inbox = InputMailbox::new();
let driver =
Driver::new(&model, &lease, &inbox, &tools, &cancellation).with_tool_phase(phase);
let consumer_accepted = Arc::clone(&accepted_calls);
let consumer = async move {
expect_generation_prelude(&mut stream, true).await;
for source_index in 0..=reject_index {
let delivery = stream.recv().await.expect("tool call delivery");
assert_eq!(event_identity(&delivery).0, DecisionPoint::ToolCall);
if source_index == reject_index {
if drop_delivery {
drop(delivery);
} else {
delivery.reject().unwrap();
}
} else {
consumer_accepted.fetch_add(1, Ordering::SeqCst);
delivery.accept().unwrap();
}
}
};
let (terminal, ()) = tokio::join!(driver.run(RequestPlan::new(), &source), consumer);
assert!(matches!(terminal, Terminal::Stopped));
assert_eq!(executions.load(Ordering::SeqCst), 0);
let snapshot = lease.transcript_snapshot().unwrap();
assert_eq!(snapshot.version(), Version::ZERO);
assert!(snapshot.items().is_empty());
assert!(!super::event::testing::has_pending_strong_decision(&source));
let successor = release_and_read_successor(&session, &lease);
assert_eq!(successor.version(), Version::ZERO);
assert!(successor.items().is_empty());
}
}
#[tokio::test]
async fn frozen_control_overrides_real_delivery_acceptance_before_tool_execution() {
for point in [FreezePoint::Stop, FreezePoint::Fatal] {
let (session, lease) = lease(Vec::new());
lease.admit().unwrap();
let call = model_tool_call("call", "tool");
let model = FakeModel::ready(tool_generation(&[call]));
let executions = Arc::new(AtomicUsize::new(0));
let accepted_calls = Arc::new(AtomicUsize::new(0));
let acceptance_at_execute = Arc::new(Mutex::new(Vec::new()));
let tools = executable_tools(
&["tool"],
ExecutorMode::Success,
false,
&executions,
&accepted_calls,
&acceptance_at_execute,
);
let hooks = empty_hooks();
let reservations = ReservationResources::new(0, lease.state().reader());
let coord = generation_coord();
let emitter = SilentEmitter;
let controller = TranscriptController::new(&lease, &hooks, &reservations, &coord, &emitter);
let cancellation = Token::new();
let (producer, mut source) = channel(&cancellation);
let mut stream = source.take_stream().unwrap();
let facts = FactBuffer::default();
let outward = Outward::new(&producer, &facts);
let phase = ToolPhase::new(outward, &hooks, &[], &controller, &coord, None);
let inbox = InputMailbox::new();
let driver =
Driver::new(&model, &lease, &inbox, &tools, &cancellation).with_tool_phase(phase);
let mut run = Box::pin(driver.run(RequestPlan::new(), &source));
tokio::select! {
() = expect_generation_prelude(&mut stream, true) => {}
terminal = &mut run => panic!("run ended before ToolCall: {:?}", terminal.outcome()),
}
let delivery = tokio::select! {
delivery = stream.recv() => delivery.expect("pending ToolCall"),
terminal = &mut run => panic!("run ended before ToolCall: {:?}", terminal.outcome()),
};
assert_eq!(delivery.decision_point(), Some(DecisionPoint::ToolCall));
let (control, pause) = pause_before_reject(&source, point);
let interruption = thread::spawn(move || match point {
FreezePoint::Stop => control.request_stop(),
FreezePoint::Fatal => control.fail(),
});
pause.wait_until_frozen();
accepted_calls.fetch_add(1, Ordering::SeqCst);
delivery
.accept()
.expect("consumer wins the internal Decision CAS in the frozen window");
pause.resume_rejection();
interruption.join().expect("interruption does not panic");
let terminal = run.await;
match point {
FreezePoint::Stop => assert!(matches!(terminal, Terminal::Stopped)),
FreezePoint::Fatal => assert!(matches!(
terminal,
Terminal::Failed(Error::Event(ValidationError::PayloadMismatch))
)),
}
assert_eq!(executions.load(Ordering::SeqCst), 0);
assert_eq!(accepted_calls.load(Ordering::SeqCst), 1);
assert!(acceptance_at_execute.lock().unwrap().is_empty());
let snapshot = lease.transcript_snapshot().unwrap();
assert_eq!(snapshot.version(), Version::ZERO);
assert!(snapshot.items().is_empty());
assert!(!super::event::testing::has_pending_strong_decision(&source));
let successor = release_and_read_successor(&session, &lease);
assert_eq!(successor.version(), Version::ZERO);
assert!(successor.items().is_empty());
}
}
#[tokio::test]
async fn consumer_drop_with_pending_tool_call_enters_stopped_and_executes_nothing() {
let (session, lease) = lease(Vec::new());
lease.admit().unwrap();
let call = model_tool_call("call", "tool");
let model = FakeModel::ready(tool_generation(&[call]));
let executions = Arc::new(AtomicUsize::new(0));
let accepted_calls = Arc::new(AtomicUsize::new(0));
let acceptance_at_execute = Arc::new(Mutex::new(Vec::new()));
let tools = executable_tools(
&["tool"],
ExecutorMode::Success,
false,
&executions,
&accepted_calls,
&acceptance_at_execute,
);
let hooks = empty_hooks();
let reservations = ReservationResources::new(0, lease.state().reader());
let coord = generation_coord();
let emitter = SilentEmitter;
let controller = TranscriptController::new(&lease, &hooks, &reservations, &coord, &emitter);
let cancellation = Token::new();
let (producer, mut source) = channel(&cancellation);
let mut stream = source.take_stream().unwrap();
let facts = FactBuffer::default();
let outward = Outward::new(&producer, &facts);
let phase = ToolPhase::new(outward, &hooks, &[], &controller, &coord, None);
let inbox = InputMailbox::new();
let driver = Driver::new(&model, &lease, &inbox, &tools, &cancellation).with_tool_phase(phase);
let consumer = async move {
expect_generation_prelude(&mut stream, true).await;
let pending = stream.recv().await.expect("pending ToolCall");
assert_eq!(pending.decision_point(), Some(DecisionPoint::ToolCall));
drop(stream);
drop(pending);
};
let (terminal, ()) = tokio::join!(driver.run(RequestPlan::new(), &source), consumer);
assert!(matches!(terminal, Terminal::Stopped));
assert_eq!(executions.load(Ordering::SeqCst), 0);
assert_eq!(accepted_calls.load(Ordering::SeqCst), 0);
assert!(lease.transcript_snapshot().unwrap().items().is_empty());
assert!(!super::event::testing::has_pending_strong_decision(&source));
let successor = release_and_read_successor(&session, &lease);
assert_eq!(successor.version(), Version::ZERO);
assert!(successor.items().is_empty());
}
#[tokio::test]
async fn consumer_drop_after_tool_call_acceptance_interrupts_pending_executor_before_release() {
let (session, lease) = lease(Vec::new());
lease.admit().unwrap();
let model = FakeModel::ready(tool_generation(&[model_tool_call("call", "tool")]));
let entered = Arc::new(Notify::new());
let counts = Arc::new(PendingExecutorCounts::default());
let tools = pending_executable_tool(&entered, &counts);
let hooks = empty_hooks();
let reservations = ReservationResources::new(0, lease.state().reader());
let coord = generation_coord();
let emitter = SilentEmitter;
let controller = TranscriptController::new(&lease, &hooks, &reservations, &coord, &emitter);
let cancellation = Token::new();
let (producer, mut source) = channel(&cancellation);
let mut stream = source.take_stream().unwrap();
let facts = FactBuffer::default();
let outward = Outward::new(&producer, &facts);
let phase = ToolPhase::new(outward, &hooks, &[], &controller, &coord, None);
let inbox = InputMailbox::new();
let driver = Driver::new(&model, &lease, &inbox, &tools, &cancellation).with_tool_phase(phase);
let mut run = Box::pin(driver.run(RequestPlan::new(), &source));
tokio::select! {
() = expect_generation_prelude(&mut stream, true) => {}
terminal = &mut run => panic!("run ended before ToolCall: {:?}", terminal.outcome()),
}
let tool_call = tokio::select! {
delivery = stream.recv() => delivery.expect("ToolCall before execution"),
terminal = &mut run => panic!("run ended before ToolCall: {:?}", terminal.outcome()),
};
assert_eq!(tool_call.decision_point(), Some(DecisionPoint::ToolCall));
tool_call.accept().unwrap();
tokio::select! {
() = entered.notified() => {}
terminal = &mut run => panic!("run ended before executor became pending: {:?}", terminal.outcome()),
}
assert_eq!(counts.active.load(Ordering::SeqCst), 1);
drop(stream);
let terminal = tokio::time::timeout(std::time::Duration::from_secs(1), &mut run)
.await
.expect("consumer drop interrupts the pending executor");
assert!(matches!(terminal, Terminal::Stopped));
assert_eq!(counts.active.load(Ordering::SeqCst), 0);
assert_eq!(counts.dropped.load(Ordering::SeqCst), 1);
assert!(!super::event::testing::has_pending_strong_decision(&source));
let snapshot = lease.transcript_snapshot().unwrap();
assert_eq!(snapshot.version(), Version::ZERO);
assert!(snapshot.items().is_empty());
let successor = release_and_read_successor(&session, &lease);
assert_eq!(successor.version(), Version::ZERO);
assert!(successor.items().is_empty());
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum PendingCancelSeam {
ToolCall,
ToolResult,
AfterToolCommit,
}
#[tokio::test]
async fn parent_cancel_at_each_pending_decision_point_is_cancelled_and_stops_later_phases() {
for pending_seam in [
PendingCancelSeam::ToolCall,
PendingCancelSeam::ToolResult,
PendingCancelSeam::AfterToolCommit,
] {
let (_session, lease) = lease(Vec::new());
lease.admit().unwrap();
let call = model_tool_call("call", "tool");
let model = FakeModel::ready(tool_generation(&[call]));
let executions = Arc::new(AtomicUsize::new(0));
let accepted_calls = Arc::new(AtomicUsize::new(0));
let acceptance_at_execute = Arc::new(Mutex::new(Vec::new()));
let tools = executable_tools(
&["tool"],
ExecutorMode::Success,
false,
&executions,
&accepted_calls,
&acceptance_at_execute,
);
let after_calls = Arc::new(Mutex::new(Vec::new()));
let mut hook_builder = hook::RegistryBuilder::new();
for (label, emit_facts) in [("first", true), ("later", false)] {
hook_builder.register(hook::Registration::new(label.to_owned()).tool(Arc::new(
AfterToolHooks {
label,
calls: Arc::clone(&after_calls),
emit_facts,
replace_staged: true,
return_item: true,
settle_named_calls: false,
active: None,
},
)));
}
let hooks = hook_builder.build_runtime().unwrap();
let reservations = ReservationResources::new(hooks.len(), lease.state().reader());
let coord = generation_coord();
let emitter = SilentEmitter;
let controller = TranscriptController::new(&lease, &hooks, &reservations, &coord, &emitter);
let states = [run_state::Store::new(), run_state::Store::new()];
let reader = lease.state().reader();
let callback_resources = [
CallbackResources::new(&coord, &states[0], &reader, None, &emitter),
CallbackResources::new(&coord, &states[1], &reader, None, &emitter),
];
let parent = Token::new();
let (producer, mut source) = channel(&parent);
let mut stream = source.take_stream().unwrap();
let facts = FactBuffer::default();
let outward = Outward::new(&producer, &facts);
let phase = ToolPhase::new(
outward,
&hooks,
&callback_resources,
&controller,
&coord,
None,
);
let inbox = InputMailbox::new();
let driver = Driver::new(&model, &lease, &inbox, &tools, &parent).with_tool_phase(phase);
let owner_done = Arc::new(Notify::new());
let driver_done = Arc::clone(&owner_done);
let run = async {
let terminal = driver.run(RequestPlan::new(), &source).await;
driver_done.notify_one();
terminal
};
let consumer_accepted = Arc::clone(&accepted_calls);
let consumer = async {
expect_generation_prelude(&mut stream, true).await;
let tool_call = stream.recv().await.expect("pending ToolCall");
assert_eq!(tool_call.decision_point(), Some(DecisionPoint::ToolCall));
if pending_seam == PendingCancelSeam::ToolCall {
parent.cancel();
owner_done.notified().await;
assert_eq!(tool_call.reject(), Err(DecisionError::AlreadyDecided));
return;
}
consumer_accepted.fetch_add(1, Ordering::SeqCst);
tool_call.accept().unwrap();
let tool_result = stream.recv().await.expect("pending ToolResult");
assert_eq!(
tool_result.decision_point(),
Some(DecisionPoint::ToolResult)
);
if pending_seam == PendingCancelSeam::ToolResult {
parent.cancel();
owner_done.notified().await;
assert_eq!(tool_result.reject(), Err(DecisionError::AlreadyDecided));
return;
}
tool_result.accept().unwrap();
let after = stream.recv().await.expect("pending AfterToolCommit fact");
assert_eq!(after.decision_point(), Some(DecisionPoint::AfterToolCommit));
parent.cancel();
owner_done.notified().await;
assert_eq!(after.reject(), Err(DecisionError::AlreadyDecided));
};
let (terminal, ()) = tokio::join!(run, consumer);
assert!(
matches!(terminal, Terminal::Cancelled),
"pending seam: {pending_seam:?}"
);
assert_eq!(
executions.load(Ordering::SeqCst),
usize::from(pending_seam != PendingCancelSeam::ToolCall),
"pending seam: {pending_seam:?}"
);
let snapshot = lease.transcript_snapshot().unwrap();
if pending_seam == PendingCancelSeam::ToolCall {
assert_eq!(snapshot.version(), Version::ZERO);
assert!(snapshot.items().is_empty());
} else {
assert_eq!(snapshot.version(), Version::from_u64(2));
assert_eq!(snapshot.items().len(), 2);
}
let expected_after = if pending_seam == PendingCancelSeam::AfterToolCommit {
vec!["first:call"]
} else {
Vec::new()
};
assert_eq!(
*after_calls.lock().unwrap(),
expected_after,
"cancellation must prevent every later callback phase: {pending_seam:?}"
);
}
}
#[tokio::test]
async fn rejected_or_dropped_tool_result_stops_remaining_publication_after_atomic_append() {
for drop_delivery in [false, true] {
let (session, lease) = lease(Vec::new());
lease.admit().unwrap();
let calls = [
model_tool_call("call-1", "first"),
model_tool_call("call-2", "second"),
];
let model = FakeModel::ready(tool_generation(&calls));
let executions = Arc::new(AtomicUsize::new(0));
let accepted_calls = Arc::new(AtomicUsize::new(0));
let acceptance_at_execute = Arc::new(Mutex::new(Vec::new()));
let tools = executable_tools(
&["first", "second"],
ExecutorMode::Success,
false,
&executions,
&accepted_calls,
&acceptance_at_execute,
);
let after_calls = Arc::new(Mutex::new(Vec::new()));
let mut hook_builder = hook::RegistryBuilder::new();
hook_builder.register(
hook::Registration::new("must-not-run".to_owned()).tool(Arc::new(AfterToolHooks {
label: "must-not-run",
calls: Arc::clone(&after_calls),
emit_facts: false,
replace_staged: false,
return_item: false,
settle_named_calls: false,
active: None,
})),
);
let hooks = hook_builder.build_runtime().unwrap();
let reservations = ReservationResources::new(hooks.len(), lease.state().reader());
let coord = generation_coord();
let emitter = SilentEmitter;
let controller = TranscriptController::new(&lease, &hooks, &reservations, &coord, &emitter);
let run_state = run_state::Store::new();
let reader = lease.state().reader();
let callback_resources = [CallbackResources::new(
&coord, &run_state, &reader, None, &emitter,
)];
let cancellation = Token::new();
let (producer, mut source) = channel(&cancellation);
let mut stream = source.take_stream().unwrap();
let facts = FactBuffer::default();
let outward = Outward::new(&producer, &facts);
let phase = ToolPhase::new(
outward,
&hooks,
&callback_resources,
&controller,
&coord,
None,
);
let inbox = InputMailbox::new();
let driver =
Driver::new(&model, &lease, &inbox, &tools, &cancellation).with_tool_phase(phase);
let consumer_accepted = Arc::clone(&accepted_calls);
let consumer = async {
expect_generation_prelude(&mut stream, true).await;
for expected in ["call-1", "call-2"] {
let delivery = stream.recv().await.expect("tool call delivery");
assert_eq!(
event_identity(&delivery),
(DecisionPoint::ToolCall, expected.to_owned())
);
consumer_accepted.fetch_add(1, Ordering::SeqCst);
delivery.accept().unwrap();
}
let delivery = stream.recv().await.expect("first tool result");
assert_eq!(
event_identity(&delivery),
(DecisionPoint::ToolResult, "call-1".to_owned())
);
if drop_delivery {
drop(delivery);
} else {
delivery.reject().unwrap();
}
};
let (terminal, ()) = tokio::join!(driver.run(RequestPlan::new(), &source), consumer);
assert!(matches!(terminal, Terminal::Stopped));
assert_eq!(executions.load(Ordering::SeqCst), 2);
assert!(
after_calls.lock().unwrap().is_empty(),
"consumer stop at ToolResult must prevent every after hook"
);
let snapshot = lease.transcript_snapshot().unwrap();
assert_eq!(snapshot.version(), Version::from_u64(3));
assert_eq!(snapshot.items().len(), 3, "append is all-or-nothing");
let next = stream.recv();
tokio::pin!(next);
assert_pending_once(next.as_mut(), "the second ToolResult must not be published");
assert!(!super::event::testing::has_pending_strong_decision(&source));
let successor = release_and_read_successor(&session, &lease);
assert_eq!(successor.version(), Version::from_u64(3));
assert_eq!(successor.items().len(), 3);
}
}
#[tokio::test]
async fn ordinary_errors_commit_business_results_but_fatal_and_cancel_commit_zero_partial() {
for (label, mode, fatal_mapping, commits) in [
("ordinary", ExecutorMode::OrdinaryError, false, true),
("fatal", ExecutorMode::Success, true, false),
("cancel", ExecutorMode::Success, false, false),
] {
let (_session, lease) = lease(Vec::new());
lease.admit().unwrap();
let call = model_tool_call("call", "tool");
let model = FakeModel::ready(tool_generation(&[call]));
let executions = Arc::new(AtomicUsize::new(0));
let accepted_calls = Arc::new(AtomicUsize::new(0));
let acceptance_at_execute = Arc::new(Mutex::new(Vec::new()));
let tools = executable_tools(
&["tool"],
mode,
fatal_mapping,
&executions,
&accepted_calls,
&acceptance_at_execute,
);
let hooks = empty_hooks();
let reservations = ReservationResources::new(0, lease.state().reader());
let coord = generation_coord();
let emitter = SilentEmitter;
let controller = TranscriptController::new(&lease, &hooks, &reservations, &coord, &emitter);
let cancellation = Token::new();
let (producer, mut source) = channel(&cancellation);
let mut stream = source.take_stream().unwrap();
let facts = FactBuffer::default();
let outward = Outward::new(&producer, &facts);
let phase = ToolPhase::new(outward, &hooks, &[], &controller, &coord, None);
let inbox = InputMailbox::new();
let mut driver =
Driver::new(&model, &lease, &inbox, &tools, &cancellation).with_tool_phase(phase);
let consumer_accepted = Arc::clone(&accepted_calls);
let consumer_cancellation = cancellation.clone();
let consumer = async {
expect_generation_prelude(&mut stream, true).await;
let delivery = stream.recv().await.expect("tool call delivery");
assert_eq!(event_identity(&delivery).0, DecisionPoint::ToolCall);
consumer_accepted.fetch_add(1, Ordering::SeqCst);
delivery.accept().unwrap();
if label == "cancel" {
consumer_cancellation.cancel();
}
if commits {
let delivery = stream.recv().await.expect("ordinary error result");
assert_eq!(event_identity(&delivery).0, DecisionPoint::ToolResult);
delivery.accept().unwrap();
}
};
let (result, ()) = tokio::time::timeout(std::time::Duration::from_secs(2), async {
tokio::join!(driver.advance(RequestPlan::new()), consumer)
})
.await
.unwrap_or_else(|_| panic!("{label} case timed out"));
assert_eq!(
executions.load(Ordering::SeqCst),
usize::from(label != "cancel")
);
let snapshot = lease.transcript_snapshot().unwrap();
if commits {
expect_generation(result);
assert_eq!(snapshot.version(), Version::from_u64(2));
assert_eq!(snapshot.items().len(), 2);
} else {
if label == "fatal" {
assert!(matches!(result, Err(Error::Route(_))));
} else {
assert!(
matches!(result, Err(Error::Cancelled)),
"parent cancellation before final confirmation must revoke caller acceptance"
);
}
assert_eq!(snapshot.version(), Version::ZERO);
assert!(
snapshot.items().is_empty(),
"fatal/cancel leaked a partial append"
);
let next = driver.advance(RequestPlan::new()).await;
if label == "cancel" {
assert!(matches!(next, Err(Error::Cancelled)));
} else {
assert!(expect_advance(next).is_none());
}
}
}
}
#[tokio::test]
async fn silent_after_hook_commits_staged_replacement_before_returned_batch_without_marker() {
let (_session, lease) = lease(Vec::new());
lease.admit().unwrap();
let call = model_tool_call("call", "tool");
let model = FakeModel::ready(tool_generation(&[call]));
let executions = Arc::new(AtomicUsize::new(0));
let accepted_calls = Arc::new(AtomicUsize::new(0));
let acceptance_at_execute = Arc::new(Mutex::new(Vec::new()));
let tools = executable_tools(
&["tool"],
ExecutorMode::Success,
false,
&executions,
&accepted_calls,
&acceptance_at_execute,
);
let calls = Arc::new(Mutex::new(Vec::new()));
let mut hook_builder = hook::RegistryBuilder::new();
hook_builder.register(hook::Registration::new("silent".to_owned()).tool(Arc::new(
AfterToolHooks {
label: "silent",
calls: Arc::clone(&calls),
emit_facts: false,
replace_staged: true,
return_item: true,
settle_named_calls: false,
active: None,
},
)));
let hooks = hook_builder.build_runtime().unwrap();
let reservations = ReservationResources::new(hooks.len(), lease.state().reader());
let coord = generation_coord();
let emitter = SilentEmitter;
let controller = TranscriptController::new(&lease, &hooks, &reservations, &coord, &emitter);
let run_state = run_state::Store::new();
let reader = lease.state().reader();
let callback_resources = [CallbackResources::new(
&coord, &run_state, &reader, None, &emitter,
)];
let cancellation = Token::new();
let (producer, mut source) = channel(&cancellation);
let mut stream = source.take_stream().unwrap();
let facts = FactBuffer::default();
let outward = Outward::new(&producer, &facts);
let phase = ToolPhase::new(
outward,
&hooks,
&callback_resources,
&controller,
&coord,
None,
);
let inbox = InputMailbox::new();
let mut driver =
Driver::new(&model, &lease, &inbox, &tools, &cancellation).with_tool_phase(phase);
let consumer = async {
expect_generation_prelude(&mut stream, true).await;
for seam in [DecisionPoint::ToolCall, DecisionPoint::ToolResult] {
let delivery = stream.recv().await.expect("main tool fact");
assert_eq!(delivery.decision_point(), Some(seam));
delivery.accept().unwrap();
}
};
let (result, ()) = tokio::join!(driver.advance(RequestPlan::new()), consumer);
expect_generation(result);
assert_eq!(*calls.lock().unwrap(), ["silent:call"]);
expect_kinds(&mut stream, &[Kind::GenerationEnd]).await;
let next = stream.recv();
tokio::pin!(next);
assert_pending_once(
next.as_mut(),
"a callback that emitted nothing must not synthesize a marker",
);
let snapshot = lease.transcript_snapshot().unwrap();
assert_eq!(snapshot.version(), Version::from_u64(4));
assert_eq!(snapshot.items().len(), 2);
assert!(
matches!(&snapshot.items()[0], Item::System(item) if item.text() == "silent:call:staged")
);
assert!(
matches!(&snapshot.items()[1], Item::System(item) if item.text() == "silent:call:returned")
);
}
#[tokio::test]
async fn after_hooks_follow_routed_source_and_registration_order_with_fifo_real_facts() {
let (_session, lease) = lease(Vec::new());
lease.admit().unwrap();
let calls = [
model_tool_call("blocked-call", "blocked"),
model_tool_call("exec-call", "exec"),
model_tool_call("short-call", "short"),
model_tool_call("unknown-call", "unknown"),
];
let model = FakeModel::ready(tool_generation(&calls));
let executions = Arc::new(AtomicUsize::new(0));
let accepted_calls = Arc::new(AtomicUsize::new(0));
let acceptance_at_execute = Arc::new(Mutex::new(Vec::new()));
let mut tool_builder = tool::RegistryBuilder::new();
tool_builder.register(client_tool(
"exec",
ExecutorMode::Success,
false,
&executions,
&accepted_calls,
&acceptance_at_execute,
));
let tools = tool_builder.build().unwrap();
let after_calls = Arc::new(Mutex::new(Vec::new()));
let mut hook_builder = hook::RegistryBuilder::new();
for (label, emit_facts) in [("first", true), ("second", false)] {
hook_builder.register(hook::Registration::new(label.to_owned()).tool(Arc::new(
AfterToolHooks {
label,
calls: Arc::clone(&after_calls),
emit_facts,
replace_staged: false,
return_item: false,
settle_named_calls: true,
active: None,
},
)));
}
let hooks = hook_builder.build_runtime().unwrap();
let reservations = ReservationResources::new(hooks.len(), lease.state().reader());
let coord = generation_coord();
let emitter = SilentEmitter;
let controller = TranscriptController::new(&lease, &hooks, &reservations, &coord, &emitter);
let states = [run_state::Store::new(), run_state::Store::new()];
let reader = lease.state().reader();
let callback_resources = [
CallbackResources::new(&coord, &states[0], &reader, None, &emitter),
CallbackResources::new(&coord, &states[1], &reader, None, &emitter),
];
let cancellation = Token::new();
let (producer, mut source) = channel(&cancellation);
let mut stream = source.take_stream().unwrap();
let facts = FactBuffer::default();
let outward = Outward::new(&producer, &facts);
let phase = ToolPhase::new(
outward,
&hooks,
&callback_resources,
&controller,
&coord,
None,
);
let inbox = InputMailbox::new();
let mut driver =
Driver::new(&model, &lease, &inbox, &tools, &cancellation).with_tool_phase(phase);
let observed = Arc::new(Mutex::new(Vec::new()));
let consumer_observed = Arc::clone(&observed);
let consumer_accepted = Arc::clone(&accepted_calls);
let consumer = async {
expect_generation_prelude(&mut stream, true).await;
for _ in 0..4 {
let delivery = stream.recv().await.expect("tool call");
assert_eq!(delivery.decision_point(), Some(DecisionPoint::ToolCall));
consumer_accepted.fetch_add(1, Ordering::SeqCst);
delivery.accept().unwrap();
}
for _ in 0..4 {
let delivery = stream.recv().await.expect("tool result");
assert_eq!(delivery.decision_point(), Some(DecisionPoint::ToolResult));
delivery.accept().unwrap();
}
for _ in 0..6 {
let delivery = stream.recv().await.expect("real after-hook fact");
assert_eq!(
delivery.decision_point(),
Some(DecisionPoint::AfterToolCommit)
);
let fact = match delivery.event().kind() {
Kind::Audit => delivery.event().as_audit().unwrap().kind().to_owned(),
Kind::Error => delivery.event().as_error().unwrap().to_string(),
kind => panic!("unexpected after-hook kind: {kind:?}"),
};
consumer_observed
.lock()
.unwrap()
.push((delivery.event().coord().unwrap().call_id().to_owned(), fact));
delivery.accept().unwrap();
}
};
let (result, ()) = tokio::join!(driver.advance(RequestPlan::new()), consumer);
expect_generation(result);
assert_eq!(executions.load(Ordering::SeqCst), 1);
assert_eq!(
*after_calls.lock().unwrap(),
[
"first:exec-call",
"second:exec-call",
"first:unknown-call",
"second:unknown-call",
]
);
assert_eq!(
*observed.lock().unwrap(),
[
("exec-call".to_owned(), "first:exec-call:audit-1".to_owned()),
("exec-call".to_owned(), "after-error".to_owned()),
("exec-call".to_owned(), "first:exec-call:audit-2".to_owned()),
(
"unknown-call".to_owned(),
"first:unknown-call:audit-1".to_owned(),
),
("unknown-call".to_owned(), "after-error".to_owned()),
(
"unknown-call".to_owned(),
"first:unknown-call:audit-2".to_owned(),
),
]
);
let snapshot = lease.transcript_snapshot().unwrap();
assert_eq!(snapshot.version(), Version::from_u64(5));
assert_eq!(snapshot.items().len(), 5);
}
#[tokio::test]
async fn rejected_or_dropped_after_fact_discards_callback_transcript_and_stops_fanout() {
for drop_delivery in [false, true] {
let (session, lease) = lease(Vec::new());
lease.admit().unwrap();
let calls = [
model_tool_call("call-1", "first"),
model_tool_call("call-2", "second"),
];
let model = FakeModel::ready(tool_generation(&calls));
let executions = Arc::new(AtomicUsize::new(0));
let accepted_calls = Arc::new(AtomicUsize::new(0));
let acceptance_at_execute = Arc::new(Mutex::new(Vec::new()));
let tools = executable_tools(
&["first", "second"],
ExecutorMode::Success,
false,
&executions,
&accepted_calls,
&acceptance_at_execute,
);
let after_calls = Arc::new(Mutex::new(Vec::new()));
let mut hook_builder = hook::RegistryBuilder::new();
for (label, emit_facts) in [("first", true), ("later", false)] {
hook_builder.register(hook::Registration::new(label.to_owned()).tool(Arc::new(
AfterToolHooks {
label,
calls: Arc::clone(&after_calls),
emit_facts,
replace_staged: true,
return_item: true,
settle_named_calls: false,
active: None,
},
)));
}
let hooks = hook_builder.build_runtime().unwrap();
let reservations = ReservationResources::new(hooks.len(), lease.state().reader());
let coord = generation_coord();
let emitter = SilentEmitter;
let controller = TranscriptController::new(&lease, &hooks, &reservations, &coord, &emitter);
let states = [run_state::Store::new(), run_state::Store::new()];
let reader = lease.state().reader();
let callback_resources = [
CallbackResources::new(&coord, &states[0], &reader, None, &emitter),
CallbackResources::new(&coord, &states[1], &reader, None, &emitter),
];
let cancellation = Token::new();
let (producer, mut source) = channel(&cancellation);
let mut stream = source.take_stream().unwrap();
let facts = FactBuffer::default();
let outward = Outward::new(&producer, &facts);
let phase = ToolPhase::new(
outward,
&hooks,
&callback_resources,
&controller,
&coord,
None,
);
let inbox = InputMailbox::new();
let driver =
Driver::new(&model, &lease, &inbox, &tools, &cancellation).with_tool_phase(phase);
let consumer_accepted = Arc::clone(&accepted_calls);
let consumer = async {
expect_generation_prelude(&mut stream, true).await;
for _ in 0..2 {
let delivery = stream.recv().await.expect("tool call");
consumer_accepted.fetch_add(1, Ordering::SeqCst);
delivery.accept().unwrap();
}
for _ in 0..2 {
stream.recv().await.expect("tool result").accept().unwrap();
}
let delivery = stream.recv().await.expect("first real hook fact");
assert_eq!(
delivery.decision_point(),
Some(DecisionPoint::AfterToolCommit)
);
if drop_delivery {
drop(delivery);
} else {
delivery.reject().unwrap();
}
};
let (terminal, ()) = tokio::join!(driver.run(RequestPlan::new(), &source), consumer);
assert!(matches!(terminal, Terminal::Stopped));
assert_eq!(*after_calls.lock().unwrap(), ["first:call-1"]);
let snapshot = lease.transcript_snapshot().unwrap();
assert_eq!(snapshot.version(), Version::from_u64(3));
assert_eq!(snapshot.items().len(), 3);
assert!(matches!(snapshot.items()[0], Item::Assistant(_)));
assert!(!super::event::testing::has_pending_strong_decision(&source));
let successor = release_and_read_successor(&session, &lease);
assert_eq!(successor.version(), Version::from_u64(3));
assert_eq!(successor.items().len(), 3);
}
}
#[tokio::test]
async fn accepted_after_facts_and_hook_transactions_survive_late_run_cancellation() {
let (session, lease) = lease(Vec::new());
lease.admit().unwrap();
let call = model_tool_call("call", "tool");
let model = FakeModel::ready(tool_generation(&[call]));
let executions = Arc::new(AtomicUsize::new(0));
let accepted_calls = Arc::new(AtomicUsize::new(0));
let acceptance_at_execute = Arc::new(Mutex::new(Vec::new()));
let tools = executable_tools(
&["tool"],
ExecutorMode::Success,
false,
&executions,
&accepted_calls,
&acceptance_at_execute,
);
let after_calls = Arc::new(Mutex::new(Vec::new()));
let active_callbacks = Arc::new(AtomicUsize::new(0));
let mut hook_builder = hook::RegistryBuilder::new();
hook_builder.register(hook::Registration::new("durable".to_owned()).tool(Arc::new(
AfterToolHooks {
label: "durable",
calls: Arc::clone(&after_calls),
emit_facts: true,
replace_staged: true,
return_item: true,
settle_named_calls: false,
active: Some(Arc::clone(&active_callbacks)),
},
)));
let hooks = hook_builder.build_runtime().unwrap();
let reservations = ReservationResources::new(hooks.len(), lease.state().reader());
let coord = generation_coord();
let emitter = SilentEmitter;
let controller = TranscriptController::new(&lease, &hooks, &reservations, &coord, &emitter);
let run_state = run_state::Store::new();
let reader = lease.state().reader();
let callback_resources = [CallbackResources::new(
&coord, &run_state, &reader, None, &emitter,
)];
let cancellation = Token::new();
let (producer, mut source) = channel(&cancellation);
let mut stream = source.take_stream().unwrap();
let facts = FactBuffer::default();
let outward = Outward::new(&producer, &facts);
let phase = ToolPhase::new(
outward,
&hooks,
&callback_resources,
&controller,
&coord,
None,
);
let inbox = InputMailbox::new();
let driver = Driver::new(&model, &lease, &inbox, &tools, &cancellation).with_tool_phase(phase);
let late_cancel = cancellation.clone();
let consumer = async {
expect_generation_prelude(&mut stream, true).await;
stream.recv().await.unwrap().accept().unwrap();
stream.recv().await.unwrap().accept().unwrap();
for index in 0..3 {
let delivery = stream.recv().await.expect("buffered hook fact");
if index == 2 {
let confirmation = pause_after_final_confirm(&source);
delivery.accept().unwrap();
confirmation.wait_until_confirmed().await;
late_cancel.cancel();
confirmation.resume_return();
} else {
delivery.accept().unwrap();
}
}
};
let (terminal, ()) = tokio::join!(driver.run(RequestPlan::new(), &source), consumer);
assert!(matches!(terminal, Terminal::Cancelled));
assert!(cancellation.is_cancelled());
assert_eq!(*after_calls.lock().unwrap(), ["durable:call"]);
assert_eq!(
active_callbacks.load(Ordering::SeqCst),
0,
"terminal return joins the cooperative blocking callback owner"
);
assert!(!super::event::testing::has_pending_strong_decision(&source));
let snapshot = lease.transcript_snapshot().unwrap();
assert_eq!(snapshot.version(), Version::from_u64(4));
assert_eq!(snapshot.items().len(), 2);
assert!(
matches!(&snapshot.items()[0], Item::System(item) if item.text() == "durable:call:staged")
);
assert!(
matches!(&snapshot.items()[1], Item::System(item) if item.text() == "durable:call:returned")
);
let successor = release_and_read_successor(&session, &lease);
assert_eq!(successor.version(), Version::from_u64(4));
assert_eq!(successor.items().len(), 2);
}
#[tokio::test]
async fn each_after_emit_pauses_callback_until_its_own_consumer_ack() {
let (_session, lease) = lease(Vec::new());
lease.admit().unwrap();
let call = model_tool_call("call", "tool");
let model = FakeModel::ready(tool_generation(&[call]));
let executions = Arc::new(AtomicUsize::new(0));
let accepted_calls = Arc::new(AtomicUsize::new(0));
let acceptance_at_execute = Arc::new(Mutex::new(Vec::new()));
let tools = executable_tools(
&["tool"],
ExecutorMode::Success,
false,
&executions,
&accepted_calls,
&acceptance_at_execute,
);
let (callback_ack_probe, mut callback_acks) = watch::channel(0_usize);
let mut hook_builder = hook::RegistryBuilder::new();
hook_builder.register(
hook::Registration::new("per-event".to_owned()).tool(Arc::new(PerEventPauseHooks)),
);
let hooks = hook_builder.build_runtime().unwrap();
let reservations = ReservationResources::new(hooks.len(), lease.state().reader());
let coord = generation_coord();
let emitter = SilentEmitter;
let controller = TranscriptController::new(&lease, &hooks, &reservations, &coord, &emitter);
let run_state = run_state::Store::new();
let reader = lease.state().reader();
let callback_resources = [CallbackResources::new(
&coord, &run_state, &reader, None, &emitter,
)];
let cancellation = Token::new();
let (producer, mut source) = channel(&cancellation);
let mut stream = source.take_stream().unwrap();
let facts = FactBuffer::default();
let outward = Outward::new(&producer, &facts);
let phase = ToolPhase::new(
outward,
&hooks,
&callback_resources,
&controller,
&coord,
None,
)
.with_test_callback_ack_probe(callback_ack_probe.clone());
let inbox = InputMailbox::new();
let mut driver =
Driver::new(&model, &lease, &inbox, &tools, &cancellation).with_tool_phase(phase);
let consumer_accepted = Arc::clone(&accepted_calls);
let consumer = async {
expect_generation_prelude(&mut stream, true).await;
let call = stream.recv().await.expect("tool call");
consumer_accepted.fetch_add(1, Ordering::SeqCst);
call.accept().unwrap();
stream.recv().await.expect("tool result").accept().unwrap();
let first = stream.recv().await.expect("first after fact");
assert_eq!(first.event().as_audit().unwrap().kind(), "pause-1");
assert_eq!(*callback_acks.borrow(), 0);
{
let next = stream.recv();
tokio::pin!(next);
assert_pending_once(
next.as_mut(),
"second emit must not run while the first Delivery is pending",
);
}
first.accept().unwrap();
callback_acks
.changed()
.await
.expect("first callback acknowledgement is recorded");
assert_eq!(*callback_acks.borrow(), 1);
let second = stream.recv().await.expect("second after fact");
assert_eq!(second.event().as_audit().unwrap().kind(), "pause-2");
assert_eq!(*callback_acks.borrow(), 1);
second.accept().unwrap();
callback_acks
.changed()
.await
.expect("second callback acknowledgement is recorded");
assert_eq!(*callback_acks.borrow(), 2);
};
let (result, ()) = tokio::join!(driver.advance(RequestPlan::new()), consumer);
expect_generation(result);
assert_eq!(*callback_ack_probe.borrow(), 2);
}
#[tokio::test]
async fn invalid_after_fact_aborts_and_joins_callback_before_returning_event_error() {
let (_session, lease) = lease(Vec::new());
lease.admit().unwrap();
let call = model_tool_call("call", "tool");
let model = FakeModel::ready(tool_generation(&[call]));
let executions = Arc::new(AtomicUsize::new(0));
let accepted_calls = Arc::new(AtomicUsize::new(0));
let acceptance_at_execute = Arc::new(Mutex::new(Vec::new()));
let tools = executable_tools(
&["tool"],
ExecutorMode::Success,
false,
&executions,
&accepted_calls,
&acceptance_at_execute,
);
let continued = Arc::new(AtomicUsize::new(0));
let mut hook_builder = hook::RegistryBuilder::new();
hook_builder.register(hook::Registration::new("invalid".to_owned()).tool(Arc::new(
InvalidAfterFactHooks {
continued: Arc::clone(&continued),
},
)));
let hooks = hook_builder.build_runtime().unwrap();
let reservations = ReservationResources::new(hooks.len(), lease.state().reader());
let coord = generation_coord();
let emitter = SilentEmitter;
let controller = TranscriptController::new(&lease, &hooks, &reservations, &coord, &emitter);
let run_state = run_state::Store::new();
let reader = lease.state().reader();
let callback_resources = [CallbackResources::new(
&coord, &run_state, &reader, None, &emitter,
)];
let cancellation = Token::new();
let (producer, mut source) = channel(&cancellation);
let mut stream = source.take_stream().unwrap();
let facts = FactBuffer::default();
let outward = Outward::new(&producer, &facts);
let phase = ToolPhase::new(
outward,
&hooks,
&callback_resources,
&controller,
&coord,
None,
);
let inbox = InputMailbox::new();
let driver = Driver::new(&model, &lease, &inbox, &tools, &cancellation).with_tool_phase(phase);
let consumer_accepted = Arc::clone(&accepted_calls);
let consumer = async {
expect_generation_prelude(&mut stream, true).await;
let call = stream.recv().await.expect("tool call");
consumer_accepted.fetch_add(1, Ordering::SeqCst);
call.accept().unwrap();
stream.recv().await.expect("tool result").accept().unwrap();
};
let (terminal, ()) = tokio::join!(driver.run(RequestPlan::new(), &source), consumer);
cancellation.cancel();
assert!(matches!(terminal.failure(), Some(Error::Event(_))));
assert_eq!(
terminal.outcome(),
crate::value::lifecycle::Outcome::Failed,
"a late cancellation cannot replace an event-validation fatal"
);
assert_eq!(continued.load(Ordering::SeqCst), 0);
let snapshot = lease.transcript_snapshot().unwrap();
assert_eq!(snapshot.version(), Version::from_u64(2));
assert_eq!(snapshot.items().len(), 2);
}
#[tokio::test]
async fn session_write_after_emit_occurs_only_after_accept_and_never_after_reject_or_drop() {
for decision in [Some(true), Some(false), None] {
let (_session, lease) = lease(Vec::new());
lease.admit().unwrap();
let key = session_state::Key::new();
let call = model_tool_call("call", "tool");
let model = FakeModel::ready(tool_generation(&[call]));
let executions = Arc::new(AtomicUsize::new(0));
let accepted_calls = Arc::new(AtomicUsize::new(0));
let acceptance_at_execute = Arc::new(Mutex::new(Vec::new()));
let tools = executable_tools(
&["tool"],
ExecutorMode::Success,
false,
&executions,
&accepted_calls,
&acceptance_at_execute,
);
let writes = Arc::new(AtomicUsize::new(0));
let mut hook_builder = hook::RegistryBuilder::new();
hook_builder.register(
hook::Registration::new("writer".to_owned())
.requires_session_state(true)
.tool(Arc::new(WriteAfterEmitHooks {
key: key.clone(),
writes: Arc::clone(&writes),
})),
);
let hooks = hook_builder.build_runtime().unwrap();
let reservations = ReservationResources::new(hooks.len(), lease.state().reader());
let coord = generation_coord();
let emitter = SilentEmitter;
let controller = TranscriptController::new(&lease, &hooks, &reservations, &coord, &emitter);
let run_state = run_state::Store::new();
let reader = lease.state().reader();
let callback_resources = [CallbackResources::new(
&coord,
&run_state,
&reader,
Some(lease.state()),
&emitter,
)];
let cancellation = Token::new();
let (producer, mut source) = channel(&cancellation);
let mut stream = source.take_stream().unwrap();
let facts = FactBuffer::default();
let outward = Outward::new(&producer, &facts);
let phase = ToolPhase::new(
outward,
&hooks,
&callback_resources,
&controller,
&coord,
None,
);
let inbox = InputMailbox::new();
let mut driver =
Driver::new(&model, &lease, &inbox, &tools, &cancellation).with_tool_phase(phase);
let consumer_accepted = Arc::clone(&accepted_calls);
let consumer = async {
expect_generation_prelude(&mut stream, true).await;
let call = stream.recv().await.expect("tool call");
consumer_accepted.fetch_add(1, Ordering::SeqCst);
call.accept().unwrap();
stream.recv().await.expect("tool result").accept().unwrap();
let after = stream.recv().await.expect("after fact");
assert_eq!(writes.load(Ordering::SeqCst), 0);
assert_eq!(lease.state().load(&key), None);
match decision {
Some(true) => after.accept().unwrap(),
Some(false) => after.reject().unwrap(),
None => drop(after),
}
};
let (result, ()) = tokio::join!(driver.advance(RequestPlan::new()), consumer);
if decision == Some(true) {
expect_generation(result);
assert_eq!(writes.load(Ordering::SeqCst), 1);
assert_eq!(lease.state().load(&key), Some(AfterWriteSnapshotValue(7)));
} else {
assert!(matches!(result, Err(Error::AfterToolExecuteRejected)));
assert_eq!(writes.load(Ordering::SeqCst), 0);
assert_eq!(lease.state().load(&key), None);
}
}
}
async fn catch_abort_write_case(mode: CatchAbortMode) {
let (_session, lease) = lease(Vec::new());
lease.admit().unwrap();
let set_key = session_state::Key::new();
let load_or_store_key = session_state::Key::new();
let delete_key = session_state::Key::new();
lease
.state()
.set(&delete_key, &AfterWriteSnapshotValue(9))
.unwrap();
let call = model_tool_call("call", "tool");
let model = FakeModel::ready(tool_generation(&[call]));
let executions = Arc::new(AtomicUsize::new(0));
let accepted_calls = Arc::new(AtomicUsize::new(0));
let acceptance_at_execute = Arc::new(Mutex::new(Vec::new()));
let tools = executable_tools(
&["tool"],
ExecutorMode::Success,
false,
&executions,
&accepted_calls,
&acceptance_at_execute,
);
let observation = Arc::new(Mutex::new(None));
let completed = Arc::new(Notify::new());
let mut hook_builder = hook::RegistryBuilder::new();
hook_builder.register(
hook::Registration::new("catch-abort-writer".to_owned())
.requires_session_state(true)
.tool(Arc::new(CatchAbortWriteHooks {
mode,
set_key: set_key.clone(),
load_or_store_key: load_or_store_key.clone(),
delete_key: delete_key.clone(),
observation: Arc::clone(&observation),
completed: Arc::clone(&completed),
})),
);
let hooks = hook_builder.build_runtime().unwrap();
let reservations = ReservationResources::new(hooks.len(), lease.state().reader());
let coord = generation_coord();
let emitter = SilentEmitter;
let controller = TranscriptController::new(&lease, &hooks, &reservations, &coord, &emitter);
let run_state = run_state::Store::new();
let reader = lease.state().reader();
let callback_resources = [CallbackResources::new(
&coord,
&run_state,
&reader,
Some(lease.state()),
&emitter,
)];
let cancellation = Token::new();
let (producer, mut source) = channel(&cancellation);
let mut stream = source.take_stream().unwrap();
let facts = FactBuffer::default();
let outward = Outward::new(&producer, &facts);
let phase = ToolPhase::new(
outward,
&hooks,
&callback_resources,
&controller,
&coord,
None,
);
let inbox = InputMailbox::new();
let mut driver =
Driver::new(&model, &lease, &inbox, &tools, &cancellation).with_tool_phase(phase);
let consumer_accepted = Arc::clone(&accepted_calls);
let callback_completed = Arc::clone(&completed);
let confirmation_source = &source;
let consumer = async move {
expect_generation_prelude(&mut stream, true).await;
let call = stream.recv().await.expect("tool call");
consumer_accepted.fetch_add(1, Ordering::SeqCst);
call.accept().unwrap();
stream.recv().await.expect("tool result").accept().unwrap();
match mode {
CatchAbortMode::Accept => {
let delivery = stream.recv().await.expect("accepted after fact");
let confirmation = pause_after_final_confirm(confirmation_source);
delivery.accept().unwrap();
confirmation.wait_until_confirmed().await;
confirmation.resume_return();
}
CatchAbortMode::Reject => stream
.recv()
.await
.expect("rejected after fact")
.reject()
.unwrap(),
CatchAbortMode::DropDelivery => {
drop(stream.recv().await.expect("dropped after fact"));
}
CatchAbortMode::Invalid => callback_completed.notified().await,
CatchAbortMode::ChannelClose => {
let pending = stream
.recv()
.await
.expect("pending after fact before channel close");
drop(stream);
callback_completed.notified().await;
drop(pending);
}
}
};
let (result, ()) = tokio::join!(driver.advance(RequestPlan::new()), consumer);
let accepted = matches!(mode, CatchAbortMode::Accept);
if accepted {
expect_generation(result);
} else if matches!(mode, CatchAbortMode::Invalid) {
assert!(matches!(result, Err(Error::Event(_))), "mode: {mode:?}");
} else {
assert!(
matches!(result, Err(Error::AfterToolExecuteRejected)),
"mode: {mode:?}"
);
}
assert_eq!(
*observation.lock().unwrap(),
Some(CatchAbortObservation {
caught_abort: !accepted,
writes_succeeded: [accepted; 3],
}),
"mode: {mode:?}"
);
if accepted {
assert_eq!(
lease.state().load(&set_key),
Some(AfterWriteSnapshotValue(1))
);
assert_eq!(
lease.state().load(&load_or_store_key),
Some(AfterWriteSnapshotValue(2))
);
assert_eq!(lease.state().load(&delete_key), None);
} else {
assert_eq!(lease.state().load(&set_key), None);
assert_eq!(lease.state().load(&load_or_store_key), None);
assert_eq!(
lease.state().load(&delete_key),
Some(AfterWriteSnapshotValue(9))
);
}
}
#[tokio::test]
async fn callback_permit_blocks_writes_after_caught_rejection_abort() {
catch_abort_write_case(CatchAbortMode::Reject).await;
}
#[tokio::test]
async fn callback_permit_blocks_writes_after_caught_delivery_drop_abort() {
catch_abort_write_case(CatchAbortMode::DropDelivery).await;
}
#[tokio::test]
async fn callback_permit_blocks_writes_after_caught_invalid_fact_abort() {
catch_abort_write_case(CatchAbortMode::Invalid).await;
}
#[tokio::test]
async fn callback_permit_blocks_writes_after_caught_channel_close_abort() {
catch_abort_write_case(CatchAbortMode::ChannelClose).await;
}
#[tokio::test]
async fn callback_permit_allows_writes_after_finally_confirmed_acceptance() {
catch_abort_write_case(CatchAbortMode::Accept).await;
}
#[tokio::test]
async fn dropping_a_pending_callback_join_retains_synchronous_cleanup_authority() {
assert!(super::driver::managed_callback_join_retains_drop_authority().await);
}
#[tokio::test]
async fn dropping_run_future_at_after_seam_revokes_writes_and_joins_callback() {
let (session, lease) = lease(Vec::new());
lease.admit().unwrap();
let set_key = session_state::Key::new();
let load_or_store_key = session_state::Key::new();
let delete_key = session_state::Key::new();
lease
.state()
.set(&set_key, &AfterWriteSnapshotValue(7))
.unwrap();
lease
.state()
.set(&load_or_store_key, &AfterWriteSnapshotValue(8))
.unwrap();
lease
.state()
.set(&delete_key, &AfterWriteSnapshotValue(9))
.unwrap();
let call = model_tool_call("call", "tool");
let model = FakeModel::ready(tool_generation(&[call]));
let executions = Arc::new(AtomicUsize::new(0));
let accepted_calls = Arc::new(AtomicUsize::new(0));
let acceptance_at_execute = Arc::new(Mutex::new(Vec::new()));
let tools = executable_tools(
&["tool"],
ExecutorMode::Success,
false,
&executions,
&accepted_calls,
&acceptance_at_execute,
);
let active = Arc::new(AtomicUsize::new(0));
let observation = Arc::new(Mutex::new(None));
let callback_abort_order = Arc::new(Mutex::new(Vec::new()));
let mut hook_builder = hook::RegistryBuilder::new();
hook_builder.register(
hook::Registration::new("drop-owner".to_owned())
.requires_session_state(true)
.tool(Arc::new(DropOwnerCatchAbortHooks {
set_key: set_key.clone(),
load_or_store_key: load_or_store_key.clone(),
delete_key: delete_key.clone(),
observation: Arc::clone(&observation),
active: Arc::clone(&active),
})),
);
let hooks = hook_builder.build_runtime().unwrap();
let reservations = ReservationResources::new(hooks.len(), lease.state().reader());
let coord = generation_coord();
let emitter = SilentEmitter;
let controller = TranscriptController::new(&lease, &hooks, &reservations, &coord, &emitter);
let run_state = run_state::Store::new();
let reader = lease.state().reader();
let callback_resources = [CallbackResources::new(
&coord,
&run_state,
&reader,
Some(lease.state()),
&emitter,
)];
let cancellation = Token::new();
let (producer, mut source) = channel(&cancellation);
let mut stream = source.take_stream().unwrap();
let facts = FactBuffer::default();
let outward = Outward::new(&producer, &facts);
let phase = ToolPhase::new(
outward,
&hooks,
&callback_resources,
&controller,
&coord,
None,
)
.with_test_callback_abort_order(Arc::clone(&callback_abort_order));
let inbox = InputMailbox::new();
let driver = Driver::new(&model, &lease, &inbox, &tools, &cancellation).with_tool_phase(phase);
let mut run = Box::pin(driver.run(RequestPlan::new(), &source));
tokio::select! {
() = expect_generation_prelude(&mut stream, true) => {}
terminal = &mut run => panic!("run ended before ToolCall: {:?}", terminal.outcome()),
}
let tool_call = tokio::select! {
delivery = stream.recv() => delivery.expect("tool call"),
terminal = &mut run => panic!("run ended before ToolCall: {:?}", terminal.outcome()),
};
accepted_calls.fetch_add(1, Ordering::SeqCst);
tool_call.accept().unwrap();
let tool_result = tokio::select! {
delivery = stream.recv() => delivery.expect("tool result"),
terminal = &mut run => panic!("run ended before ToolResult: {:?}", terminal.outcome()),
};
tool_result.accept().unwrap();
let after = tokio::select! {
delivery = stream.recv() => delivery.expect("pending after fact"),
terminal = &mut run => panic!("run ended before after fact: {:?}", terminal.outcome()),
};
assert_eq!(after.decision_point(), Some(DecisionPoint::AfterToolCommit));
assert_eq!(active.load(Ordering::SeqCst), 1);
drop(run);
assert_eq!(after.accept(), Err(DecisionError::AlreadyDecided));
assert_eq!(after.reject(), Err(DecisionError::AlreadyDecided));
assert!(!super::event::testing::has_pending_strong_decision(&source));
assert_eq!(
*callback_abort_order.lock().unwrap(),
vec![CallbackAbortStep::Revoked, CallbackAbortStep::AbortAckSent],
"owner Drop must revoke callback writes before sending the abort ack"
);
assert_eq!(
active.load(Ordering::SeqCst),
0,
"future Drop must synchronously drain the blocking owner"
);
assert_eq!(
*observation.lock().unwrap(),
Some(DropOwnerAbortObservation {
caught_abort: true,
write_errors: [WriteError::LeaseReleased; 3],
}),
"owner Drop must revoke writes before aborting the pending emit"
);
assert_eq!(
lease.state().load(&set_key),
Some(AfterWriteSnapshotValue(7))
);
assert_eq!(
lease.state().load(&load_or_store_key),
Some(AfterWriteSnapshotValue(8))
);
assert_eq!(
lease.state().load(&delete_key),
Some(AfterWriteSnapshotValue(9))
);
drop(after);
let snapshot = lease.transcript_snapshot().unwrap();
assert_eq!(snapshot.version(), Version::from_u64(2));
assert_eq!(snapshot.items().len(), 2);
let successor = release_and_read_successor(&session, &lease);
assert_eq!(successor.version(), Version::from_u64(2));
assert_eq!(successor.items().len(), 2);
}
#[test]
fn accepted_after_event_survives_real_hook_panic_without_callback_transcript_partial() {
let (_session, lease) = lease(Vec::new());
lease.admit().unwrap();
let call = model_tool_call("call", "tool");
let model = FakeModel::ready(tool_generation(&[call]));
let executions = Arc::new(AtomicUsize::new(0));
let accepted_calls = Arc::new(AtomicUsize::new(0));
let acceptance_at_execute = Arc::new(Mutex::new(Vec::new()));
let tools = executable_tools(
&["tool"],
ExecutorMode::Success,
false,
&executions,
&accepted_calls,
&acceptance_at_execute,
);
let active = Arc::new(AtomicUsize::new(0));
let mut hook_builder = hook::RegistryBuilder::new();
hook_builder.register(
hook::Registration::new("panicking-hook".to_owned()).tool(Arc::new(PanicAfterEmitHooks {
active: Arc::clone(&active),
})),
);
let hooks = hook_builder.build_runtime().unwrap();
let reservations = ReservationResources::new(hooks.len(), lease.state().reader());
let coord = generation_coord();
let emitter = SilentEmitter;
let controller = TranscriptController::new(&lease, &hooks, &reservations, &coord, &emitter);
let run_state = run_state::Store::new();
let reader = lease.state().reader();
let callback_resources = [CallbackResources::new(
&coord, &run_state, &reader, None, &emitter,
)];
let cancellation = Token::new();
let (producer, mut source) = channel(&cancellation);
let mut stream = source.take_stream().unwrap();
let facts = FactBuffer::default();
let outward = Outward::new(&producer, &facts);
let phase = ToolPhase::new(
outward,
&hooks,
&callback_resources,
&controller,
&coord,
None,
);
let inbox = InputMailbox::new();
let mut driver =
Driver::new(&model, &lease, &inbox, &tools, &cancellation).with_tool_phase(phase);
let accepted_after = Arc::new(AtomicUsize::new(0));
let consumer_after = Arc::clone(&accepted_after);
let consumer_accepted = Arc::clone(&accepted_calls);
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_time()
.build()
.unwrap();
let panic = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
runtime.block_on(async {
let consumer = async {
expect_generation_prelude(&mut stream, true).await;
let call = stream.recv().await.expect("tool call");
consumer_accepted.fetch_add(1, Ordering::SeqCst);
call.accept().unwrap();
stream.recv().await.expect("tool result").accept().unwrap();
let after = stream.recv().await.expect("accepted event before panic");
assert_eq!(
after.event().as_audit().unwrap().kind(),
"accepted-before-panic"
);
after.accept().unwrap();
consumer_after.fetch_add(1, Ordering::SeqCst);
};
let _ = tokio::join!(driver.advance(RequestPlan::new()), consumer);
});
}))
.expect_err("the real hook panic must propagate");
let attributed = panic
.downcast::<AfterToolHookPanic>()
.expect("real hook panic is distinct from the private abort sentinel");
assert_eq!(attributed.hook(), "panicking-hook");
assert!(attributed.payload_is::<&'static str>());
assert_eq!(accepted_after.load(Ordering::SeqCst), 1);
assert_eq!(
active.load(Ordering::SeqCst),
0,
"blocking owner was joined"
);
let snapshot = lease.transcript_snapshot().unwrap();
assert_eq!(snapshot.version(), Version::from_u64(2));
assert_eq!(snapshot.items().len(), 2);
assert!(snapshot.items().iter().all(|item| {
!matches!(item, Item::System(system) if system.text() == "must-not-commit")
}));
}
struct InputAdmitHooks {
label: &'static str,
calls: Arc<Mutex<Vec<String>>>,
decision: InputAdmitDecision,
rewrite: bool,
cancel_token: Option<Token>,
}
impl InputHooks for InputAdmitHooks {
fn before_input_admit(&self, scope: &InputAdmitScope<'_>) -> InputAdmitDecision {
let seen = scope
.items()
.iter()
.filter_map(|item| match item {
Item::User(user) => Some(
user.parts()
.iter()
.map(|part| part.fields().text().to_owned())
.collect::<Vec<_>>()
.join("|"),
),
_ => None,
})
.collect::<Vec<_>>()
.join(",");
self.calls
.lock()
.unwrap()
.push(format!("{}:{:?}:{seen}", self.label, scope.target()));
let _ = scope.coord();
let _ = scope.run_state();
let _ = scope.session_state();
let _ = scope.session_state_writer();
if let Some(token) = &self.cancel_token {
token.cancel();
}
match &self.decision {
InputAdmitDecision::Modify(_) if self.rewrite => {
let rewritten = scope
.items()
.iter()
.cloned()
.map(|item| match item {
Item::User(user) => {
let parts = user
.parts()
.iter()
.map(|part| {
let mut text = part.fields().text().to_owned();
text.insert_str(0, "tag:");
Part::text(text)
})
.collect();
Item::User(User::new(parts, user.visibility()))
}
other => other,
})
.collect();
InputAdmitDecision::Modify(rewritten)
}
InputAdmitDecision::Modify(replacement) => {
InputAdmitDecision::Modify(replacement.clone())
}
InputAdmitDecision::Admit => InputAdmitDecision::Admit,
InputAdmitDecision::Reject(reason) => InputAdmitDecision::Reject(reason.clone()),
}
}
}
fn input_hook_registration(
label: &'static str,
calls: &Arc<Mutex<Vec<String>>>,
decision: InputAdmitDecision,
rewrite: bool,
cancel_token: Option<Token>,
) -> hook::Registration {
hook::Registration::new(label.to_owned()).input(Arc::new(InputAdmitHooks {
label,
calls: Arc::clone(calls),
decision,
rewrite,
cancel_token,
}))
}
fn assert_restored(inbox: &InputMailbox, target: InputTarget, text: &str) {
let queued = inbox.drain();
assert_eq!(
queued.len(),
1,
"the claimed batch is restored, not destroyed"
);
assert_eq!(queued[0].target(), target);
assert!(
matches!(&queued[0].items(), [Item::User(user)] if user.parts()[0].fields().text() == text),
"the restored batch is the original, unrewritten one"
);
}
fn admit_claimed(
harness: &InputAdmitHarness,
inbox: &InputMailbox,
) -> Result<InputAdmitOutcome, Error> {
let controller = harness.controller();
let callback_resources = harness.callback_resources();
let (producer, _source) = channel(&harness.cancellation);
let facts = FactBuffer::default();
let outward = Outward::new(&producer, &facts);
let phase = ToolPhase::new(
outward,
&harness.hooks,
&callback_resources,
&controller,
&harness.coord,
None,
);
let mut claim = inbox
.claim_conserved(InputTarget::NextTurn)
.expect("the test queued a NextTurn batch");
phase.admit_input(&mut claim, &harness.cancellation)
}
struct PanicOnPublish;
impl TranscriptHooks for PanicOnPublish {
fn reserve(
&self,
_scope: &ReadScope<'_>,
_mutation: &TranscriptMutation,
) -> HookResult<Box<dyn TranscriptReservation>> {
Ok(Box::new(PanickingPublish))
}
}
struct PanickingPublish;
impl TranscriptReservation for PanickingPublish {
fn publish(self: Box<Self>) {
panic!("publisher panicked after commit");
}
}
struct PanicOnReserve;
impl TranscriptHooks for PanicOnReserve {
fn reserve(
&self,
_scope: &ReadScope<'_>,
_mutation: &TranscriptMutation,
) -> HookResult<Box<dyn TranscriptReservation>> {
panic!("reservation panicked before commit");
}
}
struct RefuseReservation;
impl TranscriptHooks for RefuseReservation {
fn reserve(
&self,
_scope: &ReadScope<'_>,
_mutation: &TranscriptMutation,
) -> HookResult<Box<dyn TranscriptReservation>> {
Err(HookError::new(FakeError("reservation refused")))
}
}
struct PanicOnAdmit;
impl InputHooks for PanicOnAdmit {
fn before_input_admit(&self, _scope: &InputAdmitScope<'_>) -> InputAdmitDecision {
panic!("input hook panicked");
}
}
#[test]
fn a_publisher_panic_after_commit_does_not_restore_the_committed_batch() {
let harness = InputAdmitHarness::new(
Vec::new(),
vec![
hook::Registration::new("panic-publish".to_owned())
.transcript(Arc::new(PanicOnPublish)),
],
);
let inbox = InputMailbox::new();
inbox.push(InputBatch::new(InputTarget::NextTurn, vec![user("steer")]).unwrap());
let caught = std::panic::catch_unwind(AssertUnwindSafe(|| admit_claimed(&harness, &inbox)));
assert!(caught.is_err(), "the publisher panic propagates");
assert!(inbox.is_empty(), "a committed batch is never restored");
let snapshot = harness.lease.transcript_snapshot().unwrap();
assert_eq!(snapshot.version(), Version::from_u64(1));
assert!(
matches!(&snapshot.items(), [Item::User(user)] if user.parts()[0].fields().text() == "steer")
);
}
#[test]
fn a_reservation_panic_before_commit_restores_the_batch() {
let harness = InputAdmitHarness::new(
Vec::new(),
vec![
hook::Registration::new("panic-reserve".to_owned())
.transcript(Arc::new(PanicOnReserve)),
],
);
let inbox = InputMailbox::new();
inbox.push(InputBatch::new(InputTarget::NextTurn, vec![user("steer")]).unwrap());
let caught = std::panic::catch_unwind(AssertUnwindSafe(|| admit_claimed(&harness, &inbox)));
assert!(caught.is_err());
assert_eq!(
harness.lease.transcript_snapshot().unwrap().version(),
Version::ZERO
);
assert_restored(&inbox, InputTarget::NextTurn, "steer");
}
#[test]
fn an_input_hook_panic_restores_the_batch() {
let harness = InputAdmitHarness::new(
Vec::new(),
vec![hook::Registration::new("panic-admit".to_owned()).input(Arc::new(PanicOnAdmit))],
);
let inbox = InputMailbox::new();
inbox.push(InputBatch::new(InputTarget::NextTurn, vec![user("steer")]).unwrap());
let caught = std::panic::catch_unwind(AssertUnwindSafe(|| admit_claimed(&harness, &inbox)));
assert!(caught.is_err());
assert_restored(&inbox, InputTarget::NextTurn, "steer");
}
#[test]
fn a_rewritten_batch_whose_append_fails_restores_the_original() {
let calls = Arc::new(Mutex::new(Vec::new()));
let harness = InputAdmitHarness::new(
Vec::new(),
vec![
input_hook_registration(
"rewrite",
&calls,
InputAdmitDecision::Modify(Vec::new()),
true,
None,
),
hook::Registration::new("refuse".to_owned()).transcript(Arc::new(RefuseReservation)),
],
);
let inbox = InputMailbox::new();
inbox.push(InputBatch::new(InputTarget::NextTurn, vec![user("steer")]).unwrap());
let result = admit_claimed(&harness, &inbox);
assert!(matches!(
result,
Err(Error::InputAdmitTranscript(ControllerError::Dispatch(_)))
));
assert_eq!(*calls.lock().unwrap(), ["rewrite:NextTurn:steer"]);
assert_restored(&inbox, InputTarget::NextTurn, "steer");
}
#[test]
fn a_refused_batch_is_settled_rather_than_restored() {
let calls = Arc::new(Mutex::new(Vec::new()));
let harness = InputAdmitHarness::new(
Vec::new(),
vec![input_hook_registration(
"reject",
&calls,
InputAdmitDecision::Reject("no".to_owned()),
false,
None,
)],
);
let inbox = InputMailbox::new();
inbox.push(InputBatch::new(InputTarget::NextTurn, vec![user("steer")]).unwrap());
let result = admit_claimed(&harness, &inbox);
assert!(matches!(result, Ok(InputAdmitOutcome::Rejected(reason)) if reason == "no"));
assert!(inbox.is_empty());
}
struct InputAdmitHarness {
_session: Session,
lease: Lease,
hooks: hook::RuntimeRegistry,
reservations: ReservationResources,
run_state: run_state::Store,
reader: crate::state::session_state::Reader,
coord: Coord,
emitter: SilentEmitter,
cancellation: Token,
}
impl InputAdmitHarness {
fn new(seed: Vec<Item>, registrations: Vec<hook::Registration>) -> Self {
Self::with_cancellation(seed, registrations, Token::new())
}
fn with_cancellation(
seed: Vec<Item>,
registrations: Vec<hook::Registration>,
cancellation: Token,
) -> Self {
let (session, lease) = lease(seed);
lease.admit().unwrap();
let mut builder = hook::RegistryBuilder::new();
for registration in registrations {
builder.register(registration);
}
let hooks = builder.build_runtime().unwrap();
let reservations = ReservationResources::new(hooks.len(), lease.state().reader());
let coord = generation_coord();
let run_state = run_state::Store::new();
let reader = lease.state().reader();
let emitter = SilentEmitter;
Self {
_session: session,
lease,
hooks,
reservations,
run_state,
reader,
coord,
emitter,
cancellation,
}
}
fn callback_resources(&self) -> Vec<CallbackResources<'_>> {
(0..self.hooks.len())
.map(|_| {
CallbackResources::new(
&self.coord,
&self.run_state,
&self.reader,
None,
&self.emitter,
)
})
.collect()
}
fn controller(&self) -> TranscriptController<'_> {
TranscriptController::new(
&self.lease,
&self.hooks,
&self.reservations,
&self.coord,
&self.emitter,
)
}
}
#[tokio::test]
async fn a_user_only_record_admitted_through_the_mailbox_never_reaches_the_model() {
let harness = InputAdmitHarness::new(Vec::new(), Vec::new());
let hidden = Item::System(System::new(
"todo".to_owned(),
"render this, do not send it".to_owned(),
ModelVisibility::Hidden,
));
let inbox = InputMailbox::new();
inbox.push(InputBatch::new(InputTarget::NextStep, vec![hidden, user("ask")]).unwrap());
let model = FakeModel::scripted(vec![
Start::Ready(Ok(FakeEvents::ready(vec![Ok(done())]))),
Start::Ready(Ok(FakeEvents::ready(vec![Ok(done())]))),
]);
let tools = empty_tools();
let controller = harness.controller();
let callback_resources = harness.callback_resources();
let (producer, _source) = channel(&harness.cancellation);
let facts = FactBuffer::default();
let outward = Outward::new(&producer, &facts);
let phase = ToolPhase::new(
outward,
&harness.hooks,
&callback_resources,
&controller,
&harness.coord,
None,
);
let mut driver = Driver::new(
&model,
&harness.lease,
&inbox,
&tools,
&harness.cancellation,
)
.with_tool_phase(phase);
expect_generation(driver.advance(RequestPlan::new()).await);
expect_advance(driver.advance(RequestPlan::new()).await);
expect_generation(driver.advance(RequestPlan::new()).await);
let snapshot = harness.lease.transcript_snapshot().unwrap();
assert_eq!(snapshot.items().len(), 2, "both records are recorded");
assert!(matches!(&snapshot.items()[0], Item::System(item)
if item.subtype() == "todo" && item.visibility() == ModelVisibility::Hidden));
assert!(matches!(&snapshot.items()[1], Item::User(item)
if item.visibility() == ModelVisibility::Visible));
let requests = model.requests.lock().unwrap();
assert_eq!(requests.len(), 2);
assert!(
requests
.iter()
.flat_map(|request| request.messages())
.all(|item| !matches!(item, Item::System(_))),
"no request carries the user-only record"
);
assert!(
requests[1]
.messages()
.iter()
.any(|item| matches!(item, Item::User(item)
if item.parts()[0].fields().text() == "ask")),
"the model-visible half of the same batch arrives"
);
}
#[tokio::test]
async fn next_step_input_is_admitted_rewritten_and_seen_exactly_once() {
let calls = Arc::new(Mutex::new(Vec::new()));
let harness = InputAdmitHarness::new(
Vec::new(),
vec![input_hook_registration(
"tag",
&calls,
InputAdmitDecision::Modify(vec![user("placeholder")]),
true,
None,
)],
);
let inbox = InputMailbox::new();
inbox.push(InputBatch::new(InputTarget::NextStep, vec![user("steer")]).unwrap());
let model = FakeModel::scripted(vec![
Start::Ready(Ok(FakeEvents::ready(vec![Ok(done())]))),
Start::Ready(Ok(FakeEvents::ready(vec![Ok(done())]))),
]);
let tools = empty_tools();
let controller = harness.controller();
let callback_resources = harness.callback_resources();
let (producer, _source) = channel(&harness.cancellation);
let facts = FactBuffer::default();
let outward = Outward::new(&producer, &facts);
let phase = ToolPhase::new(
outward,
&harness.hooks,
&callback_resources,
&controller,
&harness.coord,
None,
);
let mut driver = Driver::new(
&model,
&harness.lease,
&inbox,
&tools,
&harness.cancellation,
)
.with_tool_phase(phase);
expect_generation(driver.advance(RequestPlan::new()).await);
let input = expect_advance(driver.advance(RequestPlan::new()).await);
let (target, text) = input_text(input);
assert_eq!(target, InputTarget::NextStep);
assert_eq!(
text, "tag:steer",
"the rewrite is visible on the returned batch"
);
expect_generation(driver.advance(RequestPlan::new()).await);
let snapshot = harness.lease.transcript_snapshot().unwrap();
assert_eq!(snapshot.version(), Version::from_u64(1));
assert_eq!(snapshot.items().len(), 1);
assert!(matches!(&snapshot.items()[0], Item::User(user)
if user.parts()[0].fields().text() == "tag:steer"));
let requests = model.requests.lock().unwrap();
let last = requests.last().expect("the steer generation ran");
assert!(
last.messages()
.iter()
.any(|item| matches!(item, Item::User(user)
if user.parts()[0].fields().text() == "tag:steer")),
"the next model Request reads the appended steer content exactly once"
);
drop(requests);
assert!(inbox.is_empty());
assert_eq!(model.calls(), 2);
}
#[tokio::test]
async fn next_turn_input_is_admitted_and_read_by_next_generation_exactly_once() {
let calls = Arc::new(Mutex::new(Vec::new()));
let harness = InputAdmitHarness::new(
Vec::new(),
vec![input_hook_registration(
"admit",
&calls,
InputAdmitDecision::Admit,
false,
None,
)],
);
let inbox = InputMailbox::new();
inbox.push(InputBatch::new(InputTarget::NextTurn, vec![user("turn-steer")]).unwrap());
let model = FakeModel::scripted(vec![
Start::Ready(Ok(FakeEvents::ready(vec![Ok(done())]))),
Start::Ready(Ok(FakeEvents::ready(vec![Ok(done())]))),
]);
let tools = empty_tools();
let controller = harness.controller();
let callback_resources = harness.callback_resources();
let (producer, _source) = channel(&harness.cancellation);
let facts = FactBuffer::default();
let outward = Outward::new(&producer, &facts);
let phase = ToolPhase::new(
outward,
&harness.hooks,
&callback_resources,
&controller,
&harness.coord,
None,
);
let mut driver = Driver::new(
&model,
&harness.lease,
&inbox,
&tools,
&harness.cancellation,
)
.with_tool_phase(phase);
expect_generation(driver.advance(RequestPlan::new()).await);
let input = expect_advance(driver.advance(RequestPlan::new()).await);
let (target, text) = input_text(input);
assert_eq!(target, InputTarget::NextTurn);
assert_eq!(text, "turn-steer");
expect_generation(driver.advance(RequestPlan::new()).await);
assert!(expect_advance(driver.advance(RequestPlan::new()).await).is_none());
let snapshot = harness.lease.transcript_snapshot().unwrap();
assert_eq!(snapshot.version(), Version::from_u64(1));
assert_eq!(snapshot.items().len(), 1);
assert!(matches!(&snapshot.items()[0], Item::User(user)
if user.parts()[0].fields().text() == "turn-steer"));
assert!(inbox.is_empty());
assert_eq!(model.calls(), 2);
}
#[tokio::test]
async fn rejected_input_suppresses_the_extra_generation_without_append() {
let calls = Arc::new(Mutex::new(Vec::new()));
let harness = InputAdmitHarness::new(
Vec::new(),
vec![input_hook_registration(
"reject",
&calls,
InputAdmitDecision::Reject("not allowed".to_owned()),
false,
None,
)],
);
let inbox = InputMailbox::new();
inbox.push(InputBatch::new(InputTarget::NextTurn, vec![user("steer")]).unwrap());
let model = FakeModel::ready(vec![Ok(done())]);
let tools = empty_tools();
let controller = harness.controller();
let callback_resources = harness.callback_resources();
let (producer, _source) = channel(&harness.cancellation);
let facts = FactBuffer::default();
let outward = Outward::new(&producer, &facts);
let phase = ToolPhase::new(
outward,
&harness.hooks,
&callback_resources,
&controller,
&harness.coord,
None,
);
let mut driver = Driver::new(
&model,
&harness.lease,
&inbox,
&tools,
&harness.cancellation,
)
.with_tool_phase(phase);
expect_generation(driver.advance(RequestPlan::new()).await);
assert!(expect_advance(driver.advance(RequestPlan::new()).await).is_none());
let snapshot = harness.lease.transcript_snapshot().unwrap();
assert_eq!(snapshot.version(), Version::ZERO);
assert!(
snapshot.items().is_empty(),
"a rejected batch never appends"
);
assert!(
inbox.is_empty(),
"a rejected batch is settled, not restored"
);
assert_eq!(
model.calls(),
1,
"rejection suppresses the extra generation"
);
assert_eq!(*calls.lock().unwrap(), ["reject:NextTurn:steer"]);
}
#[tokio::test]
async fn rejected_next_step_input_falls_through_to_the_turn_boundary() {
let calls = Arc::new(Mutex::new(Vec::new()));
let harness = InputAdmitHarness::new(
Vec::new(),
vec![input_hook_registration(
"reject-step",
&calls,
InputAdmitDecision::Reject("not allowed".to_owned()),
false,
None,
)],
);
let inbox = InputMailbox::new();
inbox.push(InputBatch::new(InputTarget::NextStep, vec![user("step-steer")]).unwrap());
let model = FakeModel::ready(vec![Ok(done())]);
let tools = empty_tools();
let controller = harness.controller();
let callback_resources = harness.callback_resources();
let (producer, _source) = channel(&harness.cancellation);
let facts = FactBuffer::default();
let outward = Outward::new(&producer, &facts);
let phase = ToolPhase::new(
outward,
&harness.hooks,
&callback_resources,
&controller,
&harness.coord,
None,
);
let mut driver = Driver::new(
&model,
&harness.lease,
&inbox,
&tools,
&harness.cancellation,
)
.with_tool_phase(phase);
expect_generation(driver.advance(RequestPlan::new()).await);
assert!(
expect_advance(driver.advance(RequestPlan::new()).await).is_none(),
"the rejected Step batch falls through to a Turn boundary with no input"
);
let snapshot = harness.lease.transcript_snapshot().unwrap();
assert_eq!(snapshot.version(), Version::ZERO);
assert!(
snapshot.items().is_empty(),
"a rejected Step batch never appends"
);
assert!(
inbox.is_empty(),
"a rejected Step batch is settled, not restored"
);
assert_eq!(
model.calls(),
1,
"rejection at the Step boundary opens no extra generation"
);
assert_eq!(*calls.lock().unwrap(), ["reject-step:NextStep:step-steer"]);
}
#[tokio::test]
async fn input_admit_append_cancelled_at_reservation_is_a_cancellation() {
let cancellation = Token::new();
let harness = InputAdmitHarness::with_cancellation(
Vec::new(),
vec![
hook::Registration::new("cancel-at-reserve".to_owned()).transcript(Arc::new(
CancelAtReservation {
cancellation: cancellation.clone(),
},
)),
],
cancellation.clone(),
);
let inbox = InputMailbox::new();
inbox.push(InputBatch::new(InputTarget::NextTurn, vec![user("steer")]).unwrap());
let model = FakeModel::ready(vec![Ok(done())]);
let tools = empty_tools();
let controller = harness.controller();
let callback_resources = harness.callback_resources();
let (producer, _source) = channel(&harness.cancellation);
let facts = FactBuffer::default();
let outward = Outward::new(&producer, &facts);
let phase = ToolPhase::new(
outward,
&harness.hooks,
&callback_resources,
&controller,
&harness.coord,
None,
);
let mut driver = Driver::new(
&model,
&harness.lease,
&inbox,
&tools,
&harness.cancellation,
)
.with_tool_phase(phase);
expect_generation(driver.advance(RequestPlan::new()).await);
let result = driver.advance(RequestPlan::new()).await;
assert!(
matches!(result, Err(Error::Cancelled)),
"a cancelled append is reported as cancellation, not InputAdmitTranscript"
);
let snapshot = harness.lease.transcript_snapshot().unwrap();
assert_eq!(snapshot.version(), Version::ZERO);
assert!(
snapshot.items().is_empty(),
"the aborted transaction leaves zero partial items"
);
assert_restored(&inbox, InputTarget::NextTurn, "steer");
assert_eq!(
model.calls(),
1,
"no extra generation follows a cancelled append"
);
}
struct CancelAtReservation {
cancellation: Token,
}
impl TranscriptHooks for CancelAtReservation {
fn reserve(
&self,
_scope: &ReadScope<'_>,
_mutation: &TranscriptMutation,
) -> HookResult<Box<dyn TranscriptReservation>> {
self.cancellation.cancel();
Ok(Box::new(InertReservation))
}
}
struct InertReservation;
impl TranscriptReservation for InertReservation {
fn publish(self: Box<Self>) {}
}
#[tokio::test]
async fn cancelled_input_admit_fails_closed_with_zero_partial() {
let cancellation = Token::new();
let harness = InputAdmitHarness::with_cancellation(
Vec::new(),
vec![input_hook_registration(
"cancel",
&Arc::new(Mutex::new(Vec::new())),
InputAdmitDecision::Admit,
false,
Some(cancellation.clone()),
)],
cancellation.clone(),
);
let inbox = InputMailbox::new();
inbox.push(InputBatch::new(InputTarget::NextTurn, vec![user("steer")]).unwrap());
let model = FakeModel::ready(vec![Ok(done())]);
let tools = empty_tools();
let controller = harness.controller();
let callback_resources = harness.callback_resources();
let (producer, _source) = channel(&harness.cancellation);
let facts = FactBuffer::default();
let outward = Outward::new(&producer, &facts);
let phase = ToolPhase::new(
outward,
&harness.hooks,
&callback_resources,
&controller,
&harness.coord,
None,
);
let mut driver = Driver::new(
&model,
&harness.lease,
&inbox,
&tools,
&harness.cancellation,
)
.with_tool_phase(phase);
expect_generation(driver.advance(RequestPlan::new()).await);
let result = driver.advance(RequestPlan::new()).await;
assert!(matches!(result, Err(Error::Cancelled)));
let snapshot = harness.lease.transcript_snapshot().unwrap();
assert_eq!(snapshot.version(), Version::ZERO);
assert!(
snapshot.items().is_empty(),
"cancelled admission appends nothing"
);
assert_restored(&inbox, InputTarget::NextTurn, "steer");
}
#[tokio::test]
async fn empty_modify_input_fails_closed_without_partial() {
let harness = InputAdmitHarness::new(
Vec::new(),
vec![input_hook_registration(
"empty",
&Arc::new(Mutex::new(Vec::new())),
InputAdmitDecision::Modify(Vec::new()),
false,
None,
)],
);
let inbox = InputMailbox::new();
inbox.push(InputBatch::new(InputTarget::NextTurn, vec![user("steer")]).unwrap());
let model = FakeModel::ready(vec![Ok(done())]);
let tools = empty_tools();
let controller = harness.controller();
let callback_resources = harness.callback_resources();
let (producer, _source) = channel(&harness.cancellation);
let facts = FactBuffer::default();
let outward = Outward::new(&producer, &facts);
let phase = ToolPhase::new(
outward,
&harness.hooks,
&callback_resources,
&controller,
&harness.coord,
None,
);
let mut driver = Driver::new(
&model,
&harness.lease,
&inbox,
&tools,
&harness.cancellation,
)
.with_tool_phase(phase);
expect_generation(driver.advance(RequestPlan::new()).await);
let result = driver.advance(RequestPlan::new()).await;
assert!(matches!(
result,
Err(Error::Input(InputAdmitError::EmptyModify))
));
let snapshot = harness.lease.transcript_snapshot().unwrap();
assert_eq!(snapshot.version(), Version::ZERO);
assert!(snapshot.items().is_empty());
assert_restored(&inbox, InputTarget::NextTurn, "steer");
}
#[tokio::test]
async fn empty_reject_reason_input_fails_closed() {
let harness = InputAdmitHarness::new(
Vec::new(),
vec![input_hook_registration(
"empty-reject",
&Arc::new(Mutex::new(Vec::new())),
InputAdmitDecision::Reject(String::new()),
false,
None,
)],
);
let inbox = InputMailbox::new();
inbox.push(InputBatch::new(InputTarget::NextTurn, vec![user("steer")]).unwrap());
let model = FakeModel::ready(vec![Ok(done())]);
let tools = empty_tools();
let controller = harness.controller();
let callback_resources = harness.callback_resources();
let (producer, _source) = channel(&harness.cancellation);
let facts = FactBuffer::default();
let outward = Outward::new(&producer, &facts);
let phase = ToolPhase::new(
outward,
&harness.hooks,
&callback_resources,
&controller,
&harness.coord,
None,
);
let mut driver = Driver::new(
&model,
&harness.lease,
&inbox,
&tools,
&harness.cancellation,
)
.with_tool_phase(phase);
expect_generation(driver.advance(RequestPlan::new()).await);
let result = driver.advance(RequestPlan::new()).await;
assert!(matches!(
result,
Err(Error::Input(InputAdmitError::EmptyRejectReason))
));
let snapshot = harness.lease.transcript_snapshot().unwrap();
assert_eq!(snapshot.version(), Version::ZERO);
assert_restored(&inbox, InputTarget::NextTurn, "steer");
}
#[tokio::test]
async fn input_admit_transcript_failure_fails_closed_zero_partial() {
let harness = InputAdmitHarness::new(Vec::new(), Vec::new());
let inbox = InputMailbox::new();
inbox.push(InputBatch::new(InputTarget::NextTurn, vec![user("steer")]).unwrap());
let model = FakeModel::ready(vec![Ok(done())]);
let tools = empty_tools();
let controller = harness.controller();
let callback_resources = harness.callback_resources();
let (producer, _source) = channel(&harness.cancellation);
let facts = FactBuffer::default();
let outward = Outward::new(&producer, &facts);
let phase = ToolPhase::new(
outward,
&harness.hooks,
&callback_resources,
&controller,
&harness.coord,
None,
);
let mut driver = Driver::new(
&model,
&harness.lease,
&inbox,
&tools,
&harness.cancellation,
)
.with_tool_phase(phase);
expect_generation(driver.advance(RequestPlan::new()).await);
harness.lease.release();
let result = driver.advance(RequestPlan::new()).await;
assert!(matches!(
result,
Err(Error::InputAdmitTranscript(ControllerError::Lease(
LeaseError::Released
)))
));
assert_eq!(
model.calls(),
1,
"no extra generation after a failed append"
);
assert_restored(&inbox, InputTarget::NextTurn, "steer");
let snapshot = harness._session.transcript().snapshot().unwrap();
assert_eq!(snapshot.version(), Version::ZERO);
assert!(snapshot.items().is_empty());
}
#[tokio::test]
async fn input_admit_rewrites_chain_in_registration_order() {
let calls = Arc::new(Mutex::new(Vec::new()));
let harness = InputAdmitHarness::new(
Vec::new(),
vec![
input_hook_registration(
"first",
&calls,
InputAdmitDecision::Modify(vec![user("placeholder")]),
true,
None,
),
input_hook_registration(
"second",
&calls,
InputAdmitDecision::Modify(vec![user("placeholder")]),
true,
None,
),
],
);
let inbox = InputMailbox::new();
inbox.push(InputBatch::new(InputTarget::NextStep, vec![user("steer")]).unwrap());
let model = FakeModel::scripted(vec![
Start::Ready(Ok(FakeEvents::ready(vec![Ok(done())]))),
Start::Ready(Ok(FakeEvents::ready(vec![Ok(done())]))),
]);
let tools = empty_tools();
let controller = harness.controller();
let callback_resources = harness.callback_resources();
let (producer, _source) = channel(&harness.cancellation);
let facts = FactBuffer::default();
let outward = Outward::new(&producer, &facts);
let phase = ToolPhase::new(
outward,
&harness.hooks,
&callback_resources,
&controller,
&harness.coord,
None,
);
let mut driver = Driver::new(
&model,
&harness.lease,
&inbox,
&tools,
&harness.cancellation,
)
.with_tool_phase(phase);
expect_generation(driver.advance(RequestPlan::new()).await);
let input = expect_advance(driver.advance(RequestPlan::new()).await);
let (_, text) = input_text(input);
assert_eq!(
text, "tag:tag:steer",
"each registration chains onto the previous rewrite"
);
let observed = calls.lock().unwrap().clone();
assert_eq!(
observed,
[
"first:NextStep:steer".to_owned(),
"second:NextStep:tag:steer".to_owned()
]
);
let snapshot = harness.lease.transcript_snapshot().unwrap();
assert_eq!(snapshot.items().len(), 1);
assert!(matches!(&snapshot.items()[0], Item::User(user)
if user.parts()[0].fields().text() == "tag:tag:steer"));
}
#[tokio::test]
async fn owner_run_admits_step_and_turn_inputs_into_the_transcript() {
let calls = Arc::new(Mutex::new(Vec::new()));
let harness = InputAdmitHarness::new(
Vec::new(),
vec![input_hook_registration(
"admit",
&calls,
InputAdmitDecision::Admit,
false,
None,
)],
);
let inbox = InputMailbox::new();
inbox.push(InputBatch::new(InputTarget::NextStep, vec![user("step")]).unwrap());
inbox.push(InputBatch::new(InputTarget::NextTurn, vec![user("turn")]).unwrap());
let model = FakeModel::scripted(vec![
Start::Ready(Ok(FakeEvents::ready(vec![Ok(done())]))),
Start::Ready(Ok(FakeEvents::ready(vec![Ok(done())]))),
Start::Ready(Ok(FakeEvents::ready(vec![Ok(done())]))),
]);
let tools = empty_tools();
let controller = harness.controller();
let callback_resources = harness.callback_resources();
let (producer, source) = channel(&harness.cancellation);
let facts = FactBuffer::default();
let outward = Outward::new(&producer, &facts);
let phase = ToolPhase::new(
outward,
&harness.hooks,
&callback_resources,
&controller,
&harness.coord,
None,
);
let driver = Driver::new(
&model,
&harness.lease,
&inbox,
&tools,
&harness.cancellation,
)
.with_tool_phase(phase);
let terminal = driver.run(RequestPlan::new(), &source).await;
assert!(matches!(terminal, Terminal::Completed));
assert_eq!(model.calls(), 3, "initial + step + turn generations");
let snapshot = harness.lease.transcript_snapshot().unwrap();
assert_eq!(snapshot.version(), Version::from_u64(2));
assert_eq!(snapshot.items().len(), 2);
assert!(matches!(&snapshot.items()[0], Item::User(user)
if user.parts()[0].fields().text() == "step"));
assert!(matches!(&snapshot.items()[1], Item::User(user)
if user.parts()[0].fields().text() == "turn"));
assert!(inbox.is_empty());
assert_eq!(
*calls.lock().unwrap(),
[
"admit:NextStep:step".to_owned(),
"admit:NextTurn:turn".to_owned()
]
);
}
#[derive(Clone, Copy)]
enum ReturnedReservationInterruption {
Fail,
Cancel,
Panic,
}
#[derive(Clone)]
struct BoundaryPlan {
before_turn: Vec<Item>,
after_turn: Vec<Item>,
before_step: Vec<Item>,
after_step: AfterStepAnswer,
before_model: BeforeModelAnswer,
after_model: Option<Assistant>,
decide_stop: Option<StopCode>,
stop_after_decisions: usize,
compose_continuation: ContinuationAnswer,
failing: Option<&'static str>,
staged: Option<Item>,
cancel_at: Option<&'static str>,
fail_reservation: Option<usize>,
cancel_reservation: Option<usize>,
cancel_publish_reservation: Option<usize>,
returned_reservation_interruption: Option<ReturnedReservationInterruption>,
reject_next_turn_input: Option<&'static str>,
}
impl Default for BoundaryPlan {
fn default() -> Self {
Self {
before_turn: Vec::new(),
after_turn: Vec::new(),
before_step: Vec::new(),
after_step: AfterStepAnswer::Continue,
before_model: BeforeModelAnswer::Continue,
after_model: None,
decide_stop: None,
stop_after_decisions: 0,
compose_continuation: ContinuationAnswer::Keep,
failing: None,
staged: None,
cancel_at: None,
fail_reservation: None,
cancel_reservation: None,
cancel_publish_reservation: None,
returned_reservation_interruption: None,
reject_next_turn_input: None,
}
}
}
#[derive(Clone, Copy)]
enum AfterStepAnswer {
Continue,
EndTurn,
}
#[derive(Clone)]
enum ContinuationAnswer {
Keep,
Replace(Vec<Part>),
}
#[derive(Clone)]
enum BeforeModelAnswer {
Continue,
Modify,
ShortCircuit(Assistant),
Abort(String),
}
struct BoundaryHooks {
name: &'static str,
plan: BoundaryPlan,
calls: Arc<Mutex<Vec<String>>>,
observations: Arc<Mutex<Vec<String>>>,
model_events: Arc<Mutex<Vec<String>>>,
audits: Arc<Mutex<Vec<String>>>,
cancellation: Token,
reservations: Arc<AtomicUsize>,
stop_decisions: AtomicUsize,
}
impl BoundaryHooks {
fn record(&self, boundary: &str) {
self.calls
.lock()
.expect("boundary log lock")
.push(format!("{}:{boundary}", self.name));
if self.plan.cancel_at == Some(boundary) {
self.cancellation.cancel();
}
}
fn fail<T>(&self, boundary: &str) -> Option<HookResult<T>> {
self.plan
.failing
.filter(|failing| *failing == boundary)
.map(|failing| Err(HookError::new(FakeError(failing))))
}
}
struct BoundaryReservation {
cancel_on_publish: Option<Token>,
}
impl TranscriptReservation for BoundaryReservation {
fn publish(self: Box<Self>) {
if let Some(cancellation) = self.cancel_on_publish {
cancellation.cancel();
}
}
fn abort(self: Box<Self>) {}
}
impl TranscriptHooks for BoundaryHooks {
fn reserve(
&self,
_scope: &ReadScope<'_>,
mutation: &TranscriptMutation,
) -> HookResult<Box<dyn TranscriptReservation>> {
let index = self.reservations.fetch_add(1, Ordering::SeqCst);
if self.plan.fail_reservation == Some(index) {
return Err(HookError::new(FakeError("reservation refused")));
}
if self.plan.cancel_reservation == Some(index) {
self.cancellation.cancel();
}
let contains_returned = mutation
.items()
.iter()
.any(|item| matches!(item, Item::System(system) if system.text() == "returned"));
if contains_returned {
match self.plan.returned_reservation_interruption {
Some(ReturnedReservationInterruption::Fail) => {
return Err(HookError::new(FakeError("combined reservation refused")));
}
Some(ReturnedReservationInterruption::Cancel) => self.cancellation.cancel(),
Some(ReturnedReservationInterruption::Panic) => {
panic!("combined reservation panicked");
}
None => {}
}
}
let cancel_on_publish = (self.plan.cancel_publish_reservation == Some(index))
.then(|| self.cancellation.clone());
Ok(Box::new(BoundaryReservation { cancel_on_publish }))
}
}
impl RunHooks for BoundaryHooks {
fn prepare(&self, _scope: &PrepareScope<'_>) -> HookResult<PrepareDecision> {
Ok(PrepareDecision::Continue)
}
fn compose_continuation(
&self,
scope: &ContinuationScope<'_>,
) -> HookResult<ContinuationDecision> {
self.record(&format!(
"compose_continuation(turns={},continuations={},continuation={})",
scope.turns(),
scope.continuations(),
scope.continuation().len()
));
if let Some(failure) = self.fail("compose_continuation") {
return failure;
}
if scope.continuations() > 0 {
return Ok(ContinuationDecision::Keep);
}
Ok(match &self.plan.compose_continuation {
ContinuationAnswer::Keep => ContinuationDecision::Keep,
ContinuationAnswer::Replace(parts) => ContinuationDecision::Replace(parts.clone()),
})
}
fn decide_stop(&self, _scope: &StopDecisionScope<'_>) -> HookResult<StopDecision> {
self.record("decide_stop");
if let Some(failure) = self.fail("decide_stop") {
return failure;
}
let decision = self.stop_decisions.fetch_add(1, Ordering::SeqCst);
if decision < self.plan.stop_after_decisions {
return Ok(StopDecision::Continue);
}
Ok(self
.plan
.decide_stop
.clone()
.map_or(StopDecision::Continue, StopDecision::Stop))
}
}
impl TurnStepHooks for BoundaryHooks {
fn before_turn(&self, scope: &TurnScope<'_>) -> HookResult<Vec<Item>> {
self.record(&format!("before_turn(trigger={})", scope.trigger().len()));
if let Some(failure) = self.fail("before_turn") {
return failure;
}
if let Some(item) = self.plan.staged.clone() {
scope
.staged()
.append_batch(vec![item])
.expect("the staging area accepts an append");
}
Ok(self.plan.before_turn.clone())
}
fn after_turn(&self, scope: &TurnScope<'_>) -> HookResult<Vec<Item>> {
self.record(&format!("after_turn(trigger={})", scope.trigger().len()));
if let Some(failure) = self.fail("after_turn") {
return failure;
}
Ok(self.plan.after_turn.clone())
}
fn before_step(&self, scope: &mut BeforeStepScope<'_>) -> HookResult<Vec<Item>> {
self.record(&format!("before_step(model={})", scope.model().id()));
if let Some(failure) = self.fail("before_step") {
return failure;
}
scope
.plan()
.push_ephemeral(Part::text(format!("{}-ephemeral", self.name)));
Ok(self.plan.before_step.clone())
}
fn after_step(&self, scope: &AfterStepScope<'_>) -> HookResult<AfterStepDecision> {
self.record(&format!(
"after_step(model={})",
scope.snapshot().model().id()
));
if let Some(failure) = self.fail("after_step") {
return failure;
}
Ok(match self.plan.after_step {
AfterStepAnswer::Continue => AfterStepDecision::Continue(Vec::new()),
AfterStepAnswer::EndTurn => AfterStepDecision::EndTurn(Vec::new()),
})
}
}
impl ModelHooks for BoundaryHooks {
fn before_model(&self, scope: &BeforeModelScope<'_>) -> HookResult<BeforeModelDecision> {
self.record(&format!(
"before_model(messages={},ephemeral={})",
scope.request().messages().len(),
scope.request().ephemeral().len()
));
if let Some(failure) = self.fail("before_model") {
return failure;
}
Ok(match &self.plan.before_model {
BeforeModelAnswer::Continue => BeforeModelDecision::Continue,
BeforeModelAnswer::Modify => BeforeModelDecision::Modify(scope.request().clone()),
BeforeModelAnswer::ShortCircuit(output) => {
BeforeModelDecision::ShortCircuit(output.clone())
}
BeforeModelAnswer::Abort(reason) => BeforeModelDecision::Abort(reason.clone()),
})
}
fn after_model(&self, scope: &AfterModelScope<'_>) -> AfterModelDecision {
self.record(&format!(
"after_model(generated={},finish={})",
scope.generated(),
scope.finish_reason()
));
self.plan
.after_model
.clone()
.map_or(AfterModelDecision::Continue, AfterModelDecision::Modify)
}
}
impl InputHooks for BoundaryHooks {
fn before_input_admit(&self, scope: &InputAdmitScope<'_>) -> InputAdmitDecision {
self.record(&format!("before_input_admit(target={:?})", scope.target()));
match self.plan.reject_next_turn_input {
Some(reason) if scope.target() == InputTarget::NextTurn => {
InputAdmitDecision::Reject(reason.to_owned())
}
_ => InputAdmitDecision::Admit,
}
}
}
impl ObserverHooks for BoundaryHooks {
fn run_admitted(&self, _scope: &ReadScope<'_>) {}
fn lifecycle(&self, observation: &LifecycleObservation, _scope: &ObserverScope<'_>) {
self.observations
.lock()
.expect("observation log lock")
.push(
format!(
"{:?}@{} {}",
observation.kind(),
observation.coord(),
observation_detail(observation)
)
.trim_end()
.to_owned(),
);
}
fn model_event(&self, _scope: &ObserverScope<'_>, event: &model::Event) {
self.model_events
.lock()
.expect("model event log lock")
.push(match event {
model::Event::ContentDelta { kind, .. } => format!("delta:{kind:?}"),
model::Event::ContentFinal { kind, .. } => format!("final:{kind:?}"),
model::Event::ToolCallFinal { call, .. } => {
format!("call:{}", call.id().as_str())
}
model::Event::Usage(usage) => format!("usage:{}", usage.total),
model::Event::Done { finish_reason, .. } => format!("done:{finish_reason}"),
model::Event::ToolCallStart { .. }
| model::Event::ToolCallArgumentsDelta { .. } => "call-progress".to_owned(),
});
}
fn audit_event(&self, _scope: &ReadScope<'_>, event: &audit::Event) {
self.audits
.lock()
.expect("audit log lock")
.push(event.kind().to_owned());
}
}
type ContentItem = (Kind, Option<String>);
struct BoundaryProbe {
calls: Arc<Mutex<Vec<String>>>,
observations: Arc<Mutex<Vec<String>>>,
model_events: Arc<Mutex<Vec<String>>>,
audits: Arc<Mutex<Vec<String>>>,
content_items: Arc<Mutex<Vec<ContentItem>>>,
cancellation: Token,
reservations: Arc<AtomicUsize>,
}
impl BoundaryProbe {
fn new() -> Self {
Self {
calls: Arc::new(Mutex::new(Vec::new())),
observations: Arc::new(Mutex::new(Vec::new())),
model_events: Arc::new(Mutex::new(Vec::new())),
audits: Arc::new(Mutex::new(Vec::new())),
content_items: Arc::new(Mutex::new(Vec::new())),
cancellation: Token::new(),
reservations: Arc::new(AtomicUsize::new(0)),
}
}
fn content_items(&self) -> Vec<ContentItem> {
self.content_items
.lock()
.expect("content item log lock")
.clone()
}
fn hooks(&self, name: &'static str, plan: BoundaryPlan) -> Arc<BoundaryHooks> {
Arc::new(BoundaryHooks {
name,
plan,
calls: Arc::clone(&self.calls),
observations: Arc::clone(&self.observations),
model_events: Arc::clone(&self.model_events),
audits: Arc::clone(&self.audits),
cancellation: self.cancellation.clone(),
reservations: Arc::clone(&self.reservations),
stop_decisions: AtomicUsize::new(0),
})
}
fn registry(&self, plans: &[(&'static str, BoundaryPlan)]) -> hook::RuntimeRegistry {
let mut builder = hook::RegistryBuilder::new();
for (name, plan) in plans {
let hooks = self.hooks(name, plan.clone());
builder.register(
hook::Registration::new((*name).to_owned())
.run(Arc::clone(&hooks) as Arc<dyn RunHooks>)
.turn_step(Arc::clone(&hooks) as Arc<dyn TurnStepHooks>)
.model(Arc::clone(&hooks) as Arc<dyn ModelHooks>)
.transcript(Arc::clone(&hooks) as Arc<dyn TranscriptHooks>)
.input(Arc::clone(&hooks) as Arc<dyn InputHooks>)
.observer(hooks as Arc<dyn ObserverHooks>),
);
}
builder.build_runtime().expect("boundary registry builds")
}
fn calls(&self) -> Vec<String> {
self.calls.lock().expect("boundary log lock").clone()
}
fn observation_kinds(&self) -> Vec<String> {
self.observations
.lock()
.expect("observation log lock")
.iter()
.map(|entry| {
entry
.split_once('@')
.expect("observation entries carry a coordinate")
.0
.to_owned()
})
.collect()
}
fn observation_facts(&self) -> Vec<String> {
self.observations
.lock()
.expect("observation log lock")
.iter()
.map(|entry| {
let (kind, rest) = entry
.split_once('@')
.expect("observation entries carry a coordinate");
let detail = rest
.split_once(' ')
.map_or("", |(_coord, detail)| detail)
.to_owned();
if detail.is_empty() {
kind.to_owned()
} else {
format!("{kind} {detail}")
}
})
.collect()
}
fn audits(&self) -> Vec<String> {
self.audits.lock().expect("audit log lock").clone()
}
}
fn observation_detail(observation: &LifecycleObservation) -> String {
match observation {
LifecycleObservation::TurnEnd { end, steps, .. } => {
format!("{} steps={steps}", end_detail(end))
}
LifecycleObservation::StepEnd { end, .. } | LifecycleObservation::RunEnd { end, .. } => {
end_detail(end)
}
LifecycleObservation::GenerationEnd {
end,
finish_reason,
completion,
..
} => format!(
"{} finish={finish_reason:?} completion={completion:?}",
end_detail(end)
),
LifecycleObservation::ToolCallStart { call, .. } => {
format!("call={}", call.id().as_str())
}
LifecycleObservation::ToolCallEnd(ended) => {
let milestone = ended.progress();
format!(
"call={} name={} {} progress={:?} disposition={:?} result={}",
ended.original_call().id().as_str(),
ended.effective_call().name().as_str(),
end_detail(ended.end()),
milestone.kind(),
milestone.disposition(),
milestone.result().is_some(),
)
}
LifecycleObservation::RunStart { .. }
| LifecycleObservation::TurnStart { .. }
| LifecycleObservation::StepStart { .. }
| LifecycleObservation::GenerationStart { .. } => String::new(),
}
}
fn end_detail(end: &LifecycleEnd) -> String {
let mut detail = format!("end={:?}", end.outcome());
if let Some(code) = end.stop_code() {
detail.push_str(&format!(" code={}", code.as_str()));
}
if let Some(error) = end.error() {
detail.push_str(&format!(" error={error}"));
match error.source() {
Some(cause) => detail.push_str(&format!(" cause={cause}")),
None => detail.push_str(" cause=none"),
}
}
detail
}
fn driver_producer_guard(producer: Arc<super::event::Producer>) -> Arc<super::event::Producer> {
producer
}
fn system_item(text: &str) -> Item {
Item::System(System::new(
"boundary".to_owned(),
text.to_owned(),
ModelVisibility::Visible,
))
}
fn text_generation(text: &str) -> Vec<Result<model::Event, FakeError>> {
vec![
Ok(model::Event::ContentDelta {
id: "text".to_owned(),
kind: ContentKind::Text,
delta: text.to_owned(),
}),
Ok(model::Event::ContentFinal {
id: "text".to_owned(),
kind: ContentKind::Text,
content: text.to_owned(),
extensions: extensions(),
}),
Ok(model::Event::Usage(TokenUsage::default())),
Ok(done()),
]
}
fn thinking_generation() -> Vec<Result<model::Event, FakeError>> {
vec![
Ok(model::Event::ContentDelta {
id: "thought".to_owned(),
kind: ContentKind::Thinking,
delta: "considering".to_owned(),
}),
Ok(model::Event::ContentFinal {
id: "thought".to_owned(),
kind: ContentKind::Thinking,
content: "considering".to_owned(),
extensions: extensions(),
}),
Ok(done()),
]
}
async fn run_boundary_driver(
probe: &BoundaryProbe,
hooks: &hook::RuntimeRegistry,
events: Vec<Vec<Result<model::Event, FakeError>>>,
report: &super::driver::Report,
trigger: Vec<Item>,
) -> (Terminal, Vec<Kind>, Snapshot) {
run_boundary_driver_on_session(
probe,
hooks,
events,
report,
trigger,
Session::new("session".to_owned(), None),
)
.await
}
async fn run_boundary_driver_on_session(
probe: &BoundaryProbe,
hooks: &hook::RuntimeRegistry,
events: Vec<Vec<Result<model::Event, FakeError>>>,
report: &super::driver::Report,
trigger: Vec<Item>,
session: Session,
) -> (Terminal, Vec<Kind>, Snapshot) {
let lease = session.try_acquire().expect("fresh session is available");
lease.admit().unwrap();
let model = FakeModel::scripted(
events
.into_iter()
.map(|script| Start::Ready(Ok(FakeEvents::ready(script))))
.collect(),
);
let reservations = ReservationResources::new(hooks.len(), lease.state().reader());
let coord = generation_coord();
let emitter = SilentEmitter;
let controller = TranscriptController::new(&lease, hooks, &reservations, &coord, &emitter);
let states = (0..hooks.len())
.map(|_| run_state::Store::new())
.collect::<Vec<_>>();
let reader = lease.state().reader();
let callback_resources = states
.iter()
.map(|state| CallbackResources::new(&coord, state, &reader, None, &emitter))
.collect::<Vec<_>>();
let cancellation = probe.cancellation.clone();
let (producer, mut source) = channel(&cancellation);
let mut stream = source.take_stream().unwrap();
let facts = FactBuffer::default();
let outward = Outward::new(&producer, &facts);
let phase = ToolPhase::new(
outward,
hooks,
&callback_resources,
&controller,
&coord,
None,
);
let inbox = InputMailbox::new();
let tools = empty_tools();
let driver = Driver::new(&model, &lease, &inbox, &tools, &cancellation)
.with_tool_phase(phase)
.with_turn_trigger(trigger)
.with_report(report);
let terminal = driver.run(RequestPlan::new(), &source).await;
drop(driver_producer_guard(producer));
let mut kinds = Vec::new();
while let Some(delivery) = stream.recv().await {
let kind = delivery.event().kind();
if matches!(
kind,
Kind::TextDelta | Kind::Text | Kind::ThinkingDelta | Kind::Thinking
) {
probe
.content_items
.lock()
.expect("content item log lock")
.push((kind, delivery.event().item_id().map(str::to_owned)));
}
kinds.push(kind);
}
let snapshot = lease.transcript_snapshot().unwrap();
(terminal, kinds, snapshot)
}
#[tokio::test]
async fn a_claimed_batch_is_announced_whether_it_is_recorded_or_refused() {
let probe = BoundaryProbe::new();
let hooks = probe.registry(&[(
"gatekeeper",
BoundaryPlan {
reject_next_turn_input: Some("out of scope"),
..BoundaryPlan::default()
},
)]);
let inbox = InputMailbox::new();
inbox.push(InputBatch::new(InputTarget::NextStep, vec![user("keep going")]).unwrap());
inbox.push(InputBatch::new(InputTarget::NextTurn, vec![user("unrelated")]).unwrap());
let report = super::driver::Report::default();
let (_session, lease) = lease(Vec::new());
lease.admit().unwrap();
let model = FakeModel::scripted(vec![
Start::Ready(Ok(FakeEvents::ready(text_generation("first")))),
Start::Ready(Ok(FakeEvents::ready(text_generation("second")))),
]);
let reservations = ReservationResources::new(hooks.len(), lease.state().reader());
let coord = generation_coord();
let facts = FactBuffer::default();
let controller = TranscriptController::new(&lease, &hooks, &reservations, &coord, &facts);
let states = [run_state::Store::new()];
let reader = lease.state().reader();
let callback_resources = [CallbackResources::new(
&coord, &states[0], &reader, None, &facts,
)];
let cancellation = probe.cancellation.clone();
let (producer, mut source) = channel(&cancellation);
let mut stream = source.take_stream().unwrap();
let phase = ToolPhase::new(
Outward::new(&producer, &facts),
&hooks,
&callback_resources,
&controller,
&coord,
None,
);
let tools = empty_tools();
let terminal = Driver::new(&model, &lease, &inbox, &tools, &cancellation)
.with_tool_phase(phase)
.with_report(&report)
.run(RequestPlan::new(), &source)
.await;
drop(driver_producer_guard(producer));
let mut announced = Vec::new();
let mut kinds = Vec::new();
while let Some(delivery) = stream.recv().await {
kinds.push(delivery.event().kind());
if let Some(input) = delivery.event().admitted_input() {
announced.push(format!("{:?}:{}", input.origin(), input.items().len()));
}
if let Some(reason) = delivery.event().rejection_reason() {
announced.push(format!("Rejected:{reason}"));
}
}
assert!(matches!(terminal, Terminal::Completed));
assert_eq!(
announced,
["Mailbox:1", "Rejected:out of scope"],
"the recorded batch and the refused one are both reported"
);
assert_eq!(
kinds,
[
Kind::TurnStart,
Kind::StepStart,
Kind::TextDelta,
Kind::Text,
Kind::GenerationEnd,
Kind::StepEnd,
Kind::InputAdmitted,
Kind::StepStart,
Kind::TextDelta,
Kind::Text,
Kind::GenerationEnd,
Kind::StepEnd,
Kind::TurnEnd,
Kind::InputRejected,
]
);
assert_eq!(
lease
.transcript_snapshot()
.unwrap()
.items()
.iter()
.filter(|item| matches!(item, Item::User(_)))
.count(),
1
);
}
#[tokio::test]
async fn mailbox_commit_survives_interruption_without_input_admitted() {
let probe = BoundaryProbe::new();
let hooks = probe.registry(&[(
"cancel-after-mailbox-commit",
BoundaryPlan {
cancel_publish_reservation: Some(1),
..BoundaryPlan::default()
},
)]);
let inbox = InputMailbox::new();
inbox.push(InputBatch::new(InputTarget::NextTurn, vec![user("committed steer")]).unwrap());
let report = super::driver::Report::default();
let (terminal, kinds, snapshot) = run_with_tools(
&probe,
&hooks,
&empty_tools(),
&inbox,
vec![text_generation("first"), text_generation("must not run")],
&report,
accept_everything,
)
.await;
assert!(matches!(terminal, Terminal::Cancelled));
assert_eq!(report.turns(), 1, "interruption prevents the next Turn");
assert!(
inbox.is_empty(),
"the committed batch was claimed exactly once"
);
assert!(
!kinds.contains(&Kind::InputAdmitted),
"ordinary admission projection fails closed after interruption"
);
assert!(
snapshot.items().iter().any(
|item| matches!(item, Item::User(user) if user.parts()[0].fields().text() == "committed steer")
),
"the transcript remains the authoritative committed state"
);
}
#[tokio::test]
async fn a_restored_batch_is_admitted_exactly_once_by_a_successor_run() {
let session = Session::new("conserve".to_owned(), None);
let inbox = session.input_mailbox();
inbox.push(InputBatch::new(InputTarget::NextTurn, vec![user("conserved")]).unwrap());
let failing = BoundaryProbe::new();
let failing_hooks = failing.registry(&[(
"refuse-input-commit",
BoundaryPlan {
fail_reservation: Some(1),
..BoundaryPlan::default()
},
)]);
let (terminal, _kinds, snapshot) = run_with_tools_on_session(
&failing,
&failing_hooks,
&empty_tools(),
&inbox,
vec![text_generation("first"), text_generation("must not run")],
&super::driver::Report::default(),
accept_everything,
session.clone(),
)
.await;
assert!(matches!(
terminal.failure(),
Some(Error::InputAdmitTranscript(_))
));
assert!(
!snapshot
.items()
.iter()
.any(|item| matches!(item, Item::User(_))),
"the refused append commits nothing"
);
assert_eq!(inbox.len(), 1, "the batch is back in the Session mailbox");
let successor = BoundaryProbe::new();
let successor_hooks = successor.registry(&[("plain", BoundaryPlan::default())]);
let (terminal, kinds, snapshot) = run_with_tools_on_session(
&successor,
&successor_hooks,
&empty_tools(),
&inbox,
vec![text_generation("second"), text_generation("answer")],
&super::driver::Report::default(),
accept_everything,
session,
)
.await;
assert!(terminal.failure().is_none());
assert!(inbox.is_empty());
assert_eq!(
kinds
.iter()
.filter(|kind| **kind == Kind::InputAdmitted)
.count(),
1
);
assert_eq!(
snapshot
.items()
.iter()
.filter(|item| matches!(item, Item::User(user) if user.parts()[0].fields().text() == "conserved"))
.count(),
1,
"the restored batch is committed exactly once"
);
}
#[tokio::test]
async fn every_content_event_names_the_provider_item_it_belongs_to() {
let script = vec![
Ok(model::Event::ContentDelta {
id: "thought".to_owned(),
kind: ContentKind::Thinking,
delta: "let me ".to_owned(),
}),
Ok(model::Event::ContentDelta {
id: "answer".to_owned(),
kind: ContentKind::Text,
delta: "42".to_owned(),
}),
Ok(model::Event::ContentDelta {
id: "thought".to_owned(),
kind: ContentKind::Thinking,
delta: "think".to_owned(),
}),
Ok(model::Event::ContentFinal {
id: "thought".to_owned(),
kind: ContentKind::Thinking,
content: "let me think".to_owned(),
extensions: extensions(),
}),
Ok(model::Event::ContentFinal {
id: "answer".to_owned(),
kind: ContentKind::Text,
content: "42".to_owned(),
extensions: extensions(),
}),
Ok(done()),
];
let probe = BoundaryProbe::new();
let hooks = empty_hooks();
let report = super::driver::Report::default();
let (terminal, _kinds, snapshot) =
run_boundary_driver(&probe, &hooks, vec![script], &report, Vec::new()).await;
assert!(matches!(terminal, Terminal::Completed));
assert_eq!(
probe.content_items(),
[
(Kind::ThinkingDelta, Some("thought".to_owned())),
(Kind::TextDelta, Some("answer".to_owned())),
(Kind::ThinkingDelta, Some("thought".to_owned())),
(Kind::Thinking, Some("thought".to_owned())),
(Kind::Text, Some("answer".to_owned())),
]
);
assert_eq!(
snapshot
.items()
.iter()
.filter_map(|item| match item {
Item::Assistant(assistant) => Some(assistant),
_ => None,
})
.flat_map(|assistant| assistant.content())
.filter_map(|segment| match segment {
Segment::Text { extensions, .. } | Segment::Thinking { extensions, .. } =>
extensions.item_id(),
Segment::ToolCall { .. } => None,
})
.collect::<Vec<_>>(),
["thought", "answer"]
);
}
struct FactEmittingHooks {
blank_audit_kind: bool,
}
impl ObserverHooks for FactEmittingHooks {
fn lifecycle(&self, observation: &LifecycleObservation, scope: &ObserverScope<'_>) {
if let LifecycleObservation::GenerationEnd { finish_reason, .. } = observation {
scope.emit_audit(audit::Event::new(format!("finish:{finish_reason}"), None));
}
}
fn model_event(&self, scope: &ObserverScope<'_>, event: &model::Event) {
if let model::Event::Usage(usage) = event {
let kind = if self.blank_audit_kind {
String::new()
} else {
format!("usage:{}", usage.total)
};
scope.emit_audit(audit::Event::new(kind, None));
}
}
}
impl TurnStepHooks for FactEmittingHooks {
fn before_turn(&self, _scope: &TurnScope<'_>) -> HookResult<Vec<Item>> {
Ok(Vec::new())
}
fn after_turn(&self, _scope: &TurnScope<'_>) -> HookResult<Vec<Item>> {
Ok(Vec::new())
}
fn before_step(&self, _scope: &mut BeforeStepScope<'_>) -> HookResult<Vec<Item>> {
Ok(Vec::new())
}
fn after_step(&self, scope: &AfterStepScope<'_>) -> HookResult<AfterStepDecision> {
scope.emit_error(Arc::new(FakeError("degraded")));
Ok(AfterStepDecision::Continue(Vec::new()))
}
}
fn fact_emitting_hooks(blank_audit_kind: bool) -> hook::RuntimeRegistry {
let hooks = Arc::new(FactEmittingHooks { blank_audit_kind });
let mut builder = hook::RegistryBuilder::new();
builder.register(
hook::Registration::new("facts".to_owned())
.turn_step(Arc::clone(&hooks) as Arc<dyn TurnStepHooks>)
.observer(hooks as Arc<dyn ObserverHooks>),
);
builder
.build_runtime()
.expect("fact-emitting registry builds")
}
fn event_label(delivery: &Delivery) -> String {
match delivery.event().kind() {
Kind::Audit => format!("audit:{}", delivery.event().as_audit().unwrap().kind()),
Kind::Error => format!("error:{}", delivery.event().as_error().unwrap()),
kind => format!("{kind:?}"),
}
}
#[tokio::test]
async fn a_fact_a_callback_emits_is_published_where_the_callback_ran() {
let probe = BoundaryProbe::new();
let hooks = fact_emitting_hooks(false);
let report = super::driver::Report::default();
let (_session, lease) = lease(Vec::new());
lease.admit().unwrap();
let model = FakeModel::scripted(vec![Start::Ready(Ok(FakeEvents::ready(text_generation(
"answered",
))))]);
let reservations = ReservationResources::new(hooks.len(), lease.state().reader());
let coord = generation_coord();
let facts = FactBuffer::default();
let controller = TranscriptController::new(&lease, &hooks, &reservations, &coord, &facts);
let states = [run_state::Store::new()];
let reader = lease.state().reader();
let callback_resources = [CallbackResources::new(
&coord, &states[0], &reader, None, &facts,
)];
let cancellation = probe.cancellation.clone();
let (producer, mut source) = channel(&cancellation);
let mut stream = source.take_stream().unwrap();
let phase = ToolPhase::new(
Outward::new(&producer, &facts),
&hooks,
&callback_resources,
&controller,
&coord,
None,
);
let inbox = InputMailbox::new();
let tools = empty_tools();
let terminal = Driver::new(&model, &lease, &inbox, &tools, &cancellation)
.with_tool_phase(phase)
.with_turn_trigger(vec![user("opened the turn")])
.with_report(&report)
.run(RequestPlan::new(), &source)
.await;
drop(driver_producer_guard(producer));
let mut labels = Vec::new();
while let Some(delivery) = stream.recv().await {
labels.push(event_label(&delivery));
}
assert!(matches!(terminal, Terminal::Completed));
assert_eq!(
labels,
[
"TurnStart",
"StepStart",
"TextDelta",
"audit:usage:0",
"Text",
"GenerationEnd",
"audit:finish:stop",
"error:degraded",
"StepEnd",
"TurnEnd",
]
);
}
#[tokio::test]
async fn a_fact_with_no_kind_fails_the_run_at_the_drain() {
let probe = BoundaryProbe::new();
let hooks = fact_emitting_hooks(true);
let report = super::driver::Report::default();
let (_session, lease) = lease(Vec::new());
lease.admit().unwrap();
let model = FakeModel::scripted(vec![Start::Ready(Ok(FakeEvents::ready(text_generation(
"answered",
))))]);
let reservations = ReservationResources::new(hooks.len(), lease.state().reader());
let coord = generation_coord();
let facts = FactBuffer::default();
let controller = TranscriptController::new(&lease, &hooks, &reservations, &coord, &facts);
let states = [run_state::Store::new()];
let reader = lease.state().reader();
let callback_resources = [CallbackResources::new(
&coord, &states[0], &reader, None, &facts,
)];
let cancellation = probe.cancellation.clone();
let (producer, mut source) = channel(&cancellation);
let mut stream = source.take_stream().unwrap();
let phase = ToolPhase::new(
Outward::new(&producer, &facts),
&hooks,
&callback_resources,
&controller,
&coord,
None,
);
let inbox = InputMailbox::new();
let tools = empty_tools();
let terminal = Driver::new(&model, &lease, &inbox, &tools, &cancellation)
.with_tool_phase(phase)
.with_turn_trigger(vec![user("opened the turn")])
.with_report(&report)
.run(RequestPlan::new(), &source)
.await;
drop(driver_producer_guard(producer));
let mut labels = Vec::new();
while let Some(delivery) = stream.recv().await {
labels.push(event_label(&delivery));
}
assert!(matches!(
terminal,
Terminal::Failed(Error::Event(ValidationError::EmptyAuditKind))
));
assert_eq!(labels, ["TurnStart", "StepStart", "TextDelta"]);
}
#[tokio::test]
async fn every_boundary_dispatches_its_callbacks_and_observations_in_order() {
let probe = BoundaryProbe::new();
let hooks = probe.registry(&[(
"boundary",
BoundaryPlan {
before_turn: vec![system_item("before-turn")],
after_turn: vec![system_item("after-turn")],
before_step: vec![system_item("before-step")],
..BoundaryPlan::default()
},
)]);
let report = super::driver::Report::default();
let (terminal, kinds, snapshot) = run_boundary_driver(
&probe,
&hooks,
vec![text_generation("answered")],
&report,
vec![user("opened the turn")],
)
.await;
assert!(matches!(terminal, Terminal::Completed));
assert_eq!(report.turns(), 1, "the Run opened exactly one Turn");
assert!(report.stop_code().is_none());
assert_eq!(
probe.calls(),
[
"boundary:before_turn(trigger=1)",
"boundary:before_step(model=fake)",
"boundary:before_model(messages=2,ephemeral=1)",
"boundary:after_model(generated=true,finish=stop)",
"boundary:after_step(model=fake)",
"boundary:after_turn(trigger=1)",
"boundary:decide_stop",
"boundary:compose_continuation(turns=1,continuations=0,continuation=0)",
]
);
assert_eq!(
probe.observation_kinds(),
[
"TurnStart",
"StepStart",
"GenerationStart",
"GenerationEnd",
"StepEnd",
"TurnEnd",
]
);
assert_eq!(
*probe.model_events.lock().expect("model event log lock"),
["delta:Text", "final:Text", "usage:0", "done:stop"],
"every accepted event is observed, including usage"
);
assert_eq!(
kinds,
[
Kind::TurnStart,
Kind::StepStart,
Kind::TextDelta,
Kind::Text,
Kind::GenerationEnd,
Kind::StepEnd,
Kind::TurnEnd,
],
"the observation list above has a GenerationStart and this one does not, \
deliberately: a consumer is told a generation opened by StepStart, which \
is one-to-one with it, while the observation's payload is the frozen \
Request and must not reach the outward stream (ADR-021)"
);
let texts = snapshot
.items()
.iter()
.map(|item| match item {
Item::System(system) => system.text().to_owned(),
Item::Assistant(_) => "assistant".to_owned(),
Item::User(_) | Item::ToolResult(_) => "other".to_owned(),
})
.collect::<Vec<_>>();
assert_eq!(
texts,
["before-turn", "before-step", "assistant", "after-turn"]
);
}
#[tokio::test]
async fn a_thinking_generation_publishes_its_delta_and_consolidated_event() {
let probe = BoundaryProbe::new();
let hooks = probe.registry(&[("thinking", BoundaryPlan::default())]);
let report = super::driver::Report::default();
let (terminal, kinds, snapshot) = run_boundary_driver(
&probe,
&hooks,
vec![thinking_generation()],
&report,
Vec::new(),
)
.await;
assert!(matches!(terminal, Terminal::Completed));
assert_eq!(
kinds,
[
Kind::TurnStart,
Kind::StepStart,
Kind::ThinkingDelta,
Kind::Thinking,
Kind::GenerationEnd,
Kind::StepEnd,
Kind::TurnEnd,
]
);
assert_eq!(snapshot.items().len(), 1, "the reasoning is recorded");
}
#[tokio::test]
async fn an_after_step_end_turn_closes_the_turn_and_ends_the_run() {
let probe = BoundaryProbe::new();
let hooks = probe.registry(&[(
"ends",
BoundaryPlan {
after_step: AfterStepAnswer::EndTurn,
..BoundaryPlan::default()
},
)]);
let report = super::driver::Report::default();
let (terminal, _kinds, _snapshot) = run_boundary_driver(
&probe,
&hooks,
vec![text_generation("done")],
&report,
Vec::new(),
)
.await;
assert!(matches!(terminal, Terminal::Completed));
assert_eq!(report.turns(), 1);
assert!(
probe
.calls()
.contains(&"ends:after_turn(trigger=0)".to_owned()),
"EndTurn reaches the Turn boundary"
);
}
#[tokio::test]
async fn end_turn_outranks_a_prequeued_next_step() {
let probe = BoundaryProbe::new();
let hooks = probe.registry(&[(
"ends",
BoundaryPlan {
after_step: AfterStepAnswer::EndTurn,
..BoundaryPlan::default()
},
)]);
let executions = Arc::new(AtomicUsize::new(0));
let tools = success_tools(&["tool"], &executions);
let inbox = InputMailbox::new();
inbox.push(InputBatch::new(InputTarget::NextStep, vec![user("queued")]).unwrap());
let report = super::driver::Report::default();
let (terminal, kinds, snapshot) = run_with_tools(
&probe,
&hooks,
&tools,
&inbox,
vec![tool_generation(&[model_tool_call("call-1", "tool")])],
&report,
accept_everything,
)
.await;
assert!(matches!(terminal, Terminal::Completed));
assert_eq!(executions.load(Ordering::SeqCst), 1);
assert_eq!(inbox.len(), 1, "EndTurn leaves NextStep unclaimed");
assert!(!kinds.contains(&Kind::InputAdmitted));
assert!(
snapshot
.items()
.iter()
.all(|item| !matches!(item, Item::User(_))),
"the prequeued input never enters the closed Turn"
);
}
#[tokio::test]
async fn a_before_model_modify_replaces_the_request_the_provider_receives() {
let probe = BoundaryProbe::new();
let hooks = probe.registry(&[(
"modify",
BoundaryPlan {
before_model: BeforeModelAnswer::Modify,
..BoundaryPlan::default()
},
)]);
let report = super::driver::Report::default();
let (terminal, _kinds, snapshot) = run_boundary_driver(
&probe,
&hooks,
vec![text_generation("kept")],
&report,
Vec::new(),
)
.await;
assert!(matches!(terminal, Terminal::Completed));
assert_eq!(snapshot.items().len(), 1);
}
#[tokio::test]
async fn a_before_model_short_circuit_skips_the_provider_and_still_runs_after_model() {
let probe = BoundaryProbe::new();
let substitute = Assistant::new(
vec![Segment::Text {
text: "substituted".to_owned(),
extensions: extensions(),
}],
None,
);
let hooks = probe.registry(&[(
"short",
BoundaryPlan {
before_model: BeforeModelAnswer::ShortCircuit(substitute),
..BoundaryPlan::default()
},
)]);
let report = super::driver::Report::default();
let (terminal, kinds, snapshot) =
run_boundary_driver(&probe, &hooks, Vec::new(), &report, Vec::new()).await;
assert!(
matches!(terminal, Terminal::Completed),
"the provider was never called, so its empty script is never reached"
);
assert!(
probe
.calls()
.contains(&"short:after_model(generated=false,finish=)".to_owned())
);
assert!(
probe
.observation_facts()
.contains(&"GenerationEnd end=Completed finish=\"\" completion=Unknown".to_owned())
);
assert!(kinds.contains(&Kind::Text));
assert_eq!(
snapshot.items().len(),
1,
"the substituted answer is recorded"
);
}
#[tokio::test]
async fn a_before_model_abort_fails_the_run_with_the_hook_reason() {
let probe = BoundaryProbe::new();
let hooks = probe.registry(&[(
"abort",
BoundaryPlan {
before_model: BeforeModelAnswer::Abort("policy refused".to_owned()),
..BoundaryPlan::default()
},
)]);
let report = super::driver::Report::default();
let (terminal, _kinds, snapshot) =
run_boundary_driver(&probe, &hooks, Vec::new(), &report, Vec::new()).await;
match terminal.failure() {
Some(error @ Error::ModelAborted(reason)) => {
assert_eq!(reason, "policy refused");
assert_eq!(
error.to_string(),
"a hook aborted the model call: policy refused"
);
assert!(error.cause().is_none(), "the reason is the whole fact");
}
other => panic!("expected an abort: {:?}", other.map(ToString::to_string)),
}
assert!(snapshot.items().is_empty());
}
#[tokio::test]
async fn a_decide_stop_stop_publishes_the_single_event_stop() {
let probe = BoundaryProbe::new();
let hooks = probe.registry(&[(
"stopper",
BoundaryPlan {
decide_stop: Some(StopCode::new("budget.exhausted".to_owned()).unwrap()),
..BoundaryPlan::default()
},
)]);
let report = super::driver::Report::default();
let (terminal, kinds, _snapshot) = run_boundary_driver(
&probe,
&hooks,
vec![text_generation("done")],
&report,
Vec::new(),
)
.await;
assert!(matches!(terminal, Terminal::Stopped));
assert_eq!(kinds.last(), Some(&Kind::Stop));
assert_eq!(
kinds.iter().filter(|kind| **kind == Kind::Stop).count(),
1,
"a stopped Run publishes exactly one EventStop"
);
assert_eq!(
report.stop_code().map(|code| code.as_str().to_owned()),
Some("budget.exhausted".to_owned())
);
assert!(
!probe
.calls()
.iter()
.any(|call| call.starts_with("stopper:compose_continuation")),
"a stop decision outranks any continuation"
);
}
#[tokio::test]
async fn stop_outranks_a_prequeued_next_turn() {
let probe = BoundaryProbe::new();
let hooks = probe.registry(&[(
"stopper",
BoundaryPlan {
decide_stop: Some(StopCode::new("budget.exhausted".to_owned()).unwrap()),
..BoundaryPlan::default()
},
)]);
let inbox = InputMailbox::new();
inbox.push(InputBatch::new(InputTarget::NextTurn, vec![user("queued")]).unwrap());
let report = super::driver::Report::default();
let (terminal, kinds, snapshot) = run_with_tools(
&probe,
&hooks,
&empty_tools(),
&inbox,
vec![text_generation("done")],
&report,
accept_everything,
)
.await;
assert!(matches!(terminal, Terminal::Stopped));
assert_eq!(report.turns(), 1);
assert_eq!(inbox.len(), 1, "stop leaves NextTurn unclaimed");
assert!(!kinds.contains(&Kind::InputAdmitted));
assert!(
snapshot
.items()
.iter()
.all(|item| !matches!(item, Item::User(_)))
);
}
#[tokio::test]
async fn a_compose_continuation_opens_exactly_one_more_turn() {
let probe = BoundaryProbe::new();
let hooks = probe.registry(&[(
"continuer",
BoundaryPlan {
compose_continuation: ContinuationAnswer::Replace(vec![Part::text(
"keep going".to_owned(),
)]),
..BoundaryPlan::default()
},
)]);
let report = super::driver::Report::default();
let (terminal, _kinds, snapshot) = run_boundary_driver(
&probe,
&hooks,
vec![text_generation("first"), text_generation("second")],
&report,
Vec::new(),
)
.await;
assert!(matches!(terminal, Terminal::Completed));
assert_eq!(report.turns(), 2, "the continuation opened one more Turn");
let compose_continuation_calls = probe
.calls()
.into_iter()
.filter(|call| call.starts_with("continuer:compose_continuation"))
.collect::<Vec<_>>();
assert_eq!(
compose_continuation_calls,
[
"continuer:compose_continuation(turns=1,continuations=0,continuation=0)",
"continuer:compose_continuation(turns=2,continuations=1,continuation=0)",
]
);
assert!(
snapshot
.items()
.iter()
.any(|item| matches!(item, Item::User(user) if user.parts()[0].fields().text() == "keep going")),
"the continuation is recorded as user content"
);
assert_eq!(
probe
.calls()
.iter()
.filter(|call| call.starts_with("continuer:before_turn"))
.collect::<Vec<_>>(),
[
"continuer:before_turn(trigger=0)",
"continuer:before_turn(trigger=1)"
]
);
}
#[tokio::test]
async fn an_abstaining_compose_continuation_preserves_a_peers_continuation() {
let probe = BoundaryProbe::new();
let hooks = probe.registry(&[
(
"continuer",
BoundaryPlan {
compose_continuation: ContinuationAnswer::Replace(vec![Part::text(
"keep going".to_owned(),
)]),
..BoundaryPlan::default()
},
),
(
"quiet",
BoundaryPlan {
compose_continuation: ContinuationAnswer::Keep,
..BoundaryPlan::default()
},
),
]);
let report = super::driver::Report::default();
let (terminal, _kinds, snapshot) = run_boundary_driver(
&probe,
&hooks,
vec![text_generation("first"), text_generation("second")],
&report,
Vec::new(),
)
.await;
assert!(matches!(terminal, Terminal::Completed));
assert_eq!(report.turns(), 2, "the abstention preserved the request");
assert!(
probe.calls().iter().any(
|call| call == "quiet:compose_continuation(turns=1,continuations=0,continuation=1)"
),
"the later registration sees the earlier one's continuation: {:?}",
probe.calls()
);
assert!(
snapshot
.items()
.iter()
.any(|item| matches!(item, Item::User(user) if user.parts()[0].fields().text() == "keep going")),
"the preserved continuation is recorded as user content"
);
}
#[tokio::test]
async fn an_explicit_empty_compose_continuation_replacement_withdraws_a_peers_continuation() {
let probe = BoundaryProbe::new();
let hooks = probe.registry(&[
(
"continuer",
BoundaryPlan {
compose_continuation: ContinuationAnswer::Replace(vec![Part::text(
"keep going".to_owned(),
)]),
..BoundaryPlan::default()
},
),
(
"vetoer",
BoundaryPlan {
compose_continuation: ContinuationAnswer::Replace(Vec::new()),
..BoundaryPlan::default()
},
),
]);
let report = super::driver::Report::default();
let (terminal, _kinds, snapshot) = run_boundary_driver(
&probe,
&hooks,
vec![text_generation("only")],
&report,
Vec::new(),
)
.await;
assert!(matches!(terminal, Terminal::Completed));
assert_eq!(report.turns(), 1, "the veto ended the Run");
assert!(
probe
.calls()
.iter()
.any(|call| call
== "vetoer:compose_continuation(turns=1,continuations=0,continuation=1)"),
"the veto saw the request it withdrew: {:?}",
probe.calls()
);
assert!(
!snapshot
.items()
.iter()
.any(|item| matches!(item, Item::User(user) if user.parts()[0].fields().text() == "keep going")),
"a withdrawn continuation is never recorded"
);
}
#[tokio::test]
async fn a_failing_boundary_callback_names_its_hook_and_stops_the_run() {
for boundary in [
"before_turn",
"before_step",
"before_model",
"after_step",
"after_turn",
"decide_stop",
"compose_continuation",
] {
let probe = BoundaryProbe::new();
let hooks = probe.registry(&[(
"failing",
BoundaryPlan {
failing: Some(boundary),
..BoundaryPlan::default()
},
)]);
let report = super::driver::Report::default();
let (terminal, _kinds, _snapshot) = run_boundary_driver(
&probe,
&hooks,
vec![text_generation("answered")],
&report,
Vec::new(),
)
.await;
match terminal.failure() {
Some(error @ Error::Hook(DispatchError::Callback(failure))) => {
assert_eq!(failure.name(), "failing", "{boundary}");
assert_eq!(failure.error().to_string(), boundary);
assert_eq!(
error.to_string(),
format!("a boundary callback failed: hook failing failed: {boundary}")
);
assert_eq!(
error
.cause()
.expect("a boundary failure retains the dispatch error")
.to_string(),
format!("hook failing failed: {boundary}")
);
}
other => panic!(
"{boundary}: expected a hook failure: {:?}",
other.map(ToString::to_string)
),
}
}
}
#[tokio::test]
async fn a_boundary_batch_that_cannot_commit_fails_closed() {
let probe = BoundaryProbe::new();
let hooks = probe.registry(&[(
"committer",
BoundaryPlan {
before_turn: vec![system_item("before-turn")],
..BoundaryPlan::default()
},
)]);
let (_session, lease) = lease(Vec::new());
lease.admit().unwrap();
let model = FakeModel::ready(vec![Ok(done())]);
let reservations = ReservationResources::new(hooks.len(), lease.state().reader());
let coord = generation_coord();
let emitter = SilentEmitter;
let controller = TranscriptController::new(&lease, &hooks, &reservations, &coord, &emitter);
let states = (0..hooks.len())
.map(|_| run_state::Store::new())
.collect::<Vec<_>>();
let reader = lease.state().reader();
let callback_resources = states
.iter()
.map(|state| CallbackResources::new(&coord, state, &reader, None, &emitter))
.collect::<Vec<_>>();
let cancellation = Token::new();
let (producer, source) = channel(&cancellation);
let facts = FactBuffer::default();
let outward = Outward::new(&producer, &facts);
let phase = ToolPhase::new(
outward,
&hooks,
&callback_resources,
&controller,
&coord,
None,
);
let inbox = InputMailbox::new();
let tools = empty_tools();
let driver = Driver::new(&model, &lease, &inbox, &tools, &cancellation).with_tool_phase(phase);
lease.release();
let terminal = driver.run(RequestPlan::new(), &source).await;
match terminal.failure() {
Some(error @ Error::BoundaryTranscript(_)) => {
assert!(
error
.to_string()
.starts_with("committing a boundary batch failed:")
);
}
other => panic!(
"expected a boundary transcript failure: {:?}",
other.map(ToString::to_string)
),
}
}
#[test]
fn every_loop_failure_retains_the_stage_value_it_reports() {
let conflict = store::Error::VersionConflict {
expected: Version::ZERO,
actual: Version::from_u64(1),
}
.to_string();
let version_conflict = conflict.as_str();
let cases: Vec<(Error, Option<&str>)> = vec![
(
Error::Transcript(store::Error::VersionConflict {
expected: Version::ZERO,
actual: Version::from_u64(1),
}),
Some(version_conflict),
),
(
Error::Tools(StepResolutionError::Collision(
tool::Name::local("dup".to_owned()).unwrap(),
)),
Some("Step tool collision: dup"),
),
(
Error::Request(RequestError::EmptySystemInstructionText { index: 0 }),
Some("system instruction text at index 0 is empty"),
),
(
Error::Model(Arc::new(FakeError("provider down"))),
Some("provider down"),
),
(
Error::Stream(StreamError::MissingTerminal),
Some("model stream ended without terminal"),
),
(
Error::Coordinate(CoordError::EmptyCallId),
Some("tool-call ID must not be empty"),
),
(
Error::Event(ValidationError::PayloadMismatch),
Some("event category and payload do not match"),
),
(
Error::Prepare(DispatchError::Cancelled),
Some("cancelled around a callback"),
),
(
Error::Route(ToolBatchRouteError::new(
super::tool_batch::Error::cancelled_batch(),
)),
Some("prepared tool batch failed with no source position: tool run cancelled"),
),
(
Error::Commit(ToolBatchCommitError::Rewrite {
source_index: 0,
error: DispatchError::Cancelled,
}),
Some("cancelled around a callback"),
),
(
Error::Commit(ToolBatchCommitError::Controller(ControllerError::Lease(
LeaseError::Transcript(store::Error::VersionConflict {
expected: Version::ZERO,
actual: Version::from_u64(1),
}),
))),
Some(version_conflict),
),
(
Error::Commit(ToolBatchCommitError::Controller(
ControllerError::InvalidTransition {
previous: Version::ZERO,
delta: 0,
},
)),
None,
),
(
Error::Commit(ToolBatchCommitError::Controller(ControllerError::Lease(
LeaseError::Released,
))),
Some("session lease released"),
),
(
Error::Input(InputAdmitError::EmptyModify),
Some("a registration rewrote the batch to nothing"),
),
(
Error::AfterToolExecuteTranscript(ControllerError::Dispatch(DispatchError::Cancelled)),
Some("cancelled around a callback"),
),
(
Error::InputAdmitTranscript(ControllerError::InvalidTransition {
previous: Version::ZERO,
delta: 0,
}),
None,
),
(
Error::GenerationTranscript(ControllerError::Lease(LeaseError::Transcript(
store::Error::VersionConflict {
expected: Version::ZERO,
actual: Version::from_u64(1),
},
))),
Some(version_conflict),
),
(
Error::BoundaryTranscript(ControllerError::Lease(LeaseError::Released)),
Some("session lease released"),
),
(
Error::Hook(DispatchError::Cancelled),
Some("cancelled around a callback"),
),
(Error::ModelAborted("refused".to_owned()), None),
(
Error::Commit(ToolBatchCommitError::Rewrite {
source_index: 1,
error: DispatchError::Cancelled,
}),
Some("cancelled around a callback"),
),
(Error::Commit(ToolBatchCommitError::Cancelled), None),
(Error::Cancelled, None),
(Error::MissingToolPhase, None),
(Error::ToolCallRejected, None),
(Error::ToolResultRejected, None),
(Error::AfterToolExecuteRejected, None),
];
for (error, expected) in cases {
let rendered = error.to_string();
assert!(!rendered.is_empty());
let cause = error.cause().map(|cause| cause.to_string());
assert_eq!(cause.as_deref(), expected, "cause of {rendered}");
}
}
#[test]
fn a_transcript_mutation_failure_names_the_value_underneath_it() {
let dispatch: &dyn error::Error = &ControllerError::Dispatch(DispatchError::Cancelled);
assert_eq!(
dispatch
.source()
.expect("a dispatch failure retains the callback error")
.to_string(),
"cancelled around a callback"
);
let lease: &dyn error::Error = &ControllerError::Lease(LeaseError::Released);
assert_eq!(
lease
.source()
.expect("a lease rejection retains the lease error")
.to_string(),
"session lease released"
);
let transition: &dyn error::Error = &ControllerError::InvalidTransition {
previous: Version::ZERO,
delta: 0,
};
assert!(
transition.source().is_none(),
"the version-shape rejection is its own leaf"
);
let cancelled: &dyn error::Error = &DispatchError::Cancelled;
assert!(cancelled.source().is_none());
}
#[tokio::test]
async fn a_boundary_commits_staged_history_and_returned_items_in_one_transaction() {
let probe = BoundaryProbe::new();
let hooks = probe.registry(&[(
"rewriter",
BoundaryPlan {
staged: Some(system_item("staged")),
before_turn: vec![system_item("returned")],
..BoundaryPlan::default()
},
)]);
let report = super::driver::Report::default();
let (terminal, _kinds, snapshot) = run_boundary_driver(
&probe,
&hooks,
vec![text_generation("done")],
&report,
Vec::new(),
)
.await;
assert!(matches!(terminal, Terminal::Completed));
let texts = snapshot
.items()
.iter()
.filter_map(|item| match item {
Item::System(system) => Some(system.text().to_owned()),
_ => None,
})
.collect::<Vec<_>>();
assert_eq!(texts, ["staged", "returned"]);
assert_eq!(
snapshot.version(),
Version::from_u64(2),
"one boundary replacement plus one assistant append"
);
assert_eq!(
probe.reservations.load(Ordering::SeqCst),
2,
"staged history and returned items share one reservation; the assistant uses the other"
);
}
#[tokio::test]
async fn staged_boundary_at_max_minus_one_combines_the_suffix_in_one_replace() {
let current = Version::from_u64(u64::MAX - 1);
let session = Session::new(
"near-version-limit".to_owned(),
Some(Snapshot::new(current, vec![system_item("seed")])),
);
let probe = BoundaryProbe::new();
let hooks = probe.registry(&[(
"rewriter",
BoundaryPlan {
staged: Some(system_item("staged")),
before_turn: vec![system_item("returned")],
failing: Some("before_model"),
..BoundaryPlan::default()
},
)]);
let report = super::driver::Report::default();
let (terminal, _kinds, snapshot) = run_boundary_driver_on_session(
&probe,
&hooks,
vec![text_generation("unreached")],
&report,
Vec::new(),
session,
)
.await;
assert!(matches!(
terminal.failure(),
Some(Error::Hook(DispatchError::Callback(failure)))
if failure.name() == "rewriter"
));
assert_eq!(snapshot.version(), Version::MAX);
let texts = snapshot
.items()
.iter()
.filter_map(|item| match item {
Item::System(system) => Some(system.text()),
_ => None,
})
.collect::<Vec<_>>();
assert_eq!(texts, ["seed", "staged", "returned"]);
assert_eq!(
probe.reservations.load(Ordering::SeqCst),
1,
"the combined boundary is one Replace rather than a staging append plus a live append"
);
}
#[tokio::test]
async fn staged_boundary_error_and_cancellation_commit_neither_half() {
for interruption in [
ReturnedReservationInterruption::Fail,
ReturnedReservationInterruption::Cancel,
] {
let probe = BoundaryProbe::new();
let hooks = probe.registry(&[(
"interrupts",
BoundaryPlan {
staged: Some(system_item("staged")),
before_turn: vec![system_item("returned")],
returned_reservation_interruption: Some(interruption),
..BoundaryPlan::default()
},
)]);
let report = super::driver::Report::default();
let (terminal, _kinds, snapshot) = run_boundary_driver(
&probe,
&hooks,
vec![text_generation("unreached")],
&report,
Vec::new(),
)
.await;
match interruption {
ReturnedReservationInterruption::Fail => {
assert!(matches!(
terminal.failure(),
Some(Error::BoundaryTranscript(_))
));
}
ReturnedReservationInterruption::Cancel => {
assert!(matches!(terminal, Terminal::Cancelled));
}
ReturnedReservationInterruption::Panic => unreachable!(),
}
assert_eq!(snapshot.version(), Version::ZERO);
assert!(
snapshot.items().is_empty(),
"neither staged nor returned items commit"
);
assert_eq!(probe.reservations.load(Ordering::SeqCst), 1);
}
}
#[tokio::test]
async fn a_staged_boundary_reservation_panic_commits_neither_half() {
let session = Session::new("panic-boundary".to_owned(), None);
let inspect = session.clone();
let task = tokio::spawn(async move {
let probe = BoundaryProbe::new();
let hooks = probe.registry(&[(
"panics",
BoundaryPlan {
staged: Some(system_item("staged")),
before_turn: vec![system_item("returned")],
returned_reservation_interruption: Some(ReturnedReservationInterruption::Panic),
..BoundaryPlan::default()
},
)]);
let report = super::driver::Report::default();
run_boundary_driver_on_session(
&probe,
&hooks,
vec![text_generation("unreached")],
&report,
Vec::new(),
session,
)
.await
});
let join_error = match task.await {
Ok(_) => panic!("the reservation panic must propagate"),
Err(error) => error,
};
assert!(join_error.is_panic());
let snapshot = inspect.transcript().snapshot().unwrap();
assert_eq!(snapshot.version(), Version::ZERO);
assert!(
snapshot.items().is_empty(),
"neither staged nor returned items commit"
);
}
#[tokio::test]
async fn an_after_model_modify_replaces_the_recorded_assistant_output() {
let probe = BoundaryProbe::new();
let replacement = Assistant::new(
vec![Segment::Text {
text: "rewritten".to_owned(),
extensions: extensions(),
}],
None,
);
let hooks = probe.registry(&[(
"rewrites-output",
BoundaryPlan {
after_model: Some(replacement),
..BoundaryPlan::default()
},
)]);
let report = super::driver::Report::default();
let (terminal, _kinds, snapshot) = run_boundary_driver(
&probe,
&hooks,
vec![text_generation("original")],
&report,
Vec::new(),
)
.await;
assert!(matches!(terminal, Terminal::Completed));
let recorded = snapshot
.items()
.iter()
.filter_map(|item| match item {
Item::Assistant(assistant) => {
assistant.content().first().map(|segment| match segment {
Segment::Text { text, .. } => text.clone(),
Segment::Thinking { .. } | Segment::ToolCall { .. } => "other".to_owned(),
})
}
_ => None,
})
.collect::<Vec<_>>();
assert_eq!(recorded, ["rewritten"]);
}
fn assistant_with_tool_calls(calls: &[(&str, &str)]) -> Assistant {
Assistant::new(
calls
.iter()
.map(|(id, name)| Segment::ToolCall {
call: model_tool_call(id, name),
extensions: extensions(),
})
.collect(),
None,
)
}
fn invalid_hook_assistants() -> Vec<(&'static str, Assistant, RequestError)> {
vec![
(
"empty-id",
assistant_with_tool_calls(&[("", "tool")]),
RequestError::EmptyToolCallId,
),
(
"hidden-empty-id",
assistant_with_tool_calls(&[("", "tool")]).with_visibility(ModelVisibility::Hidden),
RequestError::EmptyToolCallId,
),
(
"empty-name",
assistant_with_tool_calls(&[("call", "")]),
RequestError::EmptyToolCallName {
call_id: "call".to_owned(),
},
),
(
"duplicate-id",
assistant_with_tool_calls(&[("call", "first"), ("call", "second")]),
RequestError::DuplicateToolCallId {
call_id: "call".to_owned(),
},
),
(
"hidden-duplicate-id",
assistant_with_tool_calls(&[("call", "first"), ("call", "second")])
.with_visibility(ModelVisibility::Hidden),
RequestError::DuplicateToolCallId {
call_id: "call".to_owned(),
},
),
]
}
#[tokio::test]
async fn invalid_hook_assistants_are_attributed_atomic_and_leave_the_session_reusable() {
for short_circuit in [true, false] {
for (case, invalid, expected) in invalid_hook_assistants() {
let session = Session::new(format!("invalid-{short_circuit}-{case}"), None);
let probe = BoundaryProbe::new();
let plan = if short_circuit {
BoundaryPlan {
before_model: BeforeModelAnswer::ShortCircuit(invalid),
..BoundaryPlan::default()
}
} else {
BoundaryPlan {
after_model: Some(invalid),
..BoundaryPlan::default()
}
};
let hooks = probe.registry(&[("invalid-assistant", plan)]);
let scripts = if short_circuit {
Vec::new()
} else {
vec![text_generation("replaced")]
};
let report = super::driver::Report::default();
let (terminal, _kinds, snapshot) = run_boundary_driver_on_session(
&probe,
&hooks,
scripts,
&report,
Vec::new(),
session.clone(),
)
.await;
let Some(Error::Hook(DispatchError::Callback(failure))) = terminal.failure() else {
panic!("{case}: an invalid hook Assistant must be an attributed hook failure");
};
assert_eq!(failure.name(), "invalid-assistant", "{case}");
assert_eq!(
failure.error().source_ref().downcast_ref::<RequestError>(),
Some(&expected),
"{case}"
);
assert_eq!(terminal.outcome(), crate::value::lifecycle::Outcome::Failed);
assert_eq!(
terminal.failure().map(Error::classify),
Some(FailureKind::Hook),
"{case}: invalid hook output is not TaskPanicked/Internal"
);
assert_eq!(snapshot.version(), Version::ZERO, "{case}");
assert!(snapshot.items().is_empty(), "{case}");
let successor_probe = BoundaryProbe::new();
let successor_hooks = empty_hooks();
let successor_report = super::driver::Report::default();
let (successor, _kinds, successor_snapshot) = run_boundary_driver_on_session(
&successor_probe,
&successor_hooks,
vec![text_generation("successor")],
&successor_report,
Vec::new(),
session,
)
.await;
assert!(matches!(successor, Terminal::Completed), "{case}");
assert_eq!(successor_snapshot.version(), Version::from_u64(1), "{case}");
assert_eq!(successor_snapshot.items().len(), 1, "{case}");
}
}
}
#[derive(Clone, Copy, Debug)]
enum HookAssistantSource {
BeforeModelShortCircuit,
AfterModelModify,
}
struct OneShotHookAssistant {
source: HookAssistantSource,
assistant: Assistant,
invocations: AtomicUsize,
}
impl ModelHooks for OneShotHookAssistant {
fn before_model(&self, _scope: &BeforeModelScope<'_>) -> HookResult<BeforeModelDecision> {
Ok(
if matches!(self.source, HookAssistantSource::BeforeModelShortCircuit)
&& self.invocations.fetch_add(1, Ordering::SeqCst) == 0
{
BeforeModelDecision::ShortCircuit(self.assistant.clone())
} else {
BeforeModelDecision::Continue
},
)
}
fn after_model(&self, _scope: &AfterModelScope<'_>) -> AfterModelDecision {
if matches!(self.source, HookAssistantSource::AfterModelModify)
&& self.invocations.fetch_add(1, Ordering::SeqCst) == 0
{
AfterModelDecision::Modify(self.assistant.clone())
} else {
AfterModelDecision::Continue
}
}
}
fn one_shot_hook_assistant(
registration: &str,
source: HookAssistantSource,
assistant: Assistant,
) -> hook::RuntimeRegistry {
let mut builder = hook::RegistryBuilder::new();
builder.register(
hook::Registration::new(registration.to_owned()).model(Arc::new(OneShotHookAssistant {
source,
assistant,
invocations: AtomicUsize::new(0),
})),
);
builder
.build_runtime()
.expect("one-shot model hook registry builds")
}
fn historical_tool_exchange(visibility: ModelVisibility) -> Snapshot {
Snapshot::new(
Version::ZERO,
vec![
Item::Assistant(
assistant_with_tool_calls(&[("historical-call", "tool")])
.with_visibility(visibility),
),
Item::ToolResult(crate::transcript::item::ToolResult::new(
Id::new("historical-call".to_owned()),
tool_call::Name::new("tool".to_owned()),
ToolResult::no_output(),
visibility,
None,
)),
],
)
}
async fn assert_model_visibility_identity_matrix(source: HookAssistantSource) {
for (case, history_visibility, output_visibility, rejected) in [
(
"visible-history-visible-output",
ModelVisibility::Visible,
ModelVisibility::Visible,
true,
),
(
"hidden-history-visible-output",
ModelVisibility::Hidden,
ModelVisibility::Visible,
false,
),
(
"visible-history-hidden-output",
ModelVisibility::Visible,
ModelVisibility::Hidden,
false,
),
] {
let history = historical_tool_exchange(history_visibility);
let session = Session::new(format!("{source:?}-{case}"), Some(history.clone()));
let executions = Arc::new(AtomicUsize::new(0));
let tools = success_tools(&["tool"], &executions);
let registration = match source {
HookAssistantSource::BeforeModelShortCircuit => "visibility-short-circuit",
HookAssistantSource::AfterModelModify => "visibility-after-model",
};
let output = assistant_with_tool_calls(&[("historical-call", "tool")])
.with_visibility(output_visibility);
let hooks = one_shot_hook_assistant(registration, source, output);
let scripts = match source {
HookAssistantSource::BeforeModelShortCircuit => {
vec![text_generation("continued after short circuit")]
}
HookAssistantSource::AfterModelModify => vec![
text_generation("discarded replacement source"),
text_generation("continued after rewrite"),
],
};
let probe = BoundaryProbe::new();
let report = super::driver::Report::default();
let (terminal, _kinds, snapshot) = run_with_tools_on_session(
&probe,
&hooks,
&tools,
&InputMailbox::new(),
scripts,
&report,
accept_everything,
session.clone(),
)
.await;
if rejected {
let Some(Error::Hook(DispatchError::Callback(failure))) = terminal.failure() else {
panic!("{source:?}/{case}: visible reuse must fail at its registration");
};
assert_eq!(failure.name(), registration, "{source:?}/{case}");
assert_eq!(
failure.error().source_ref().downcast_ref::<RequestError>(),
Some(&RequestError::DuplicateToolCallId {
call_id: "historical-call".to_owned(),
}),
"{source:?}/{case}"
);
assert_eq!(snapshot.version(), history.version(), "{source:?}/{case}");
assert!(
std::ptr::eq(snapshot.items(), history.items()),
"{source:?}/{case}: rejection commits nothing"
);
assert_eq!(executions.load(Ordering::SeqCst), 0, "{source:?}/{case}");
} else {
assert!(matches!(terminal, Terminal::Completed), "{source:?}/{case}");
assert_eq!(executions.load(Ordering::SeqCst), 1, "{source:?}/{case}");
let tool_exchange_visibilities = snapshot
.items()
.iter()
.filter_map(|item| match item {
Item::Assistant(assistant)
if assistant
.content()
.iter()
.any(|segment| matches!(segment, Segment::ToolCall { .. })) =>
{
Some(assistant.visibility())
}
_ => None,
})
.collect::<Vec<_>>();
assert_eq!(
tool_exchange_visibilities,
[history_visibility, output_visibility],
"{source:?}/{case}"
);
assert_replay_valid(&snapshot);
}
let version_before_successor = snapshot.version();
let successor_probe = BoundaryProbe::new();
let successor_report = super::driver::Report::default();
let (successor, _kinds, successor_snapshot) = run_with_tools_on_session(
&successor_probe,
&empty_hooks(),
&tools,
&InputMailbox::new(),
vec![text_generation("successor")],
&successor_report,
accept_everything,
session,
)
.await;
assert!(
matches!(successor, Terminal::Completed),
"{source:?}/{case}"
);
assert_eq!(
successor_snapshot.version(),
Version::from_u64(version_before_successor.as_u64() + 1),
"{source:?}/{case}"
);
assert_replay_valid(&successor_snapshot);
}
}
#[tokio::test]
async fn before_model_short_circuit_obeys_the_model_visibility_identity_matrix() {
assert_model_visibility_identity_matrix(HookAssistantSource::BeforeModelShortCircuit).await;
}
#[tokio::test]
async fn after_model_modify_obeys_the_model_visibility_identity_matrix() {
assert_model_visibility_identity_matrix(HookAssistantSource::AfterModelModify).await;
}
#[tokio::test]
async fn a_generation_with_only_provider_data_is_still_recorded() {
let probe = BoundaryProbe::new();
let accounting = Envelope::try_new(
"provider".to_owned(),
"usage".to_owned(),
Payload::copy_from_slice(br#"{"refused":true}"#),
)
.unwrap();
let metadata_only = Assistant::new(Vec::new(), Some(vec![accounting]));
let hooks = probe.registry(&[(
"keeps-only-metadata",
BoundaryPlan {
after_model: Some(metadata_only),
..BoundaryPlan::default()
},
)]);
let report = super::driver::Report::default();
let (terminal, _kinds, snapshot) = run_boundary_driver(
&probe,
&hooks,
vec![text_generation("original")],
&report,
Vec::new(),
)
.await;
assert!(matches!(terminal, Terminal::Completed));
let recorded = snapshot
.items()
.iter()
.filter_map(|item| match item {
Item::Assistant(assistant) => Some(assistant),
_ => None,
})
.collect::<Vec<_>>();
assert_eq!(recorded.len(), 1, "the envelopes keep the record alive");
assert!(recorded[0].content().is_empty());
let envelopes = recorded[0].provider_data().expect("presence is preserved");
assert_eq!(envelopes.len(), 1);
assert_eq!(envelopes[0].namespace(), "provider");
assert_eq!(envelopes[0].kind(), "usage");
assert_eq!(envelopes[0].data().as_bytes(), br#"{"refused":true}"#);
}
#[tokio::test]
async fn a_generation_with_neither_content_nor_provider_data_records_nothing() {
for provider_data in [None, Some(Vec::new())] {
let probe = BoundaryProbe::new();
let hooks = probe.registry(&[(
"keeps-nothing",
BoundaryPlan {
after_model: Some(Assistant::new(Vec::new(), provider_data)),
..BoundaryPlan::default()
},
)]);
let report = super::driver::Report::default();
let (terminal, _kinds, snapshot) = run_boundary_driver(
&probe,
&hooks,
vec![text_generation("original")],
&report,
Vec::new(),
)
.await;
assert!(matches!(terminal, Terminal::Completed));
assert!(
!snapshot
.items()
.iter()
.any(|item| matches!(item, Item::Assistant(_))),
);
}
}
#[tokio::test]
async fn a_cancellation_at_a_boundary_is_cancellation_not_a_callback_failure() {
let probe = BoundaryProbe::new();
let hooks = probe.registry(&[("cancelled", BoundaryPlan::default())]);
let (_session, lease) = lease(Vec::new());
lease.admit().unwrap();
let cancellation = Token::new();
let model = FakeModel::ready(vec![Ok(done())]);
let reservations = ReservationResources::new(hooks.len(), lease.state().reader());
let coord = generation_coord();
let emitter = SilentEmitter;
let controller = TranscriptController::new(&lease, &hooks, &reservations, &coord, &emitter);
let states = (0..hooks.len())
.map(|_| run_state::Store::new())
.collect::<Vec<_>>();
let reader = lease.state().reader();
let callback_resources = states
.iter()
.map(|state| CallbackResources::new(&coord, state, &reader, None, &emitter))
.collect::<Vec<_>>();
let (producer, source) = channel(&cancellation);
let facts = FactBuffer::default();
let outward = Outward::new(&producer, &facts);
let phase = ToolPhase::new(
outward,
&hooks,
&callback_resources,
&controller,
&coord,
None,
);
let inbox = InputMailbox::new();
let tools = empty_tools();
let driver = Driver::new(&model, &lease, &inbox, &tools, &cancellation).with_tool_phase(phase);
cancellation.cancel();
let terminal = driver.run(RequestPlan::new(), &source).await;
assert!(matches!(terminal, Terminal::Cancelled));
assert!(
probe.calls().is_empty(),
"no boundary callback runs after cancellation"
);
}
#[test]
fn each_boundary_adapter_refuses_to_run_after_cancellation() {
let probe = BoundaryProbe::new();
let hooks = probe.registry(&[("cancelled", BoundaryPlan::default())]);
let (_session, lease) = lease(Vec::new());
lease.admit().unwrap();
let coord = generation_coord();
let emitter = SilentEmitter;
let state = run_state::Store::new();
let reader = lease.state().reader();
let transcript = lease.transcript_reader();
let resources = || CallbackResources::new(&coord, &state, &reader, None, &emitter);
let store = Store::new();
let staged = StagedTranscript::from_store(&store);
let snapshot = StepSnapshot::freeze(
Info::new("fake".to_owned(), 1),
RequestPlan::new(),
empty_tools()
.resolve_step_without_providers(
Snapshot::new(Version::ZERO, Vec::new()),
&Token::new(),
&ExposeAll,
)
.unwrap()
.into_snapshot(),
);
let cancelled = Token::new();
cancelled.cancel();
for adapter in hooks.turn_step_hooks() {
assert!(matches!(
adapter.invoke_before_turn(resources(), &staged, &[], &cancelled),
Err(DispatchError::Cancelled)
));
assert!(matches!(
adapter.invoke_after_turn(resources(), &staged, &[], &cancelled),
Err(DispatchError::Cancelled)
));
assert!(matches!(
adapter.invoke_after_step(resources(), &staged, &snapshot, &cancelled),
Err(DispatchError::Cancelled)
));
}
for adapter in hooks.run_hooks() {
assert!(matches!(
adapter.invoke_compose_continuation(resources(), &staged, &[], 0, 0, &cancelled),
Err(DispatchError::Cancelled)
));
assert!(matches!(
adapter.invoke_decide_stop(resources(), &transcript, &cancelled),
Err(DispatchError::Cancelled)
));
}
assert!(
probe.calls().is_empty(),
"a cancelled dispatch never reaches the callback"
);
}
#[test]
fn every_loop_failure_classifies_the_stage_it_names() {
let cases: Vec<(Error, FailureKind)> = vec![
(
Error::Transcript(store::Error::VersionOverflow {
current: Version::ZERO,
delta: 1,
}),
FailureKind::Transcript,
),
(
Error::AfterToolExecuteTranscript(ControllerError::Lease(LeaseError::Released)),
FailureKind::Transcript,
),
(
Error::InputAdmitTranscript(ControllerError::Lease(LeaseError::Released)),
FailureKind::Transcript,
),
(
Error::GenerationTranscript(ControllerError::Lease(LeaseError::Released)),
FailureKind::Transcript,
),
(
Error::BoundaryTranscript(ControllerError::Lease(LeaseError::Released)),
FailureKind::Transcript,
),
(
Error::Tools(StepResolutionError::Collision(
tool::Name::local("dup".to_owned()).unwrap(),
)),
FailureKind::ToolResolution,
),
(
Error::Request(RequestError::EmptySystemInstructionText { index: 0 }),
FailureKind::Request,
),
(
Error::Model(Arc::new(FakeError("down"))),
FailureKind::Model,
),
(
Error::Stream(StreamError::MissingTerminal),
FailureKind::Stream,
),
(
Error::Event(ValidationError::PayloadMismatch),
FailureKind::Event,
),
(
Error::Coordinate(CoordError::EmptyCallId),
FailureKind::Internal,
),
(Error::Prepare(DispatchError::Cancelled), FailureKind::Hook),
(Error::Hook(DispatchError::Cancelled), FailureKind::Hook),
(Error::ModelAborted("refused".to_owned()), FailureKind::Hook),
(
Error::Route(ToolBatchRouteError::new(
super::tool_batch::Error::cancelled_batch(),
)),
FailureKind::ToolExecution,
),
(
Error::Commit(ToolBatchCommitError::Cancelled),
FailureKind::ToolCommit,
),
(
Error::Input(InputAdmitError::EmptyRejectReason),
FailureKind::Input,
),
(Error::Cancelled, FailureKind::Internal),
(Error::MissingToolPhase, FailureKind::Internal),
(Error::ToolCallRejected, FailureKind::Internal),
(Error::ToolResultRejected, FailureKind::Internal),
(Error::AfterToolExecuteRejected, FailureKind::Internal),
];
for (error, expected) in cases {
let rendered = error.to_string();
assert!(!rendered.is_empty());
assert_eq!(error.classify(), expected, "{rendered}");
}
}
#[test]
fn a_dispatch_failure_retains_the_callback_error_it_carries() {
let probe = BoundaryProbe::new();
let hooks = probe.registry(&[(
"named",
BoundaryPlan {
failing: Some("decide_stop"),
..BoundaryPlan::default()
},
)]);
let (_session, lease) = lease(Vec::new());
lease.admit().unwrap();
let coord = generation_coord();
let emitter = SilentEmitter;
let state = run_state::Store::new();
let reader = lease.state().reader();
let transcript = lease.transcript_reader();
let resources = CallbackResources::new(&coord, &state, &reader, None, &emitter);
let adapter = hooks
.run_hooks()
.next()
.expect("the registry has one run registration");
let Err(error) = adapter.invoke_decide_stop(resources, &transcript, &Token::new()) else {
panic!("the registration fails");
};
let failure: &dyn error::Error = &error;
assert_eq!(failure.to_string(), "hook named failed: decide_stop");
let attributed = failure
.source()
.expect("a callback failure retains its attribution");
assert_eq!(attributed.to_string(), "hook named failed: decide_stop");
assert_eq!(
attributed
.downcast_ref::<AttributedError>()
.map(AttributedError::name),
Some("named"),
"the registration name stays reachable as a value, not only as text"
);
assert_eq!(
attributed
.source()
.expect("the attribution retains the callback error")
.to_string(),
"decide_stop"
);
}
#[tokio::test]
async fn a_boundary_cancelled_mid_callback_is_cancellation_not_a_failure() {
for boundary in [
"before_turn(trigger=0)",
"before_step(model=fake)",
"before_model(messages=0,ephemeral=1)",
"after_model(generated=true,finish=stop)",
"after_step(model=fake)",
"after_turn(trigger=0)",
"decide_stop",
"compose_continuation(turns=1,continuations=0,continuation=0)",
] {
let probe = BoundaryProbe::new();
let hooks = probe.registry(&[(
"cancels",
BoundaryPlan {
cancel_at: Some(boundary),
..BoundaryPlan::default()
},
)]);
let report = super::driver::Report::default();
let (terminal, _kinds, _snapshot) = run_boundary_driver(
&probe,
&hooks,
vec![text_generation("answered")],
&report,
Vec::new(),
)
.await;
assert!(
matches!(terminal, Terminal::Cancelled),
"{boundary}: {:?}",
terminal.failure().map(ToString::to_string)
);
}
}
#[tokio::test]
async fn a_short_circuit_cancelled_at_after_model_is_cancellation() {
let probe = BoundaryProbe::new();
let substitute = Assistant::new(
vec![Segment::Text {
text: "substituted".to_owned(),
extensions: extensions(),
}],
None,
);
let hooks = probe.registry(&[(
"short-cancel",
BoundaryPlan {
before_model: BeforeModelAnswer::ShortCircuit(substitute),
cancel_at: Some("after_model(generated=false,finish=)"),
..BoundaryPlan::default()
},
)]);
let report = super::driver::Report::default();
let (terminal, _kinds, _snapshot) =
run_boundary_driver(&probe, &hooks, Vec::new(), &report, Vec::new()).await;
assert!(matches!(terminal, Terminal::Cancelled));
}
#[tokio::test]
async fn a_refused_reservation_fails_the_mutation_that_asked_for_it() {
let probe = BoundaryProbe::new();
let hooks = probe.registry(&[(
"refuses",
BoundaryPlan {
fail_reservation: Some(0),
..BoundaryPlan::default()
},
)]);
let report = super::driver::Report::default();
let (terminal, _kinds, snapshot) = run_boundary_driver(
&probe,
&hooks,
vec![text_generation("answered")],
&report,
Vec::new(),
)
.await;
match terminal.failure() {
Some(error @ Error::GenerationTranscript(_)) => {
assert_eq!(
error.to_string(),
"committing the generation failed: transcript reservation rejected by hook refuses: reservation refused"
);
assert_eq!(
error
.cause()
.expect("a generation commit retains its dispatch failure")
.to_string(),
"hook refuses failed: reservation refused"
);
}
other => panic!(
"expected a generation commit failure: {:?}",
other.map(ToString::to_string)
),
}
assert!(snapshot.items().is_empty(), "zero partial items");
}
#[tokio::test]
async fn a_reservation_that_cancels_is_cancellation_at_every_boundary() {
for (label, plan) in [
(
"generation",
BoundaryPlan {
cancel_reservation: Some(0),
..BoundaryPlan::default()
},
),
(
"boundary batch",
BoundaryPlan {
before_turn: vec![system_item("before-turn")],
cancel_reservation: Some(0),
..BoundaryPlan::default()
},
),
] {
let probe = BoundaryProbe::new();
let hooks = probe.registry(&[("cancels-reservation", plan)]);
let report = super::driver::Report::default();
let (terminal, _kinds, snapshot) = run_boundary_driver(
&probe,
&hooks,
vec![text_generation("answered")],
&report,
Vec::new(),
)
.await;
assert!(
matches!(terminal, Terminal::Cancelled),
"{label}: {:?}",
terminal.failure().map(ToString::to_string)
);
assert!(snapshot.items().is_empty(), "{label}: zero partial items");
}
}
#[tokio::test]
async fn an_end_turn_after_a_tool_generation_closes_the_turn_it_would_have_continued() {
let (_session, lease) = lease(Vec::new());
lease.admit().unwrap();
let call = model_tool_call("call-1", "tool");
let model = FakeModel::ready(tool_generation(&[call]));
let executions = Arc::new(AtomicUsize::new(0));
let accepted_calls = Arc::new(AtomicUsize::new(0));
let acceptance_at_execute = Arc::new(Mutex::new(Vec::new()));
let tools = executable_tools(
&["tool"],
ExecutorMode::Success,
false,
&executions,
&accepted_calls,
&acceptance_at_execute,
);
let probe = BoundaryProbe::new();
let hooks = probe.registry(&[(
"ends-after-tools",
BoundaryPlan {
after_step: AfterStepAnswer::EndTurn,
..BoundaryPlan::default()
},
)]);
let reservations = ReservationResources::new(hooks.len(), lease.state().reader());
let coord = generation_coord();
let emitter = SilentEmitter;
let controller = TranscriptController::new(&lease, &hooks, &reservations, &coord, &emitter);
let states = (0..hooks.len())
.map(|_| run_state::Store::new())
.collect::<Vec<_>>();
let reader = lease.state().reader();
let callback_resources = states
.iter()
.map(|state| CallbackResources::new(&coord, state, &reader, None, &emitter))
.collect::<Vec<_>>();
let cancellation = probe.cancellation.clone();
let (producer, mut source) = channel(&cancellation);
let mut stream = source.take_stream().unwrap();
let facts = FactBuffer::default();
let outward = Outward::new(&producer, &facts);
let phase = ToolPhase::new(
outward,
&hooks,
&callback_resources,
&controller,
&coord,
None,
);
let inbox = InputMailbox::new();
let driver = Driver::new(&model, &lease, &inbox, &tools, &cancellation).with_tool_phase(phase);
let consumer = async {
expect_generation_prelude(&mut stream, true).await;
for seam in [DecisionPoint::ToolCall, DecisionPoint::ToolResult] {
let delivery = stream.recv().await.expect("strong tool delivery");
assert_eq!(delivery.decision_point(), Some(seam));
delivery.accept().unwrap();
}
};
let (terminal, ()) = tokio::join!(driver.run(RequestPlan::new(), &source), consumer);
assert!(matches!(terminal, Terminal::Completed));
assert_eq!(
model.calls(),
1,
"EndTurn closes the turn a tool generation would otherwise have continued"
);
assert!(
probe
.calls()
.contains(&"ends-after-tools:after_turn(trigger=0)".to_owned())
);
}
#[tokio::test]
async fn a_stop_decision_without_a_report_still_publishes_its_event_stop() {
let probe = BoundaryProbe::new();
let hooks = probe.registry(&[(
"stopper",
BoundaryPlan {
decide_stop: Some(StopCode::new("no.report".to_owned()).unwrap()),
..BoundaryPlan::default()
},
)]);
let (_session, lease) = lease(Vec::new());
lease.admit().unwrap();
let model = FakeModel::ready(vec![Ok(done())]);
let reservations = ReservationResources::new(hooks.len(), lease.state().reader());
let coord = generation_coord();
let emitter = SilentEmitter;
let controller = TranscriptController::new(&lease, &hooks, &reservations, &coord, &emitter);
let states = (0..hooks.len())
.map(|_| run_state::Store::new())
.collect::<Vec<_>>();
let reader = lease.state().reader();
let callback_resources = states
.iter()
.map(|state| CallbackResources::new(&coord, state, &reader, None, &emitter))
.collect::<Vec<_>>();
let cancellation = probe.cancellation.clone();
let (producer, mut source) = channel(&cancellation);
let mut stream = source.take_stream().unwrap();
let facts = FactBuffer::default();
let outward = Outward::new(&producer, &facts);
let phase = ToolPhase::new(
outward,
&hooks,
&callback_resources,
&controller,
&coord,
None,
);
let inbox = InputMailbox::new();
let tools = empty_tools();
let driver = Driver::new(&model, &lease, &inbox, &tools, &cancellation).with_tool_phase(phase);
let terminal = driver.run(RequestPlan::new(), &source).await;
drop(driver_producer_guard(producer));
assert!(matches!(terminal, Terminal::Stopped));
let mut kinds = Vec::new();
while let Some(delivery) = stream.recv().await {
kinds.push(delivery.event().kind());
}
assert_eq!(kinds.last(), Some(&Kind::Stop));
}
fn assert_replay_valid(snapshot: &Snapshot) {
let mut issued = Vec::new();
let mut answered = Vec::new();
for item in snapshot.items() {
match item {
Item::Assistant(assistant) => {
for segment in assistant.content() {
if let Segment::ToolCall { call, .. } = segment {
issued.push(call.id().as_str().to_owned());
}
}
}
Item::ToolResult(result) => answered.push(result.call_id().as_str().to_owned()),
Item::User(_) | Item::System(_) => {}
}
}
assert_eq!(
issued, answered,
"every issued tool call must be answered by something"
);
RequestPlan::new()
.try_into_request(snapshot.items().to_vec())
.expect("a replay-valid transcript freezes into a model request");
}
async fn run_with_tools<C, F>(
probe: &BoundaryProbe,
hooks: &hook::RuntimeRegistry,
tools: &tool::Registry,
inbox: &InputMailbox,
scripts: Vec<Vec<Result<model::Event, FakeError>>>,
report: &super::driver::Report,
consumer: C,
) -> (Terminal, Vec<Kind>, Snapshot)
where
C: FnOnce(super::event::Stream) -> F,
F: Future<Output = Vec<Kind>> + Send + 'static,
{
run_with_tools_on_session(
probe,
hooks,
tools,
inbox,
scripts,
report,
consumer,
Session::new("session".to_owned(), None),
)
.await
}
#[expect(
clippy::too_many_arguments,
reason = "the existing tool-Run harness plus its explicit Session form one test boundary"
)]
async fn run_with_tools_on_session<C, F>(
probe: &BoundaryProbe,
hooks: &hook::RuntimeRegistry,
tools: &tool::Registry,
inbox: &InputMailbox,
scripts: Vec<Vec<Result<model::Event, FakeError>>>,
report: &super::driver::Report,
consumer: C,
session: Session,
) -> (Terminal, Vec<Kind>, Snapshot)
where
C: FnOnce(super::event::Stream) -> F,
F: Future<Output = Vec<Kind>> + Send + 'static,
{
let lease = session.try_acquire().expect("the Run acquires its Session");
lease.admit().unwrap();
let model = FakeModel::scripted(
scripts
.into_iter()
.map(|script| Start::Ready(Ok(FakeEvents::ready(script))))
.collect(),
);
let reservations = ReservationResources::new(hooks.len(), lease.state().reader());
let coord = generation_coord();
let emitter = SilentEmitter;
let controller = TranscriptController::new(&lease, hooks, &reservations, &coord, &emitter);
let states = (0..hooks.len())
.map(|_| run_state::Store::new())
.collect::<Vec<_>>();
let reader = lease.state().reader();
let callback_resources = states
.iter()
.map(|state| CallbackResources::new(&coord, state, &reader, None, &emitter))
.collect::<Vec<_>>();
let cancellation = probe.cancellation.clone();
let (producer, mut source) = channel(&cancellation);
let stream = source.take_stream().unwrap();
let facts = FactBuffer::default();
let outward = Outward::new(&producer, &facts);
let phase = ToolPhase::new(
outward,
hooks,
&callback_resources,
&controller,
&coord,
None,
);
let driver = Driver::new(&model, &lease, inbox, tools, &cancellation)
.with_tool_phase(phase)
.with_report(report);
let consuming = tokio::spawn(consumer(stream));
let terminal = driver.run(RequestPlan::new(), &source).await;
drop(driver_producer_guard(producer));
let kinds = consuming.await.expect("the outward consumer completes");
let snapshot = lease.transcript_snapshot().unwrap();
(terminal, kinds, snapshot)
}
async fn accept_everything(mut stream: super::event::Stream) -> Vec<Kind> {
let mut kinds = Vec::new();
while let Some(delivery) = stream.recv().await {
kinds.push(delivery.event().kind());
let _ordinary_deliveries_need_no_decision = delivery.accept();
}
kinds
}
fn tool_probe_registry(probe: &BoundaryProbe, tools: Arc<dyn ToolHooks>) -> hook::RuntimeRegistry {
let mut builder = hook::RegistryBuilder::new();
builder.register(
hook::Registration::new("observer".to_owned())
.observer(probe.hooks("observer", BoundaryPlan::default()) as Arc<dyn ObserverHooks>),
);
builder.register(hook::Registration::new("tools".to_owned()).tool(tools));
builder.build_runtime().expect("tool probe registry builds")
}
fn success_tools(names: &[&str], executions: &Arc<AtomicUsize>) -> tool::Registry {
executable_tools(
names,
ExecutorMode::Success,
false,
executions,
&Arc::new(AtomicUsize::new(0)),
&Arc::new(Mutex::new(Vec::new())),
)
}
struct RenameToolHooks;
impl ToolHooks for RenameToolHooks {
fn before_tool_call(&self, scope: &ToolCallScope<'_>) -> HookResult<ToolCallDecision> {
Ok(ToolCallDecision::Modify(ToolCall::new(
Id::new("spoofed-id".to_owned()),
tool_call::Name::new("renamed".to_owned()),
scope.call().arguments().clone(),
)))
}
fn before_tool_result(&self, _scope: &ToolResultScope<'_>) -> HookResult<ToolResultDecision> {
Ok(ToolResultDecision::Continue)
}
fn after_tool_commit(&self, _scope: &AfterToolCommitScope<'_>) -> Vec<Item> {
Vec::new()
}
}
#[tokio::test]
async fn renamed_tool_exchange_replays_in_the_second_step_and_a_successor_run() {
let session = Session::new("rename-replay".to_owned(), None);
let probe = BoundaryProbe::new();
let hooks = tool_probe_registry(&probe, Arc::new(RenameToolHooks));
let executions = Arc::new(AtomicUsize::new(0));
let tools = success_tools(&["renamed"], &executions);
let first_report = super::driver::Report::default();
let first_inbox = InputMailbox::new();
let (first_terminal, _kinds, first_snapshot) = run_with_tools_on_session(
&probe,
&hooks,
&tools,
&first_inbox,
vec![
tool_generation(&[model_tool_call("call-1", "original")]),
text_generation("second step"),
],
&first_report,
accept_everything,
session.clone(),
)
.await;
assert!(matches!(first_terminal, Terminal::Completed));
assert_eq!(executions.load(Ordering::SeqCst), 1);
assert_eq!(first_snapshot.items().len(), 3);
let Item::Assistant(call_record) = &first_snapshot.items()[0] else {
panic!("the first committed item is the tool-calling assistant");
};
let call = call_record
.content()
.iter()
.find_map(|segment| match segment {
Segment::ToolCall { call, .. } => Some(call),
Segment::Text { .. } | Segment::Thinking { .. } => None,
})
.expect("the first assistant record carries the tool call");
assert_eq!(call.id().as_str(), "call-1");
assert_eq!(call.name().as_str(), "renamed");
assert!(matches!(
&first_snapshot.items()[1],
Item::ToolResult(result)
if result.call_id().as_str() == "call-1"
&& result.tool_name().as_str() == "renamed"
));
RequestPlan::new()
.try_into_request(first_snapshot.items().to_vec())
.expect("the committed exchange is replayable without weakening validation");
let successor_probe = BoundaryProbe::new();
let successor_report = super::driver::Report::default();
let successor_inbox = InputMailbox::new();
let (successor_terminal, _kinds, successor_snapshot) = run_with_tools_on_session(
&successor_probe,
&hooks,
&tools,
&successor_inbox,
vec![text_generation("successor")],
&successor_report,
accept_everything,
session,
)
.await;
assert!(matches!(successor_terminal, Terminal::Completed));
assert_eq!(successor_snapshot.items().len(), 4);
assert_replay_valid(&successor_snapshot);
}
struct EmptyNameToolHooks;
impl ToolHooks for EmptyNameToolHooks {
fn before_tool_call(&self, scope: &ToolCallScope<'_>) -> HookResult<ToolCallDecision> {
Ok(ToolCallDecision::Modify(ToolCall::new(
Id::new("spoofed-id".to_owned()),
tool_call::Name::new(String::new()),
scope.call().arguments().clone(),
)))
}
fn before_tool_result(&self, _scope: &ToolResultScope<'_>) -> HookResult<ToolResultDecision> {
Ok(ToolResultDecision::Continue)
}
fn after_tool_commit(&self, _scope: &AfterToolCommitScope<'_>) -> Vec<Item> {
Vec::new()
}
}
#[tokio::test]
async fn empty_modified_tool_name_fails_with_typed_hook_attribution_without_committing() {
let probe = BoundaryProbe::new();
let hooks = tool_probe_registry(&probe, Arc::new(EmptyNameToolHooks));
let report = super::driver::Report::default();
let inbox = InputMailbox::new();
let (terminal, _kinds, snapshot) = run_with_tools(
&probe,
&hooks,
&empty_tools(),
&inbox,
vec![tool_generation(&[model_tool_call("call-1", "original")])],
&report,
accept_everything,
)
.await;
let Some(Error::Prepare(DispatchError::Callback(failure))) = terminal.failure() else {
panic!("the invalid rewrite must be a hook failure");
};
assert_eq!(failure.name(), "tools");
assert!(matches!(
failure.error().source_ref().downcast_ref::<RequestError>(),
Some(RequestError::EmptyToolCallName { call_id }) if call_id == "call-1"
));
assert_eq!(terminal.outcome(), crate::value::lifecycle::Outcome::Failed);
assert_eq!(snapshot.version(), Version::ZERO);
assert!(snapshot.items().is_empty());
}
struct SettlingToolHooks {
blocked: &'static str,
short_circuited: &'static str,
}
impl ToolHooks for SettlingToolHooks {
fn before_tool_call(&self, scope: &ToolCallScope<'_>) -> HookResult<ToolCallDecision> {
let name = scope.call().name().as_str();
Ok(if name == self.blocked {
ToolCallDecision::Block("policy".to_owned())
} else if name == self.short_circuited {
ToolCallDecision::ShortCircuit(ToolResult::success_text("cached".to_owned()))
} else {
ToolCallDecision::Continue
})
}
fn before_tool_result(&self, _scope: &ToolResultScope<'_>) -> HookResult<ToolResultDecision> {
Ok(ToolResultDecision::Continue)
}
fn after_tool_commit(&self, _scope: &AfterToolCommitScope<'_>) -> Vec<Item> {
Vec::new()
}
}
struct AuditAfterCommit;
impl ToolHooks for AuditAfterCommit {
fn before_tool_call(&self, _scope: &ToolCallScope<'_>) -> HookResult<ToolCallDecision> {
Ok(ToolCallDecision::Continue)
}
fn before_tool_result(&self, _scope: &ToolResultScope<'_>) -> HookResult<ToolResultDecision> {
Ok(ToolResultDecision::Continue)
}
fn after_tool_commit(&self, scope: &AfterToolCommitScope<'_>) -> Vec<Item> {
scope.emit_audit(audit::Event::new("after.commit".into(), None));
Vec::new()
}
}
#[tokio::test]
async fn an_after_tool_audit_reaches_the_observer_before_it_is_published() {
let probe = BoundaryProbe::new();
let hooks = tool_probe_registry(&probe, Arc::new(AuditAfterCommit));
let executions = Arc::new(AtomicUsize::new(0));
let tools = success_tools(&["ran"], &executions);
let report = super::driver::Report::default();
let (terminal, kinds, snapshot) = run_with_tools(
&probe,
&hooks,
&tools,
&InputMailbox::new(),
vec![
tool_generation(&[model_tool_call("call-1", "ran")]),
text_generation("done"),
],
&report,
accept_everything,
)
.await;
assert!(matches!(terminal, Terminal::Completed));
assert_eq!(
probe.audits(),
["after.commit"],
"the observer sees the fact before the outward lane publishes it"
);
assert!(
kinds.contains(&Kind::Audit),
"the same fact still reaches the outward consumer"
);
assert_replay_valid(&snapshot);
}
#[tokio::test]
async fn every_tool_call_is_bracketed_by_a_start_and_a_disposition_milestone() {
let probe = BoundaryProbe::new();
let hooks = tool_probe_registry(
&probe,
Arc::new(SettlingToolHooks {
blocked: "blocked",
short_circuited: "cached",
}),
);
let executions = Arc::new(AtomicUsize::new(0));
let tools = success_tools(&["ran", "blocked", "cached"], &executions);
let calls = [
model_tool_call("call-1", "ran"),
model_tool_call("call-2", "blocked"),
model_tool_call("call-3", "cached"),
];
let report = super::driver::Report::default();
let (terminal, _kinds, snapshot) = run_with_tools(
&probe,
&hooks,
&tools,
&InputMailbox::new(),
vec![tool_generation(&calls), text_generation("done")],
&report,
accept_everything,
)
.await;
assert!(matches!(terminal, Terminal::Completed));
assert_eq!(
executions.load(Ordering::SeqCst),
1,
"only the unsettled call reaches an executor"
);
assert_eq!(
probe.observation_kinds(),
[
"TurnStart",
"StepStart",
"GenerationStart",
"ToolCallStart",
"ToolCallStart",
"ToolCallStart",
"ToolCallEnd",
"ToolCallEnd",
"ToolCallEnd",
"GenerationEnd",
"StepEnd",
"StepStart",
"GenerationStart",
"GenerationEnd",
"StepEnd",
"TurnEnd",
],
"every announced call starts, and every started call ends, in source order"
);
let ends = probe
.observation_facts()
.into_iter()
.filter(|fact| fact.starts_with("ToolCallEnd"))
.collect::<Vec<_>>();
assert_eq!(
ends,
[
concat!(
"ToolCallEnd call=call-1 name=ran end=Completed ",
"progress=PostHooksCompleted disposition=Executed result=true"
),
concat!(
"ToolCallEnd call=call-2 name=blocked end=Completed ",
"progress=PostHooksCompleted disposition=Blocked result=true"
),
concat!(
"ToolCallEnd call=call-3 name=cached end=Completed ",
"progress=PostHooksCompleted disposition=ShortCircuited result=true"
),
],
"a settled slot the post-hook phase does not apply to still reaches its close"
);
assert_replay_valid(&snapshot);
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum InterruptSeam {
PendingToolCall,
AbandonedToolCall,
PendingToolResult,
FailedCommit,
}
fn interrupt_registry(probe: &BoundaryProbe, seam: InterruptSeam) -> hook::RuntimeRegistry {
let hooks = probe.hooks(
"observer",
BoundaryPlan {
fail_reservation: (seam == InterruptSeam::FailedCommit).then_some(0),
..BoundaryPlan::default()
},
);
let mut builder = hook::RegistryBuilder::new();
builder.register(
hook::Registration::new("observer".to_owned())
.observer(Arc::clone(&hooks) as Arc<dyn ObserverHooks>)
.transcript(hooks as Arc<dyn TranscriptHooks>),
);
builder.build_runtime().expect("interrupt registry builds")
}
#[tokio::test]
async fn an_interrupted_run_closes_every_boundary_and_leaves_a_replayable_transcript() {
for seam in [
InterruptSeam::PendingToolCall,
InterruptSeam::AbandonedToolCall,
InterruptSeam::PendingToolResult,
InterruptSeam::FailedCommit,
] {
let probe = BoundaryProbe::new();
let hooks = interrupt_registry(&probe, seam);
let executions = Arc::new(AtomicUsize::new(0));
let tools = success_tools(&["first", "second"], &executions);
let calls = [
model_tool_call("call-1", "first"),
model_tool_call("call-2", "second"),
];
let report = super::driver::Report::default();
let token = probe.cancellation.clone();
let (terminal, kinds, snapshot) = run_with_tools(
&probe,
&hooks,
&tools,
&InputMailbox::new(),
vec![tool_generation(&calls), text_generation("unreached")],
&report,
move |mut stream| async move {
let mut kinds = Vec::new();
while let Some(delivery) = stream.recv().await {
let kind = delivery.event().kind();
kinds.push(kind);
match (seam, kind) {
(InterruptSeam::AbandonedToolCall, Kind::ToolCall) => {
drop(stream);
drop(delivery);
return kinds;
}
(InterruptSeam::PendingToolCall, Kind::ToolCall)
| (InterruptSeam::PendingToolResult, Kind::ToolResult) => token.cancel(),
_ => {
let _ordinary_deliveries_need_no_decision = delivery.accept();
}
}
}
kinds
},
)
.await;
assert_replay_valid(&snapshot);
let expected_outcome = match seam {
InterruptSeam::PendingToolCall | InterruptSeam::PendingToolResult => "Cancelled",
InterruptSeam::AbandonedToolCall => "Stopped",
InterruptSeam::FailedCommit => "Failed",
};
assert_eq!(
format!("{:?}", terminal.outcome()),
expected_outcome,
"seam: {seam:?}"
);
let facts = probe.observation_facts();
let started = facts
.iter()
.filter(|fact| fact.starts_with("ToolCallStart"))
.count();
let ended = facts
.iter()
.filter(|fact| fact.starts_with("ToolCallEnd"))
.count();
assert_eq!(started, ended, "seam: {seam:?}");
assert_eq!(
started,
match seam {
InterruptSeam::PendingToolCall | InterruptSeam::AbandonedToolCall => 1,
InterruptSeam::PendingToolResult | InterruptSeam::FailedCommit => 2,
},
"seam: {seam:?}"
);
let closing = facts
.iter()
.skip_while(|fact| !fact.starts_with("ToolCallEnd"))
.cloned()
.collect::<Vec<_>>();
let (milestone, expected_end) = match seam {
InterruptSeam::PendingToolCall => (
"progress=Observed disposition=NotExecuted result=false",
"end=Cancelled error=the run was cancelled cause=none",
),
InterruptSeam::AbandonedToolCall => (
"progress=Observed disposition=NotExecuted result=false",
"end=Stopped",
),
InterruptSeam::PendingToolResult => (
"progress=ResultCommitted disposition=Executed result=true",
"end=Cancelled error=the run was cancelled cause=none",
),
InterruptSeam::FailedCommit => (
"progress=ResultAvailable disposition=Executed result=true",
concat!(
"end=Failed error=the tool batch commit failed: the atomic append failed: ",
"transcript reservation rejected by hook observer: reservation refused ",
"cause=hook observer failed: reservation refused",
),
),
};
assert_eq!(
closing.first().map(String::as_str),
Some(format!("ToolCallEnd call=call-1 name=first {expected_end} {milestone}").as_str()),
"seam: {seam:?}"
);
assert_eq!(
closing.get(started..),
Some(
[
format!("GenerationEnd {expected_end} finish=\"\" completion=Unknown"),
format!("StepEnd {expected_end}"),
format!("TurnEnd {expected_end} steps=1"),
]
.as_slice()
),
"boundaries close innermost first, seam: {seam:?}"
);
assert_eq!(
snapshot.items().len(),
match seam {
InterruptSeam::PendingToolResult => 3,
_ => 0,
},
"seam: {seam:?}"
);
assert!(
!kinds.contains(&Kind::TurnEnd) && !kinds.contains(&Kind::StepEnd),
"an interrupted Run publishes no closing event: the lane fails closed"
);
}
}
#[test]
fn every_disposition_reaches_exactly_one_milestone_family_the_validator_accepts() {
use super::driver::{CallProgress, ResultStage};
use super::hook::{ToolCallEndObservation, ToolCallProgressKind};
let interrupted = LifecycleEnd::Cancelled {
error: Arc::new(FakeError("interrupted")),
};
let coord = tool_call_coord_for("call", "tool");
let call = model_tool_call("call", "tool");
for disposition in [
tool::CallDisposition::Executed,
tool::CallDisposition::ShortCircuited,
tool::CallDisposition::Blocked,
tool::CallDisposition::Rejected,
tool::CallDisposition::Truncated,
tool::CallDisposition::NotExecuted,
] {
let mut reached = Vec::new();
for stage in [
ResultStage::Routed,
ResultStage::Committed,
ResultStage::Closed,
] {
let (end, milestone) = CallProgress::settle(
disposition,
ToolResult::success_text("out".to_owned()),
stage,
)
.terminal(&interrupted);
assert_eq!(
milestone.disposition(),
disposition,
"the milestone reports the disposition it was built from"
);
reached.push((milestone.kind(), format!("{:?}", end.outcome())));
ToolCallEndObservation::try_new(
coord.clone(),
end,
call.clone(),
call.clone(),
milestone,
)
.expect("the loop's pairing is one the validated observation accepts");
}
let expected = match disposition {
tool::CallDisposition::Truncated | tool::CallDisposition::NotExecuted => vec![
(ToolCallProgressKind::Observed, "Cancelled".to_owned()),
(ToolCallProgressKind::Observed, "Cancelled".to_owned()),
(ToolCallProgressKind::Observed, "Cancelled".to_owned()),
],
_ => vec![
(
ToolCallProgressKind::ResultAvailable,
"Cancelled".to_owned(),
),
(
ToolCallProgressKind::ResultCommitted,
"Cancelled".to_owned(),
),
(
ToolCallProgressKind::PostHooksCompleted,
"Completed".to_owned(),
),
],
};
assert_eq!(reached, expected, "disposition: {disposition:?}");
}
}
fn tool_call_coord_for(id: &str, name: &str) -> Coord {
let one = NonZeroU64::MIN;
Coord::at_tool_call(
"session".to_owned(),
"run".to_owned(),
StepIndex::new(one, one, one).unwrap(),
one,
id.to_owned(),
name.to_owned(),
)
.unwrap()
}
struct EmittingExecutor;
impl Executor for EmittingExecutor {
type Output = String;
async fn execute(
&self,
invocation: &Invocation,
call: &ToolCall,
_cancellation: Token,
) -> Result<Self::Output, ExecutorError> {
invocation
.emitter()
.emit_audit(audit::Event::new("sandbox".to_owned(), None))
.await
.map_err(|error| ExecutorError::new(error.to_string()))?;
invocation
.emitter()
.emit_error(Arc::new(FakeError("degraded")))
.await
.map_err(|error| ExecutorError::new(error.to_string()))?;
Ok(call.id().as_str().to_owned())
}
}
fn emitting_tools() -> tool::Registry {
let mut builder = tool::RegistryBuilder::new();
builder.register(tool::Registration::client(
Descriptor::new(
tool::Name::local("tool".to_owned()).unwrap(),
"emitting tool".to_owned(),
Payload::copy_from_slice(b"{}"),
),
Arc::new(EmittingExecutor),
Arc::new(|output: String| Ok(ToolResult::success_text(output))),
));
builder.build().unwrap()
}
fn full_generation(call: &ToolCall) -> Vec<Result<model::Event, FakeError>> {
vec![
Ok(model::Event::ContentDelta {
id: "thought".to_owned(),
kind: ContentKind::Thinking,
delta: "considering".to_owned(),
}),
Ok(model::Event::ContentFinal {
id: "thought".to_owned(),
kind: ContentKind::Thinking,
content: "considering".to_owned(),
extensions: extensions(),
}),
Ok(model::Event::ContentDelta {
id: "text".to_owned(),
kind: ContentKind::Text,
delta: "calling".to_owned(),
}),
Ok(model::Event::ContentFinal {
id: "text".to_owned(),
kind: ContentKind::Text,
content: "calling".to_owned(),
extensions: extensions(),
}),
Ok(model::Event::ToolCallStart {
id: call.id().clone(),
name: call.name().clone(),
}),
Ok(model::Event::ToolCallArgumentsDelta {
id: call.id().clone(),
delta: Payload::copy_from_slice(b"{}"),
}),
Ok(model::Event::ToolCallFinal {
call: call.clone(),
extensions: extensions(),
}),
Ok(model::Event::Done {
finish_reason: "tools".to_owned(),
completion: CompletionReason::ToolCalls,
provider_data: Vec::new(),
}),
]
}
#[tokio::test]
async fn every_event_kind_reaches_a_consumer_from_a_real_run() {
use super::event::testing::{ALL_KINDS, Emission, emission};
let all = ALL_KINDS;
let mut categories = all.to_vec();
categories.sort_unstable_by_key(|kind| format!("{kind:?}"));
categories.dedup();
assert_eq!(
categories.len(),
all.len(),
"the enumerated categories are a set"
);
let probe = BoundaryProbe::new();
let hooks = probe.registry(&[(
"emitter",
BoundaryPlan {
decide_stop: Some(StopCode::new("budget".to_owned()).unwrap()),
stop_after_decisions: 1,
compose_continuation: ContinuationAnswer::Replace(vec![Part::text(
"one final turn".to_owned(),
)]),
reject_next_turn_input: Some("out of scope"),
..BoundaryPlan::default()
},
)]);
let tools = emitting_tools();
let report = super::driver::Report::default();
let inbox = InputMailbox::new();
inbox.push(InputBatch::new(InputTarget::NextStep, vec![user("and one more thing")]).unwrap());
inbox.push(InputBatch::new(InputTarget::NextTurn, vec![user("unrelated")]).unwrap());
let (terminal, kinds, _snapshot) = run_with_tools(
&probe,
&hooks,
&tools,
&inbox,
vec![
full_generation(&model_tool_call("call", "tool")),
text_generation("answered"),
text_generation("final turn"),
],
&report,
accept_everything,
)
.await;
assert!(
matches!(terminal, Terminal::Stopped),
"{:?}: {}",
terminal.outcome(),
terminal
.failure()
.map(ToString::to_string)
.unwrap_or_default()
);
assert_eq!(
report.stop_code().map(|code| code.as_str().to_owned()),
Some("budget".to_owned())
);
let missing = all
.iter()
.filter(|kind| !kinds.contains(kind))
.map(|kind| format!("{kind:?}"))
.collect::<Vec<_>>();
assert!(
missing.is_empty(),
"these event categories have no production emitter: {missing:?}"
);
let unexpected = kinds
.iter()
.filter(|kind| !all.contains(kind))
.map(|kind| format!("{kind:?}"))
.collect::<Vec<_>>();
assert!(
unexpected.is_empty(),
"unenumerated categories: {unexpected:?}"
);
let exercised = kinds.iter().map(|kind| emission(*kind)).collect::<Vec<_>>();
for path in [
Emission::Content,
Emission::Lifecycle,
Emission::ToolCall,
Emission::ToolResult,
Emission::Extension,
Emission::Stop,
Emission::Input,
] {
assert!(
exercised.contains(&path),
"no event was published through {path:?}"
);
}
}
#[test]
fn a_boundary_interruption_renders_its_cause_through_display_not_debug() {
use super::driver::boundary_interruption;
let provider = boundary_interruption(&super::driver::Error::Model(Arc::new(FakeError(
"provider exploded",
))));
assert_eq!(
format!("{provider:?}"),
concat!(
r#"BoundaryInterruption { message: "the model provider failed: provider exploded", "#,
r#"cause: Some("provider exploded") }"#
)
);
assert_eq!(
provider.to_string(),
"the model provider failed: provider exploded"
);
let cancelled = boundary_interruption(&super::driver::Error::Cancelled);
assert_eq!(
format!("{cancelled:?}"),
r#"BoundaryInterruption { message: "the run was cancelled", cause: None }"#
);
}
struct ObservedEvent {
kind: Kind,
message: Option<String>,
provider_source: Option<&'static str>,
coord: Option<Coord>,
decision_point: Option<DecisionPoint>,
accepted: Result<(), DecisionError>,
debug: String,
}
struct FixedProvider {
resolution: Mutex<Option<ProviderResolution>>,
}
impl FixedProvider {
fn provider(resolution: ProviderResolution) -> Arc<dyn Provider> {
Arc::new(Self {
resolution: Mutex::new(Some(resolution)),
})
}
}
impl Provider for FixedProvider {
fn resolve<'a>(
&'a self,
_transcript: &'a Snapshot,
_cancellation: Token,
) -> Pin<Box<dyn Future<Output = ProviderResolution> + Send + 'a>> {
let resolution = self
.resolution
.lock()
.unwrap()
.take()
.expect("each provider resolves once per Step");
Box::pin(ready(resolution))
}
}
fn visible_client_tool(name: &str) -> tool::Registration {
client_tool(
name,
ExecutorMode::Success,
false,
&Arc::new(AtomicUsize::new(0)),
&Arc::new(AtomicUsize::new(0)),
&Arc::new(Mutex::new(Vec::new())),
)
}
async fn run_step_observed(
model: &FakeModel,
tools: &tool::Registry,
) -> (Terminal, Vec<ObservedEvent>, Snapshot) {
let hooks = empty_hooks();
let (_session, lease) = lease(Vec::new());
lease.admit().unwrap();
let reservations = ReservationResources::new(hooks.len(), lease.state().reader());
let coord = generation_coord();
let emitter = SilentEmitter;
let controller = TranscriptController::new(&lease, &hooks, &reservations, &coord, &emitter);
let states = (0..hooks.len())
.map(|_| run_state::Store::new())
.collect::<Vec<_>>();
let reader = lease.state().reader();
let callback_resources = states
.iter()
.map(|state| CallbackResources::new(&coord, state, &reader, None, &emitter))
.collect::<Vec<_>>();
let cancellation = Token::new();
let (producer, mut source) = channel(&cancellation);
let mut stream = source.take_stream().unwrap();
let facts = FactBuffer::default();
let outward = Outward::new(&producer, &facts);
let phase = ToolPhase::new(
outward,
&hooks,
&callback_resources,
&controller,
&coord,
None,
);
let inbox = InputMailbox::new();
let driver = Driver::new(model, &lease, &inbox, tools, &cancellation).with_tool_phase(phase);
let terminal = driver.run(RequestPlan::new(), &source).await;
drop(driver_producer_guard(producer));
let mut events = Vec::new();
while let Some(delivery) = stream.recv().await {
let event = delivery.event();
let provider_source = event
.as_error()
.and_then(|error| error.source())
.and_then(|source| source.downcast_ref::<FakeError>())
.map(|source| source.0);
events.push(ObservedEvent {
kind: event.kind(),
message: event.as_error().map(ToString::to_string),
provider_source,
coord: event.coord().cloned(),
decision_point: delivery.decision_point(),
accepted: delivery.accept(),
debug: format!("{event:?}"),
});
}
let snapshot = lease.transcript_snapshot().unwrap();
(terminal, events, snapshot)
}
#[tokio::test]
async fn provider_degradation_emits_one_ordinary_error_per_provider_in_order() {
let mut builder = tool::RegistryBuilder::new();
builder
.register(visible_client_tool("static-tool"))
.provider(FixedProvider::provider(ProviderResolution::failed(
vec![
visible_client_tool("partial-a-1"),
visible_client_tool("partial-a-2"),
],
ProviderError::new("alpha catalogue offline".to_owned())
.with_source(FakeError("catalogue transport offline")),
)))
.provider(FixedProvider::provider(ProviderResolution::success(vec![
visible_client_tool("dynamic-b"),
])))
.provider(FixedProvider::provider(ProviderResolution::failed(
vec![visible_client_tool("partial-c-1")],
ProviderError::new("gamma handshake failed".to_owned()),
)));
let tools = builder.build().unwrap();
let model = FakeModel::ready(text_generation("answered"));
let (terminal, events, _snapshot) = run_step_observed(&model, &tools).await;
assert!(matches!(terminal, Terminal::Completed));
assert_eq!(
model.calls(),
1,
"a degraded provider must not stop the model request"
);
let names = {
let requests = model.requests.lock().unwrap();
requests[0]
.tools()
.iter()
.map(|tool| tool.name().as_str().to_owned())
.collect::<Vec<_>>()
};
assert_eq!(
names,
["static-tool".to_owned(), "dynamic-b".to_owned()],
"partial entries are discarded; static and healthy dynamic entries remain"
);
let errors: Vec<&ObservedEvent> = events.iter().filter(|e| e.kind == Kind::Error).collect();
assert_eq!(
errors.len(),
2,
"one Error per degraded provider, not one per partial entry"
);
assert_eq!(
errors[0].message.as_deref(),
Some("alpha catalogue offline")
);
assert_eq!(errors[1].message.as_deref(), Some("gamma handshake failed"));
assert_eq!(
errors[0].provider_source,
Some("catalogue transport offline"),
"the outward ProviderError retains its downcastable typed source"
);
assert_eq!(errors[1].provider_source, None);
for error in &errors {
assert_eq!(
error.decision_point, None,
"a degradation is not a strong seam"
);
assert_eq!(error.accepted, Err(DecisionError::NotRequired));
assert!(
!error.debug.contains("ProviderError"),
"the Error payload renders the diagnostic through Display, not the wrapper: {}",
error.debug
);
}
let step_start = events
.iter()
.position(|e| e.kind == Kind::StepStart)
.expect("the Step opened");
let first_error = events
.iter()
.position(|e| e.kind == Kind::Error)
.expect("a degradation was published");
let last_error = events.iter().rposition(|e| e.kind == Kind::Error).unwrap();
let first_content = events
.iter()
.position(|e| matches!(e.kind, Kind::TextDelta | Kind::Text))
.expect("the model answered");
assert!(
step_start < first_error && last_error < first_content,
"degradations sit between StepStart and the model's output"
);
assert_eq!(
errors[0].coord, events[step_start].coord,
"the Error is stamped with the Step coordinate it degraded"
);
}
#[tokio::test]
async fn a_healthy_provider_emits_no_degradation_error() {
let mut builder = tool::RegistryBuilder::new();
builder
.register(visible_client_tool("static-tool"))
.provider(FixedProvider::provider(ProviderResolution::success(vec![
visible_client_tool("dynamic-b"),
])));
let tools = builder.build().unwrap();
let model = FakeModel::ready(text_generation("answered"));
let (terminal, events, _snapshot) = run_step_observed(&model, &tools).await;
assert!(matches!(terminal, Terminal::Completed));
assert!(
events.iter().all(|event| event.kind != Kind::Error),
"a successful resolution emits nothing"
);
let names = {
let requests = model.requests.lock().unwrap();
requests[0]
.tools()
.iter()
.map(|tool| tool.name().as_str().to_owned())
.collect::<Vec<_>>()
};
assert_eq!(names, ["static-tool".to_owned(), "dynamic-b".to_owned()]);
}
#[test]
fn a_provider_error_renders_its_message_verbatim() {
let provider_error = ProviderError::new("alpha catalogue offline".to_owned());
let error: Arc<dyn error::Error + Send + Sync> = Arc::new(provider_error);
assert_eq!(error.to_string(), "alpha catalogue offline");
assert_eq!(
format!("{error:?}"),
r#"ProviderError("alpha catalogue offline")"#
);
let event = super::event::Event::from_error(Arc::clone(&error));
assert_eq!(
format!("{event:?}"),
r#"Event { kind: Error, coord: None, item_id: None, payload: Error("alpha catalogue offline") }"#
);
assert_eq!(
event.as_error().map(ToString::to_string).as_deref(),
Some("alpha catalogue offline")
);
}