#![cfg(test)]
use std::collections::VecDeque;
use std::error::Error;
use std::fmt;
use std::future::{Future, ready};
use std::pin::Pin;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::{Arc, Condvar, Mutex};
use std::task::{Context, Poll};
use std::time::Duration;
use crate::cancel::Token;
use tokio::sync::Notify;
use super::agent::{Agent, CloseError, Config};
use super::event::{Kind, channel};
use super::hook::FactBuffer;
use super::hook::{self, Emitter, LifecycleObservation, ObserverHooks, ObserverScope, ReadScope};
use super::model::{self, Client, CompletionReason, Info, Request};
use super::run::{EventsTakenError, TaskAbortedError, TaskNotStartedError, TaskPanickedError};
use super::session::Session;
use super::tool::{self, Descriptor, Executor, ExecutorError, Invocation};
use crate::audit;
use crate::opaque::Payload;
use crate::redact_tests::assert_states;
use crate::transcript::item::Item;
use crate::value::content::{Part, SegmentExtensions};
use crate::value::lifecycle::Outcome;
use crate::value::tool_call::{self, Id, ToolCall, ToolResult};
#[derive(Clone)]
enum Mode {
Complete,
Pending,
Fail,
Panic,
Blocked,
Tool,
}
struct TestModel {
mode: Mode,
entered: AtomicBool,
entered_notify: Notify,
release: Notify,
ephemeral: Mutex<Vec<String>>,
messages: Mutex<Vec<String>>,
calls: AtomicUsize,
}
impl TestModel {
fn new(mode: Mode) -> Self {
Self {
mode,
entered: AtomicBool::new(false),
entered_notify: Notify::new(),
release: Notify::new(),
ephemeral: Mutex::new(Vec::new()),
messages: Mutex::new(Vec::new()),
calls: AtomicUsize::new(0),
}
}
async fn wait_until_entered(&self) {
while !self.entered.load(Ordering::Acquire) {
self.entered_notify.notified().await;
}
}
}
#[derive(Debug)]
struct ModelError;
impl fmt::Display for ModelError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("test model failed")
}
}
impl Error for ModelError {}
struct Events(VecDeque<Result<model::Event, ModelError>>);
impl futures_core::Stream for Events {
type Item = Result<model::Event, ModelError>;
fn poll_next(mut self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<Option<Self::Item>> {
Poll::Ready(self.0.pop_front())
}
}
impl Client for TestModel {
type Error = ModelError;
type Stream = Events;
fn info(&self) -> Info {
Info::new("run-test".to_owned(), 1024).with_tools(true)
}
async fn generate(&self, request: Request) -> Result<Self::Stream, Self::Error> {
self.ephemeral.lock().unwrap().extend(
request
.ephemeral()
.iter()
.map(|part| part.fields().text().to_owned()),
);
self.messages
.lock()
.unwrap()
.extend(request.messages().iter().filter_map(|item| match item {
Item::User(user) => Some(user.parts()[0].fields().text().to_owned()),
Item::Assistant(_) | Item::System(_) | Item::ToolResult(_) => None,
}));
self.entered.store(true, Ordering::Release);
self.entered_notify.notify_waiters();
match self.mode {
Mode::Complete => Ok(done_events()),
Mode::Pending => std::future::pending().await,
Mode::Fail => Err(ModelError),
Mode::Panic => {
self.release.notified().await;
panic!("core model panic");
}
Mode::Blocked => {
self.release.notified().await;
Ok(done_events())
}
Mode::Tool => {
if self.calls.fetch_add(1, Ordering::SeqCst) == 0 {
Ok(tool_events())
} else {
Ok(done_events())
}
}
}
}
}
fn done_events() -> Events {
Events(
vec![Ok(model::Event::Done {
finish_reason: "stop".to_owned(),
completion: CompletionReason::Stop,
provider_data: Vec::new(),
})]
.into(),
)
}
fn tool_events() -> Events {
let id = Id::new("call-1".to_owned());
let name = tool_call::Name::new("echo".to_owned());
let call = ToolCall::new(id.clone(), name.clone(), Payload::copy_from_slice(b"{}"));
Events(
vec![
Ok(model::Event::ToolCallStart {
id: id.clone(),
name,
}),
Ok(model::Event::ToolCallArgumentsDelta {
id,
delta: Payload::copy_from_slice(b"{}"),
}),
Ok(model::Event::ToolCallFinal {
call,
extensions: SegmentExtensions::new(String::new(), None, None),
}),
Ok(model::Event::Done {
finish_reason: "tools".to_owned(),
completion: CompletionReason::ToolCalls,
provider_data: Vec::new(),
}),
]
.into(),
)
}
struct EchoExecutor;
impl Executor for EchoExecutor {
type Output = String;
fn execute(
&self,
_invocation: &Invocation,
_call: &ToolCall,
_cancellation: Token,
) -> impl Future<Output = Result<Self::Output, ExecutorError>> + Send {
ready(Ok("echoed".to_owned()))
}
}
fn echo_tools() -> super::tool::Registry {
let mut builder = tool::RegistryBuilder::new();
builder.register(tool::Registration::client(
Descriptor::new(
tool::Name::local("echo".into()).unwrap(),
"echo".into(),
Payload::copy_from_slice(b"{}"),
),
Arc::new(EchoExecutor),
Arc::new(|output: String| Ok(ToolResult::success_text(output))),
));
builder.build().unwrap()
}
fn agent(mode: Mode) -> (Agent<TestModel>, Arc<TestModel>) {
agent_with_hooks(mode, hook::RegistryBuilder::new())
}
fn agent_with_hooks(
mode: Mode,
hooks: hook::RegistryBuilder,
) -> (Agent<TestModel>, Arc<TestModel>) {
let model = Arc::new(TestModel::new(mode));
let config = Config::new(
Arc::clone(&model),
tool::RegistryBuilder::new().build().unwrap(),
hooks,
);
(Agent::new(config).unwrap(), model)
}
async fn start(
agent: &Agent<TestModel>,
session: Session,
token: Token,
) -> super::run::Run<TestModel> {
agent
.start(session, vec![Part::text("input".into())], token)
.await
.unwrap()
}
async fn start_observed(
agent: &Agent<TestModel>,
session: Session,
token: Token,
) -> (super::run::Run<TestModel>, super::event::Stream) {
agent
.start_observed(session, vec![Part::text("input".into())], token)
.await
.unwrap()
}
const EMPTY_GENERATION: [Kind; 6] = [
Kind::InputAdmitted,
Kind::TurnStart,
Kind::StepStart,
Kind::GenerationEnd,
Kind::StepEnd,
Kind::TurnEnd,
];
async fn expect_kinds(events: &mut super::event::Stream, expected: &[Kind]) {
for kind in expected {
let delivery = events.recv().await.expect("expected outward event");
assert_eq!(delivery.event().kind(), *kind);
}
}
#[tokio::test]
async fn a_run_that_recorded_no_input_announces_none() {
let (agent, _model) = agent(Mode::Complete);
let session = Session::new("no-input".into(), None);
let run = agent
.start(session, Vec::new(), Token::new())
.await
.expect("a Run with no parts is admitted");
let mut events = run.take_events().unwrap();
let mut kinds = Vec::new();
while let Some(delivery) = events.recv().await {
kinds.push(delivery.event().kind());
}
assert_eq!(run.wait().await.outcome(), Outcome::Completed);
assert_eq!(
kinds,
&EMPTY_GENERATION[1..],
"the same sequence, without the admitted record that does not exist"
);
agent.close().await.unwrap();
}
#[tokio::test]
async fn observed_start_retains_a_fast_runs_events_after_terminal_cleanup() {
let (agent, _model) = agent(Mode::Complete);
let (run, mut events) = start_observed(
&agent,
Session::new("observed-fast".into(), None),
Token::new(),
)
.await;
let terminal = run.wait().await;
assert_eq!(terminal.outcome(), Outcome::Completed);
assert_eq!(run.take_events().err(), Some(EventsTakenError));
expect_kinds(&mut events, &EMPTY_GENERATION).await;
assert!(events.recv().await.is_none());
agent.close().await.unwrap();
}
#[tokio::test]
async fn observed_start_returns_the_same_pre_admission_errors() {
let (agent, _model) = agent(Mode::Complete);
agent.close().await.unwrap();
let error = agent
.start_observed(
Session::new("observed-closed".into(), None),
vec![Part::text("input".into())],
Token::new(),
)
.await
.unwrap_err();
assert!(matches!(error, super::agent::StartError::AgentClosed));
}
#[tokio::test]
async fn natural_completion_freezes_getters_and_releases_before_wait() {
let (agent, model) = agent(Mode::Complete);
let session = Session::new("natural".into(), None);
let revision = agent.revision().id();
let run = start(&agent, session.clone(), Token::new()).await;
assert_eq!(run.revision().id(), revision);
assert_eq!(run.session().id(), "natural");
let terminal = run.wait().await;
assert_eq!(terminal.outcome(), Outcome::Completed);
assert_eq!(terminal.coord().session_id(), "natural");
assert!(terminal.stop_code().is_none());
assert!(terminal.failure().is_none());
assert!(model.ephemeral.lock().unwrap().is_empty());
assert_eq!(*model.messages.lock().unwrap(), ["input".to_owned()]);
let successor = session.try_acquire().expect("wait follows Lease release");
drop(successor);
agent.close().await.unwrap();
}
#[tokio::test]
async fn parent_cancel_and_consumer_drop_map_distinct_terminals() {
let (canceled_agent, canceled_model) = agent(Mode::Pending);
let canceled_session = Session::new("canceled".into(), None);
let parent = Token::new();
let canceled = start(&canceled_agent, canceled_session, parent.clone()).await;
canceled_model.wait_until_entered().await;
parent.cancel();
assert_eq!(canceled.wait().await.outcome(), Outcome::Cancelled);
let (stopped_agent, stopped_model) = agent(Mode::Pending);
let stopped_session = Session::new("stopped".into(), None);
let stopped = start(&stopped_agent, stopped_session, Token::new()).await;
let events = stopped.take_events().unwrap();
stopped_model.wait_until_entered().await;
drop(events);
assert_eq!(stopped.wait().await.outcome(), Outcome::Stopped);
}
#[tokio::test]
async fn events_are_one_take_and_close_abandons_untaken_stream() {
let (agent, model) = agent(Mode::Pending);
let run = start(&agent, Session::new("events".into(), None), Token::new()).await;
let events = run.take_events().unwrap();
assert_eq!(run.take_events().err(), Some(EventsTakenError));
model.wait_until_entered().await;
drop(events);
assert_eq!(run.close().await.outcome(), Outcome::Stopped);
assert_eq!(
EventsTakenError.to_string(),
"Run events were already consumed"
);
}
#[tokio::test]
async fn wait_is_ready_only_after_driver_barrier_release_and_detach() {
let (agent, model) = agent(Mode::Blocked);
let session = Session::new("barrier".into(), None);
let run = start(&agent, session.clone(), Token::new()).await;
model.wait_until_entered().await;
assert!(session.try_acquire().is_none());
assert!(matches!(
agent.close().await,
Err(CloseError::Busy { active_runs: 1 })
));
let waiter_run = run.clone();
let waiter = tokio::spawn(async move { waiter_run.wait().await });
tokio::task::yield_now().await;
assert!(!waiter.is_finished());
model.release.notify_one();
let terminal = waiter.await.unwrap();
assert_eq!(terminal.outcome(), Outcome::Completed);
drop(session.try_acquire().expect("terminal follows release"));
agent.close().await.unwrap();
}
#[tokio::test]
async fn stop_and_multiple_waiters_share_the_same_arc_terminal() {
let (agent, model) = agent(Mode::Pending);
let run = start(&agent, Session::new("waiters".into(), None), Token::new()).await;
model.wait_until_entered().await;
let first_run = run.clone();
let second_run = run.clone();
let third_run = run.clone();
let first = tokio::spawn(async move { first_run.wait().await });
let second = tokio::spawn(async move { second_run.stop().await });
let third = tokio::spawn(async move { third_run.close().await });
let from_owner = run.wait().await;
let from_first = first.await.unwrap();
let from_second = second.await.unwrap();
let from_third = third.await.unwrap();
assert!(Arc::ptr_eq(&from_owner, &from_first));
assert!(Arc::ptr_eq(&from_owner, &from_second));
assert!(Arc::ptr_eq(&from_owner, &from_third));
assert_eq!(from_owner.outcome(), Outcome::Stopped);
}
#[tokio::test]
async fn final_handle_drop_only_requests_stop_and_owned_task_finishes_cleanup() {
let (agent, model) = agent(Mode::Pending);
let session = Session::new("drop".into(), None);
let run = start(&agent, session.clone(), Token::new()).await;
model.wait_until_entered().await;
let clone = run.clone();
drop(clone);
tokio::task::yield_now().await;
assert!(
session.try_acquire().is_none(),
"non-final Drop does nothing"
);
drop(run);
let successor = tokio::time::timeout(Duration::from_secs(1), async {
loop {
if let Some(lease) = session.try_acquire() {
break lease;
}
tokio::task::yield_now().await;
}
})
.await
.expect("the sole task remains owned after public handles drop");
drop(successor);
agent.close().await.unwrap();
}
#[tokio::test]
async fn failed_model_returns_failed_terminal_detail() {
let (agent, _model) = agent(Mode::Fail);
let run = start(&agent, Session::new("failed".into(), None), Token::new()).await;
let terminal = run.wait().await;
assert_eq!(terminal.outcome(), Outcome::Failed);
assert_eq!(
terminal.failure_kind(),
Some(super::run::FailureKind::Model),
"the provider stage is the matchable summary"
);
let failure = terminal.failure().unwrap();
assert_eq!(
failure.to_string(),
"the model provider failed: test model failed"
);
assert!(
failure
.source()
.expect("the driver failure retains its cause")
.downcast_ref::<ModelError>()
.is_some()
);
}
#[tokio::test]
async fn facade_binds_and_executes_configured_tools() {
let model = Arc::new(TestModel::new(Mode::Tool));
let agent = Agent::new(Config::new(
Arc::clone(&model),
echo_tools(),
hook::RegistryBuilder::new(),
))
.unwrap();
let run = start(&agent, Session::new("tool".into(), None), Token::new()).await;
let mut events = run.take_events().unwrap();
expect_kinds(
&mut events,
&[Kind::InputAdmitted, Kind::TurnStart, Kind::StepStart],
)
.await;
for expected in [Kind::ToolCall, Kind::ToolResult] {
let delivery = events.recv().await.unwrap();
assert_eq!(delivery.event().kind(), expected);
delivery.accept().unwrap();
}
assert_eq!(run.wait().await.outcome(), Outcome::Completed);
assert_eq!(model.calls.load(Ordering::SeqCst), 2);
}
#[tokio::test]
async fn observed_stream_keeps_a_strong_delivery_pending_until_acceptance() {
let model = Arc::new(TestModel::new(Mode::Tool));
let agent = Agent::new(Config::new(
Arc::clone(&model),
echo_tools(),
hook::RegistryBuilder::new(),
))
.unwrap();
let (run, mut events) = agent
.start_observed(
Session::new("observed-strong".into(), None),
vec![Part::text("input".into())],
Token::new(),
)
.await
.unwrap();
expect_kinds(
&mut events,
&[Kind::InputAdmitted, Kind::TurnStart, Kind::StepStart],
)
.await;
let tool_call = events.recv().await.unwrap();
assert_eq!(tool_call.event().kind(), Kind::ToolCall);
let waiter_run = run.clone();
let waiter = tokio::spawn(async move { waiter_run.wait().await });
tokio::task::yield_now().await;
assert!(!waiter.is_finished());
tool_call.accept().unwrap();
let tool_result = events.recv().await.unwrap();
assert_eq!(tool_result.event().kind(), Kind::ToolResult);
tool_result.accept().unwrap();
assert_eq!(waiter.await.unwrap().outcome(), Outcome::Completed);
assert_eq!(model.calls.load(Ordering::SeqCst), 2);
}
#[tokio::test]
async fn dropping_an_observed_stream_stops_a_pending_strong_delivery() {
let model = Arc::new(TestModel::new(Mode::Tool));
let agent = Agent::new(Config::new(
Arc::clone(&model),
echo_tools(),
hook::RegistryBuilder::new(),
))
.unwrap();
let (run, mut events) = agent
.start_observed(
Session::new("observed-drop".into(), None),
vec![Part::text("input".into())],
Token::new(),
)
.await
.unwrap();
expect_kinds(
&mut events,
&[Kind::InputAdmitted, Kind::TurnStart, Kind::StepStart],
)
.await;
let tool_call = events.recv().await.unwrap();
assert_eq!(tool_call.event().kind(), Kind::ToolCall);
drop(events);
assert_eq!(run.wait().await.outcome(), Outcome::Stopped);
drop(tool_call);
assert_eq!(model.calls.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn close_cannot_discard_an_atomically_returned_stream() {
let (agent, model) = agent(Mode::Pending);
let (run, mut events) = start_observed(
&agent,
Session::new("observed-close".into(), None),
Token::new(),
)
.await;
model.wait_until_entered().await;
assert_eq!(run.close().await.outcome(), Outcome::Stopped);
expect_kinds(
&mut events,
&[Kind::InputAdmitted, Kind::TurnStart, Kind::StepStart],
)
.await;
assert!(events.recv().await.is_none());
agent.close().await.unwrap();
}
#[tokio::test]
async fn parent_cancellation_wins_while_an_observed_delivery_is_pending() {
let model = Arc::new(TestModel::new(Mode::Tool));
let agent = Agent::new(Config::new(
Arc::clone(&model),
echo_tools(),
hook::RegistryBuilder::new(),
))
.unwrap();
let parent = Token::new();
let (run, mut events) = agent
.start_observed(
Session::new("observed-cancel".into(), None),
vec![Part::text("input".into())],
parent.clone(),
)
.await
.unwrap();
expect_kinds(
&mut events,
&[Kind::InputAdmitted, Kind::TurnStart, Kind::StepStart],
)
.await;
let tool_call = events.recv().await.unwrap();
assert_eq!(tool_call.event().kind(), Kind::ToolCall);
parent.cancel();
assert_eq!(run.wait().await.outcome(), Outcome::Cancelled);
drop(tool_call);
drop(events);
assert_eq!(model.calls.load(Ordering::SeqCst), 1);
}
struct PanickingObserver;
impl ObserverHooks for PanickingObserver {
fn run_admitted(&self, _scope: &ReadScope<'_>) {
panic!("isolated admission observer");
}
fn lifecycle(&self, _observation: &LifecycleObservation, _scope: &ObserverScope<'_>) {}
fn model_event(&self, _scope: &ObserverScope<'_>, _event: &model::Event) {}
fn audit_event(&self, _scope: &ReadScope<'_>, _event: &audit::Event) {}
}
#[tokio::test]
async fn admitted_observer_fact_is_forwarded_before_driver_completion() {
let mut hooks = hook::RegistryBuilder::new();
hooks
.register(hook::Registration::new("observer".into()).observer(Arc::new(PanickingObserver)));
let (agent, _model) = agent_with_hooks(Mode::Complete, hooks);
let run = start(
&agent,
Session::new("admission-fact".into(), None),
Token::new(),
)
.await;
let mut events = run.take_events().unwrap();
let fact = events.recv().await.unwrap();
assert_eq!(fact.event().kind(), Kind::Audit);
assert_eq!(run.wait().await.outcome(), Outcome::Completed);
expect_kinds(&mut events, &EMPTY_GENERATION).await;
assert!(events.recv().await.is_none());
}
#[tokio::test]
async fn admitted_error_fact_is_forwarded_as_an_ordinary_event() {
let token = Token::new();
let (producer, mut source) = channel(&token);
let mut events = source.take_stream().unwrap();
let emitter = FactBuffer::default();
emitter.emit_error(Arc::new(ModelError));
emitter.emit_audit(audit::Event::new("admission.fact".to_owned(), None));
super::run::publish_buffered_facts(&producer, &emitter, None)
.await
.unwrap();
let fact = events.recv().await.unwrap();
assert_eq!(fact.event().kind(), Kind::Error);
assert_eq!(
fact.event().as_error().unwrap().to_string(),
"test model failed"
);
let audit = events.recv().await.unwrap();
assert_eq!(audit.event().kind(), Kind::Audit);
assert_eq!(audit.event().as_audit().unwrap().kind(), "admission.fact");
}
#[test]
fn closed_run_errors_have_stable_sources() {
assert_eq!(
TaskAbortedError.to_string(),
"Run task was aborted after it started"
);
assert_eq!(
TaskNotStartedError.to_string(),
"Run task was dropped before it started"
);
assert_eq!(TaskPanickedError.to_string(), "Run task panicked");
let aborted: &dyn Error = &TaskAbortedError;
let consumed: &dyn Error = &EventsTakenError;
let not_started: &dyn Error = &TaskNotStartedError;
let panicked: &dyn Error = &TaskPanickedError;
assert!(aborted.source().is_none());
assert!(consumed.source().is_none());
assert!(not_started.source().is_none());
assert!(panicked.source().is_none());
}
type EndObservations = Arc<Mutex<Vec<(&'static str, Outcome, bool)>>>;
struct EndCapture {
name: &'static str,
session: Session,
observations: EndObservations,
panic_at_end: bool,
}
impl ObserverHooks for EndCapture {
fn run_admitted(&self, _scope: &ReadScope<'_>) {}
fn lifecycle(&self, observation: &LifecycleObservation, _scope: &ObserverScope<'_>) {
if let LifecycleObservation::RunEnd { end, .. } = observation {
let _error_text = end.error().map(ToString::to_string);
self.observations.lock().unwrap().push((
self.name,
end.outcome(),
self.session.try_acquire().is_none(),
));
assert_eq!(observation.coord().session_id(), self.session.id());
if self.panic_at_end {
panic!("isolated RunEnd observer panic");
}
}
}
fn model_event(&self, _scope: &ObserverScope<'_>, _event: &model::Event) {}
fn audit_event(&self, _scope: &ReadScope<'_>, _event: &audit::Event) {}
}
#[tokio::test]
async fn run_end_observes_all_four_candidates_while_lease_is_held() {
for (name, mode, expected) in [
("completed", Mode::Complete, Outcome::Completed),
("stopped", Mode::Pending, Outcome::Stopped),
("canceled", Mode::Pending, Outcome::Cancelled),
("failed", Mode::Fail, Outcome::Failed),
] {
let session = Session::new(format!("run-end-{name}"), None);
let observations = Arc::new(Mutex::new(Vec::new()));
let mut hooks = hook::RegistryBuilder::new();
hooks.register(
hook::Registration::new(format!("capture-{name}")).observer(Arc::new(EndCapture {
name,
session: session.clone(),
observations: Arc::clone(&observations),
panic_at_end: false,
})),
);
let (agent, model) = agent_with_hooks(mode, hooks);
let parent = Token::new();
let run = start(&agent, session, parent.clone()).await;
if matches!(expected, Outcome::Stopped | Outcome::Cancelled) {
model.wait_until_entered().await;
}
match expected {
Outcome::Stopped => run.request_stop(),
Outcome::Cancelled => parent.cancel(),
Outcome::Completed | Outcome::Failed => {}
}
let terminal = run.wait().await;
assert_eq!(terminal.outcome(), expected, "{name}");
assert_eq!(
*observations.lock().unwrap(),
[(name, expected, true)],
"{name} RunEnd candidate/Lease"
);
agent.close().await.unwrap();
}
}
#[tokio::test]
async fn run_end_panic_isolated_peer_continues_and_fact_is_delivered() {
let session = Session::new("run-end-peer".into(), None);
let observations = Arc::new(Mutex::new(Vec::new()));
let mut hooks = hook::RegistryBuilder::new();
for (name, panic_at_end) in [("panic", true), ("peer", false)] {
hooks.register(
hook::Registration::new(name.into()).observer(Arc::new(EndCapture {
name,
session: session.clone(),
observations: Arc::clone(&observations),
panic_at_end,
})),
);
}
let (agent, _model) = agent_with_hooks(Mode::Complete, hooks);
let run = start(&agent, session, Token::new()).await;
let mut events = run.take_events().unwrap();
expect_kinds(&mut events, &EMPTY_GENERATION).await;
let panic_fact = events.recv().await.expect("RunEnd panic audit");
assert_eq!(panic_fact.event().kind(), Kind::Audit);
assert_eq!(
panic_fact.event().as_audit().unwrap().kind(),
"hook.observer_panic"
);
let terminal = run.wait().await;
assert_eq!(terminal.outcome(), Outcome::Completed);
assert_eq!(
*observations.lock().unwrap(),
[
("panic", Outcome::Completed, true),
("peer", Outcome::Completed, true)
]
);
assert!(events.recv().await.is_none());
}
#[tokio::test]
async fn invalid_c1_fact_fails_before_driver_and_still_cleans_the_run() {
let (agent, model) = agent(Mode::Complete);
let session = Session::new("invalid-admission".into(), None);
let admitted = agent
.admit_start(
session.clone(),
vec![Part::text("input".into())],
Token::new(),
)
.unwrap();
admitted
.fact_buffer
.emit_audit(audit::Event::new(String::new(), None));
let run = super::run::Run::spawn(admitted);
let terminal = run.wait().await;
assert_eq!(terminal.outcome(), Outcome::Failed);
assert!(!model.entered.load(Ordering::Acquire));
drop(
session
.try_acquire()
.expect("fatal admission fact releases Lease"),
);
agent.close().await.unwrap();
}
struct InvalidRunEnd {
seen: Arc<Mutex<Vec<Outcome>>>,
}
impl ObserverHooks for InvalidRunEnd {
fn run_admitted(&self, _scope: &ReadScope<'_>) {}
fn lifecycle(&self, observation: &LifecycleObservation, scope: &ObserverScope<'_>) {
if let LifecycleObservation::RunEnd { end, .. } = observation {
self.seen.lock().unwrap().push(end.outcome());
scope.emit_audit(audit::Event::new(String::new(), None));
}
}
fn model_event(&self, _scope: &ObserverScope<'_>, _event: &model::Event) {}
fn audit_event(&self, _scope: &ReadScope<'_>, _event: &audit::Event) {}
}
#[tokio::test]
async fn run_end_invalid_fact_upgrades_final_after_same_candidate_reaches_peers() {
let session = Session::new("run-end-fatal".into(), None);
let seen = Arc::new(Mutex::new(Vec::new()));
let peer_seen = Arc::new(Mutex::new(Vec::new()));
let mut hooks = hook::RegistryBuilder::new();
hooks.register(
hook::Registration::new("invalid".into()).observer(Arc::new(InvalidRunEnd {
seen: Arc::clone(&seen),
})),
);
hooks.register(
hook::Registration::new("peer".into()).observer(Arc::new(EndCapture {
name: "peer",
session: session.clone(),
observations: Arc::clone(&peer_seen),
panic_at_end: false,
})),
);
let (agent, model) = agent_with_hooks(Mode::Complete, hooks);
let run = start(&agent, session, Token::new()).await;
let mut events = run.take_events().unwrap();
let terminal = run.wait().await;
assert_eq!(terminal.outcome(), Outcome::Failed);
assert_eq!(*seen.lock().unwrap(), [Outcome::Completed]);
assert_eq!(
*peer_seen.lock().unwrap(),
[("peer", Outcome::Completed, true)]
);
assert!(model.entered.load(Ordering::Acquire));
expect_kinds(&mut events, &EMPTY_GENERATION).await;
assert!(events.recv().await.is_none());
}
struct FifoObserver;
impl ObserverHooks for FifoObserver {
fn run_admitted(&self, _scope: &ReadScope<'_>) {
panic!("admission fact first");
}
fn lifecycle(&self, observation: &LifecycleObservation, scope: &ObserverScope<'_>) {
if matches!(observation, LifecycleObservation::RunEnd { .. }) {
scope.emit_audit(audit::Event::new("run.end".into(), None));
panic!("RunEnd fact last");
}
}
fn model_event(&self, _scope: &ObserverScope<'_>, _event: &model::Event) {}
fn audit_event(&self, _scope: &ReadScope<'_>, _event: &audit::Event) {}
}
#[tokio::test]
async fn c1_and_run_end_facts_share_one_fifo_without_early_close() {
let mut hooks = hook::RegistryBuilder::new();
hooks.register(hook::Registration::new("fifo".into()).observer(Arc::new(FifoObserver)));
let (agent, _model) = agent_with_hooks(Mode::Complete, hooks);
let run = start(&agent, Session::new("fact-fifo".into(), None), Token::new()).await;
let mut events = run.take_events().unwrap();
let mut kinds = Vec::new();
while let Some(delivery) = events.recv().await {
kinds.push(match delivery.event().kind() {
Kind::Audit => delivery.event().as_audit().unwrap().kind().to_owned(),
kind => format!("{kind:?}"),
});
}
assert_eq!(
kinds,
[
"hook.observer_panic",
"InputAdmitted",
"TurnStart",
"StepStart",
"GenerationEnd",
"StepEnd",
"TurnEnd",
"run.end",
"hook.observer_panic",
]
);
assert_eq!(run.wait().await.outcome(), Outcome::Completed);
}
struct BlockingEnd {
session: Session,
entered: Arc<AtomicBool>,
release: Arc<(Mutex<bool>, Condvar)>,
}
impl ObserverHooks for BlockingEnd {
fn run_admitted(&self, _scope: &ReadScope<'_>) {}
fn lifecycle(&self, observation: &LifecycleObservation, _scope: &ObserverScope<'_>) {
if matches!(observation, LifecycleObservation::RunEnd { .. }) {
assert!(self.session.try_acquire().is_none());
self.entered.store(true, Ordering::Release);
let (lock, ready) = &*self.release;
let mut released = lock.lock().unwrap();
while !*released {
released = ready.wait(released).unwrap();
}
}
}
fn model_event(&self, _scope: &ObserverScope<'_>, _event: &model::Event) {}
fn audit_event(&self, _scope: &ReadScope<'_>, _event: &audit::Event) {}
}
#[tokio::test]
async fn wait_cannot_see_terminal_until_run_end_release_detach_and_event_close() {
let session = Session::new("ordered-cleanup".into(), None);
let entered = Arc::new(AtomicBool::new(false));
let release = Arc::new((Mutex::new(false), Condvar::new()));
let mut hooks = hook::RegistryBuilder::new();
hooks.register(
hook::Registration::new("blocking-end".into()).observer(Arc::new(BlockingEnd {
session: session.clone(),
entered: Arc::clone(&entered),
release: Arc::clone(&release),
})),
);
let (agent, model) = agent_with_hooks(Mode::Blocked, hooks);
let run = start(&agent, session.clone(), Token::new()).await;
let mut events = run.take_events().unwrap();
model.wait_until_entered().await;
let waiter_run = run.clone();
let waiter = tokio::spawn(async move { waiter_run.wait().await });
let verifier_run = run.clone();
let verifier_session = session.clone();
let verifier_entered = Arc::clone(&entered);
let verifier_release = Arc::clone(&release);
let verifier = std::thread::spawn(move || {
while !verifier_entered.load(Ordering::Acquire) {
std::thread::yield_now();
}
assert!(!verifier_run.terminal_ready());
assert!(verifier_session.try_acquire().is_none());
let (lock, ready) = &*verifier_release;
*lock.lock().unwrap() = true;
ready.notify_all();
});
model.release.notify_one();
let terminal = waiter.await.unwrap();
verifier.join().unwrap();
assert_eq!(terminal.outcome(), Outcome::Completed);
expect_kinds(&mut events, &EMPTY_GENERATION).await;
assert!(events.recv().await.is_none());
drop(session.try_acquire().expect("notification follows release"));
agent.close().await.unwrap();
assert!(Arc::ptr_eq(&terminal, &run.wait().await));
}
struct HeldRunEnd {
entered: Arc<AtomicBool>,
release: Arc<(Mutex<bool>, Condvar)>,
}
impl ObserverHooks for HeldRunEnd {
fn run_admitted(&self, _scope: &ReadScope<'_>) {}
fn lifecycle(&self, observation: &LifecycleObservation, scope: &ObserverScope<'_>) {
if matches!(observation, LifecycleObservation::RunEnd { .. }) {
for index in 0..3 {
scope.emit_audit(audit::Event::new(format!("backpressure.{index}"), None));
}
self.entered.store(true, Ordering::Release);
let (lock, ready) = &*self.release;
let mut released = lock.lock().unwrap();
while !*released {
released = ready.wait(released).unwrap();
}
}
}
fn model_event(&self, _scope: &ObserverScope<'_>, _event: &model::Event) {}
fn audit_event(&self, _scope: &ReadScope<'_>, _event: &audit::Event) {}
}
#[tokio::test]
async fn no_waiter_observes_a_terminal_while_run_end_is_still_running() {
let entered = Arc::new(AtomicBool::new(false));
let release = Arc::new((Mutex::new(false), Condvar::new()));
let mut hooks = hook::RegistryBuilder::new();
hooks.register(
hook::Registration::new("held-run-end".into()).observer(Arc::new(HeldRunEnd {
entered: Arc::clone(&entered),
release: Arc::clone(&release),
})),
);
let (agent, _model) = agent_with_hooks(Mode::Complete, hooks);
let run = start(
&agent,
Session::new("held-run-end".into(), None),
Token::new(),
)
.await;
let events = run.take_events().unwrap();
let waiter_run = run.clone();
let waiter = tokio::spawn(async move { waiter_run.wait().await });
let verifier_run = run.clone();
let verifier_entered = Arc::clone(&entered);
let verifier_release = Arc::clone(&release);
let verifier = std::thread::spawn(move || {
while !verifier_entered.load(Ordering::Acquire) {
std::thread::yield_now();
}
assert!(
!verifier_run.terminal_ready(),
"no waiter may observe a terminal while RunEnd is still running"
);
drop(events);
let (lock, ready) = &*verifier_release;
*lock.lock().unwrap() = true;
ready.notify_all();
});
let from_wait = waiter.await.unwrap();
verifier.join().unwrap();
let from_close = run.close().await;
assert_eq!(from_wait.outcome(), Outcome::Stopped);
assert!(Arc::ptr_eq(&from_wait, &from_close));
}
#[test]
fn runtime_shutdown_before_first_run_task_poll_publishes_terminal_and_releases_owners() {
let runtime = tokio::runtime::Builder::new_current_thread()
.build()
.unwrap();
let (agent, model) = agent(Mode::Pending);
let session = Session::new("pre-poll-shutdown".into(), None);
let run = {
let _runtime = runtime.enter();
let mut start = std::pin::pin!(agent.start(
session.clone(),
vec![Part::text("input".into())],
Token::new(),
));
let mut context = Context::from_waker(std::task::Waker::noop());
let Poll::Ready(result) = start.as_mut().poll(&mut context) else {
panic!("Agent::start must complete in its first poll");
};
result.unwrap()
};
assert!(!model.entered.load(Ordering::Acquire));
assert!(!run.terminal_ready());
let mut events = run.take_events().unwrap();
drop(runtime);
assert!(
run.terminal_ready(),
"runtime shutdown must publish a terminal"
);
assert!(!model.entered.load(Ordering::Acquire));
let snapshot = session.transcript().snapshot().unwrap();
assert_eq!(snapshot.items().len(), 1);
assert!(
matches!(&snapshot.items()[0], Item::User(user) if user.parts()[0].fields().text() == "input"),
"Agent::start committed input before spawning the never-polled task"
);
drop(
session
.try_acquire()
.expect("runtime shutdown releases the Session lease"),
);
let mut receive = std::pin::pin!(events.recv());
let mut context = Context::from_waker(std::task::Waker::noop());
assert!(matches!(
receive.as_mut().poll(&mut context),
Poll::Ready(None)
));
let verifier = tokio::runtime::Builder::new_current_thread()
.build()
.unwrap();
let terminal = verifier.block_on(run.wait());
assert_eq!(terminal.outcome(), Outcome::Failed);
let failure = terminal.failure().unwrap();
assert!(failure.downcast_ref::<TaskNotStartedError>().is_some());
assert!(failure.downcast_ref::<TaskAbortedError>().is_none());
assert!(failure.downcast_ref::<TaskPanickedError>().is_none());
assert!(Arc::ptr_eq(&terminal, &verifier.block_on(run.wait())));
let join_error = verifier.block_on(run.join_tracked_task()).unwrap_err();
assert!(join_error.is_cancelled());
assert!(!join_error.is_panic());
verifier.block_on(agent.close()).unwrap();
}
#[test]
fn observed_stream_survives_runtime_shutdown_before_first_task_poll() {
let runtime = tokio::runtime::Builder::new_current_thread()
.build()
.unwrap();
let (agent, model) = agent(Mode::Pending);
let session = Session::new("observed-pre-poll-shutdown".into(), None);
let (run, mut events) = {
let _runtime = runtime.enter();
let mut start = std::pin::pin!(agent.start_observed(
session.clone(),
vec![Part::text("input".into())],
Token::new(),
));
let mut context = Context::from_waker(std::task::Waker::noop());
let Poll::Ready(result) = start.as_mut().poll(&mut context) else {
panic!("Agent::start_observed must complete in its first poll");
};
result.unwrap()
};
assert!(!model.entered.load(Ordering::Acquire));
assert!(!run.terminal_ready());
assert_eq!(run.take_events().err(), Some(EventsTakenError));
drop(runtime);
assert!(run.terminal_ready());
drop(
session
.try_acquire()
.expect("runtime shutdown releases the Session lease"),
);
let mut receive = std::pin::pin!(events.recv());
let mut context = Context::from_waker(std::task::Waker::noop());
assert!(matches!(
receive.as_mut().poll(&mut context),
Poll::Ready(None)
));
let verifier = tokio::runtime::Builder::new_current_thread()
.build()
.unwrap();
let terminal = verifier.block_on(run.wait());
assert_eq!(terminal.outcome(), Outcome::Failed);
assert!(
terminal
.failure()
.unwrap()
.downcast_ref::<TaskNotStartedError>()
.is_some()
);
verifier.block_on(agent.close()).unwrap();
}
#[test]
fn runtime_shutdown_after_run_task_poll_publishes_aborted_terminal_and_releases_owners() {
let runtime = tokio::runtime::Builder::new_current_thread()
.build()
.unwrap();
let (agent, model) = agent(Mode::Pending);
let session = Session::new("post-poll-shutdown".into(), None);
let run = runtime.block_on(start(&agent, session.clone(), Token::new()));
runtime.block_on(model.wait_until_entered());
assert!(model.entered.load(Ordering::Acquire));
assert!(!run.terminal_ready());
let mut events = run.take_events().unwrap();
drop(runtime);
assert!(
run.terminal_ready(),
"runtime shutdown must publish a terminal"
);
drop(
session
.try_acquire()
.expect("runtime shutdown releases the Session lease"),
);
let verifier = tokio::runtime::Builder::new_current_thread()
.build()
.unwrap();
verifier.block_on(expect_kinds(
&mut events,
&[Kind::InputAdmitted, Kind::TurnStart, Kind::StepStart],
));
assert!(verifier.block_on(events.recv()).is_none());
let terminal = verifier.block_on(run.wait());
assert_eq!(terminal.outcome(), Outcome::Failed);
let failure = terminal.failure().unwrap();
assert!(failure.downcast_ref::<TaskAbortedError>().is_some());
assert!(failure.downcast_ref::<TaskNotStartedError>().is_none());
assert!(failure.downcast_ref::<TaskPanickedError>().is_none());
assert!(Arc::ptr_eq(&terminal, &verifier.block_on(run.wait())));
let join_error = verifier.block_on(run.join_tracked_task()).unwrap_err();
assert!(join_error.is_cancelled());
assert!(!join_error.is_panic());
verifier.block_on(agent.close()).unwrap();
}
#[test]
fn host_thread_unwind_during_runtime_shutdown_is_a_task_abort_not_a_task_panic() {
let mut state = None;
let host_unwind = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
let runtime = tokio::runtime::Builder::new_current_thread()
.build()
.unwrap();
let (agent, model) = agent(Mode::Pending);
let session = Session::new("host-unwind-shutdown".into(), None);
let run = runtime.block_on(start(&agent, session.clone(), Token::new()));
runtime.block_on(model.wait_until_entered());
assert!(model.entered.load(Ordering::Acquire));
assert!(!run.terminal_ready());
let events = run.take_events().unwrap();
state = Some((agent, session, run, events));
panic!("unrelated host thread panic");
}));
assert!(host_unwind.is_err());
let (agent, session, run, mut events) = state.expect("state survives the host unwind");
assert!(
run.terminal_ready(),
"runtime shutdown must synchronously publish a terminal"
);
drop(
session
.try_acquire()
.expect("host-unwind shutdown releases the Session lease"),
);
let verifier = tokio::runtime::Builder::new_current_thread()
.build()
.unwrap();
verifier.block_on(expect_kinds(
&mut events,
&[Kind::InputAdmitted, Kind::TurnStart, Kind::StepStart],
));
assert!(verifier.block_on(events.recv()).is_none());
let terminal = verifier.block_on(run.wait());
assert_eq!(terminal.outcome(), Outcome::Failed);
let failure = terminal.failure().unwrap();
assert!(failure.downcast_ref::<TaskAbortedError>().is_some());
assert!(failure.downcast_ref::<TaskNotStartedError>().is_none());
assert!(failure.downcast_ref::<TaskPanickedError>().is_none());
assert!(Arc::ptr_eq(&terminal, &verifier.block_on(run.wait())));
let join_error = verifier.block_on(run.join_tracked_task()).unwrap_err();
assert!(join_error.is_cancelled());
assert!(!join_error.is_panic());
verifier.block_on(agent.close()).unwrap();
}
#[tokio::test]
async fn core_panic_raii_publishes_failed_cleans_and_preserves_join_panic() {
let lifecycle_calls = Arc::new(Mutex::new(Vec::new()));
let session = Session::new("core-panic".into(), None);
let mut hooks = hook::RegistryBuilder::new();
hooks.register(
hook::Registration::new("must-not-catch-core".into()).observer(Arc::new(EndCapture {
name: "must-not-catch-core",
session: session.clone(),
observations: Arc::clone(&lifecycle_calls),
panic_at_end: false,
})),
);
let (agent, model) = agent_with_hooks(Mode::Panic, hooks);
let run = start(&agent, session.clone(), Token::new()).await;
let mut events = run.take_events().unwrap();
model.wait_until_entered().await;
model.release.notify_one();
let terminal = run.wait().await;
assert_eq!(terminal.outcome(), Outcome::Failed);
let failure = terminal.failure().unwrap();
assert!(failure.downcast_ref::<TaskPanickedError>().is_some());
assert!(failure.downcast_ref::<TaskAbortedError>().is_none());
assert!(failure.downcast_ref::<TaskNotStartedError>().is_none());
assert!(Arc::ptr_eq(&terminal, &run.wait().await));
assert!(lifecycle_calls.lock().unwrap().is_empty());
expect_kinds(
&mut events,
&[Kind::InputAdmitted, Kind::TurnStart, Kind::StepStart],
)
.await;
assert!(events.recv().await.is_none());
drop(session.try_acquire().expect("panic guard released Lease"));
agent.close().await.unwrap();
let join_error = run.join_tracked_task().await.unwrap_err();
assert!(
join_error.is_panic(),
"core panic remains visible at tracked join"
);
}
#[tokio::test]
async fn run_debug_locates_the_run_without_a_model_debug_bound() {
let (agent, _model) = agent(Mode::Complete);
let run = start(&agent, Session::new("printed".into(), None), Token::new()).await;
let rendering = format!("{run:?}");
assert!(
rendering.starts_with(r#"Run { session: "printed", revision: "#),
"{rendering}"
);
assert!(rendering.ends_with("handles: 1 }"), "{rendering}");
let cloned = run.clone();
assert!(format!("{run:?}").contains("handles: 2"));
drop(cloned);
let _terminal = run.wait().await;
agent.close().await.unwrap();
}
#[tokio::test]
async fn terminal_debug_renders_a_failure_through_display_not_debug() {
let (failing_agent, _failing_model) = agent(Mode::Fail);
let (completed_agent, _completed_model) = agent(Mode::Complete);
let terminal = start(
&failing_agent,
Session::new("failed".into(), None),
Token::new(),
)
.await
.wait()
.await;
assert_states(
terminal.as_ref(),
&[
"outcome: Failed",
r#"failure: Some("the model provider failed: test model failed")"#,
"failure_kind: Some(Model)",
"stop_code: None",
],
);
let completed = start(
&completed_agent,
Session::new("done".into(), None),
Token::new(),
)
.await
.wait()
.await;
assert_states(
completed.as_ref(),
&["outcome: Completed", "failure: None", "failure_kind: None"],
);
failing_agent.close().await.unwrap();
completed_agent.close().await.unwrap();
}