#![cfg(test)]
use std::collections::VecDeque;
use std::error::Error;
use std::fmt;
use std::future::{Future, ready};
use std::pin::Pin;
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
use std::sync::{Arc, Condvar, Mutex};
use std::task::{Context, Poll};
use std::thread;
use futures_core::Stream;
use super::agent::{
Agent, CloseError, ClosePanickedError, Config, ConfigError, StartError, allocate_revision_id,
};
use super::hook::{self, AgentTeardown, AttributedError, HookError, HookResult};
use super::model::{self, Client, Info, Request, SamplingParams};
use super::session::LeaseError;
use super::tool;
use crate::opaque::{Envelope, Payload};
use crate::redact_tests::{SECRET, assert_redacted, assert_states};
use crate::transcript::store;
use crate::value::content::Part;
use crate::value::lifecycle::CoordError;
#[derive(Debug)]
pub(crate) struct ModelError;
impl fmt::Display for ModelError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("model error")
}
}
impl Error for ModelError {}
pub(crate) struct EmptyEvents(VecDeque<Result<model::Event, ModelError>>);
impl Stream for EmptyEvents {
type Item = Result<model::Event, ModelError>;
fn poll_next(mut self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<Option<Self::Item>> {
Poll::Ready(self.0.pop_front())
}
}
pub(crate) struct Model;
impl Client for Model {
type Error = ModelError;
type Stream = EmptyEvents;
fn info(&self) -> Info {
Info::new("agent-test".to_owned(), 1024).with_tools(true)
}
fn generate(
&self,
_: Request,
) -> impl Future<Output = Result<Self::Stream, Self::Error>> + Send {
ready(Ok(EmptyEvents(VecDeque::new())))
}
}
#[derive(Debug)]
struct TeardownError(&'static str);
impl fmt::Display for TeardownError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str(self.0)
}
}
impl Error for TeardownError {}
struct PanicOnDropPayload;
impl Drop for PanicOnDropPayload {
fn drop(&mut self) {
panic!("panic payload dropped")
}
}
enum TeardownResult {
Ok,
Error(&'static str),
PanicOnDrop,
}
struct RecordingTeardown {
name: &'static str,
result: TeardownResult,
calls: Arc<AtomicUsize>,
order: Arc<Mutex<Vec<&'static str>>>,
}
impl AgentTeardown for RecordingTeardown {
fn close(&self) -> HookResult<()> {
self.calls.fetch_add(1, Ordering::SeqCst);
self.order.lock().unwrap().push(self.name);
match self.result {
TeardownResult::Ok => Ok(()),
TeardownResult::Error(message) => Err(HookError::new(TeardownError(message))),
TeardownResult::PanicOnDrop => std::panic::panic_any(PanicOnDropPayload),
}
}
}
#[derive(Default)]
struct BlockState {
entered: bool,
released: bool,
}
#[derive(Default)]
struct Blocker {
state: Mutex<BlockState>,
changed: Condvar,
}
impl Blocker {
fn wait_until_entered(&self) {
let mut state = self.state.lock().unwrap();
while !state.entered {
state = self.changed.wait(state).unwrap();
}
}
fn enter_and_wait(&self) {
let mut state = self.state.lock().unwrap();
state.entered = true;
self.changed.notify_all();
while !state.released {
state = self.changed.wait(state).unwrap();
}
}
fn release(&self) {
let mut state = self.state.lock().unwrap();
state.released = true;
self.changed.notify_all();
}
}
struct BlockingTeardown {
blocker: Arc<Blocker>,
calls: Arc<AtomicUsize>,
}
impl AgentTeardown for BlockingTeardown {
fn close(&self) -> HookResult<()> {
self.calls.fetch_add(1, Ordering::SeqCst);
self.blocker.enter_and_wait();
Err(HookError::new(TeardownError("blocked error")))
}
}
fn empty_tools() -> tool::Registry {
tool::RegistryBuilder::new().build().unwrap()
}
fn empty_config() -> Config<Model> {
Config::new(Arc::new(Model), empty_tools(), hook::RegistryBuilder::new())
}
fn registration(
name: &'static str,
result: TeardownResult,
calls: &Arc<AtomicUsize>,
order: &Arc<Mutex<Vec<&'static str>>>,
) -> hook::Registration {
hook::Registration::new(name.to_owned()).teardown(Arc::new(RecordingTeardown {
name,
result,
calls: Arc::clone(calls),
order: Arc::clone(order),
}))
}
fn agent_with_registrations(registrations: Vec<hook::Registration>) -> Agent<Model> {
let mut hooks = hook::RegistryBuilder::new();
for registration in registrations {
hooks.register(registration);
}
Agent::new(Config::new(Arc::new(Model), empty_tools(), hooks)).unwrap()
}
fn close_on_thread(agent: Agent<Model>) -> Result<(), CloseError> {
tokio::runtime::Builder::new_current_thread()
.build()
.unwrap()
.block_on(agent.close())
}
#[test]
fn config_consumes_builder_validates_defaults_and_freezes_close_free_metadata() {
let model = Arc::new(Model);
let mut hooks = hook::RegistryBuilder::new();
hooks.register(hook::Registration::new("metadata".to_owned()).requires_session_state(true));
let mut config = Config::new(Arc::clone(&model), empty_tools(), hooks);
config.push_system_instruction(Part::text("be precise".to_owned()));
*config.sampling_mut() = SamplingParams {
temperature: Some(0.25),
reasoning_effort: Some("high".to_owned()),
max_output_tokens: Some(99),
parallel_tool_calls: true,
output_schema: Some(Payload::copy_from_slice(b"{}")),
stop_sequences: vec!["stop".to_owned()],
tool_choice: None,
};
config.push_model_extension(
Envelope::try_new(
"provider".to_owned(),
"defaults".to_owned(),
Payload::copy_from_slice(br#"{"enabled":true}"#),
)
.unwrap(),
);
let agent = Agent::new(config).unwrap();
let revision = agent.revision();
let cloned = revision.clone();
assert_ne!(revision.id().get(), 0);
assert_eq!(revision.id(), cloned.id());
assert!(Arc::ptr_eq(revision.model(), &model));
let _ = revision.tools();
assert_eq!(revision.hook_names(), ["metadata"]);
assert!(revision.hooks_require_session_state());
assert_eq!(
revision.system_instruction()[0].fields().text(),
"be precise"
);
assert_eq!(revision.sampling().temperature, Some(0.25));
assert_eq!(revision.model_extensions()[0].namespace(), "provider");
let next = Agent::new(empty_config()).unwrap().revision().id();
assert!(next > revision.id());
}
#[test]
fn config_assembly_reads_and_clears_without_exposing_its_containers() {
let extension = |kind: &str| {
Envelope::try_new(
"provider".to_owned(),
kind.to_owned(),
Payload::copy_from_slice(b"{}"),
)
.unwrap()
};
let mut config = empty_config();
assert!(config.system_instruction().is_empty());
assert!(config.model_extensions().is_empty());
config
.push_system_instruction(Part::text("first".to_owned()))
.extend_system_instruction([
Part::text("second".to_owned()),
Part::text("third".to_owned()),
])
.push_model_extension(extension("one"))
.extend_model_extensions([extension("two"), extension("three")]);
assert_eq!(
config
.system_instruction()
.iter()
.map(|part| part.fields().text())
.collect::<Vec<_>>(),
["first", "second", "third"]
);
assert_eq!(
config
.model_extensions()
.iter()
.map(Envelope::kind)
.collect::<Vec<_>>(),
["one", "two", "three"]
);
config.clear_system_instruction().clear_model_extensions();
assert!(config.system_instruction().is_empty());
assert!(config.model_extensions().is_empty());
let revision = Agent::new(config).unwrap().revision().clone();
assert!(revision.system_instruction().is_empty());
assert!(revision.model_extensions().is_empty());
}
#[test]
fn config_reports_hook_error_before_invalid_defaults_and_id_exhaustion_is_explicit() {
let mut invalid_hooks = hook::RegistryBuilder::new();
invalid_hooks.register(hook::Registration::new(String::new()));
let mut config = Config::new(Arc::new(Model), empty_tools(), invalid_hooks);
config.push_system_instruction(Part::text(String::new()));
let error = Agent::new(config).err().unwrap();
assert!(matches!(error, ConfigError::Hooks(_)));
assert_eq!(error.to_string(), "hook name must be non-empty");
assert!(error.source().is_some());
let mut config = empty_config();
config.push_system_instruction(Part::text(String::new()));
let error = Agent::new(config).err().unwrap();
assert!(matches!(
error,
ConfigError::InvalidDefaults(super::model::RequestError::EmptySystemInstructionText {
index: 0
})
));
assert_eq!(
error.to_string(),
"system instruction text at index 0 is empty"
);
assert!(error.source().is_some());
let counter = AtomicU64::new(7);
assert_eq!(allocate_revision_id(&counter).unwrap().get(), 7);
let exhausted = AtomicU64::new(u64::MAX);
let error = Agent::new_with_counter(empty_config(), &exhausted)
.err()
.unwrap();
assert!(matches!(error, ConfigError::RevisionIdExhausted));
assert_eq!(error.to_string(), "Agent revision identity exhausted");
assert!(error.source().is_none());
let zero = AtomicU64::new(0);
assert!(matches!(
allocate_revision_id(&zero),
Err(ConfigError::RevisionIdExhausted)
));
}
#[tokio::test]
async fn zero_hooks_and_registration_without_teardown_close_once_and_cache_success() {
let agent = Agent::new(empty_config()).unwrap();
assert!(agent.close().await.is_ok());
assert!(agent.close().await.is_ok());
let agent = agent_with_registrations(vec![hook::Registration::new("observer-only".to_owned())]);
assert!(agent.close().await.is_ok());
}
#[tokio::test]
async fn panic_on_drop_is_aggregated_without_stalling_close_or_short_circuiting_peers() {
let calls = Arc::new(AtomicUsize::new(0));
let order = Arc::new(Mutex::new(Vec::new()));
let agent = agent_with_registrations(vec![
registration(
"first",
TeardownResult::Error("first error"),
&calls,
&order,
),
registration("panic", TeardownResult::PanicOnDrop, &calls, &order),
registration("peer", TeardownResult::Ok, &calls, &order),
registration("last", TeardownResult::Error("last error"), &calls, &order),
]);
let error = agent.close().await.unwrap_err();
let CloseError::Hooks(errors) = &error else {
panic!("expected hook failures");
};
assert_eq!(*order.lock().unwrap(), ["first", "panic", "peer", "last"]);
assert_eq!(calls.load(Ordering::SeqCst), 4);
let entries = errors.entries();
assert_eq!(
entries
.iter()
.map(|failure| failure.name())
.collect::<Vec<_>>(),
["first", "panic", "last"]
);
assert_eq!(entries[0].error().to_string(), "first error");
assert!(
entries[1]
.error()
.source_ref()
.downcast_ref::<ClosePanickedError>()
.is_some()
);
assert_eq!(entries[2].error().to_string(), "last error");
assert_eq!(errors.to_string(), "3 Agent OnClose callback(s) failed");
assert_eq!(
ClosePanickedError.to_string(),
"Agent OnClose callback panicked"
);
let teardown = error
.source()
.expect("CloseError::Hooks sources TeardownError")
.downcast_ref::<super::agent::TeardownError>()
.expect("the sourced value is TeardownError");
assert!(std::ptr::eq(teardown, errors));
let first = teardown
.source()
.expect("TeardownError sources the first AttributedError")
.downcast_ref::<AttributedError>()
.expect("the sourced value is AttributedError");
assert_eq!(first.name(), "first");
assert_eq!(first.error().to_string(), "first error");
assert!(std::ptr::eq(first, &entries[0]));
assert!(
first
.source()
.and_then(Error::source)
.and_then(|error| error.downcast_ref::<ClosePanickedError>())
.is_none()
);
let mut cached_close = Box::pin(agent.close());
let waker = std::task::Waker::noop();
let mut context = Context::from_waker(waker);
let cached = match cached_close.as_mut().poll(&mut context) {
Poll::Ready(result) => result.unwrap_err(),
Poll::Pending => panic!("cached close remained pending"),
};
let CloseError::Hooks(cached_errors) = &cached else {
panic!("cached close unexpectedly busy");
};
assert!(std::ptr::eq(
entries.as_ptr(),
cached_errors.entries().as_ptr()
));
assert_eq!(calls.load(Ordering::SeqCst), 4);
}
#[tokio::test]
async fn busy_reports_exact_count_runs_no_hook_and_remains_retryable() {
let calls = Arc::new(AtomicUsize::new(0));
let order = Arc::new(Mutex::new(Vec::new()));
let agent = agent_with_registrations(vec![registration(
"close",
TeardownResult::Ok,
&calls,
&order,
)]);
let first = agent.reserve_attachment().unwrap();
let second = agent.reserve_attachment().unwrap();
assert!(matches!(
agent.close().await,
Err(CloseError::Busy { active_runs: 2 })
));
assert_eq!(calls.load(Ordering::SeqCst), 0);
drop(first);
let error = agent.close().await.unwrap_err();
assert!(matches!(error, CloseError::Busy { active_runs: 1 }));
assert_eq!(error.to_string(), "Agent has 1 active Run(s)");
assert!(error.source().is_none());
drop(second);
assert!(agent.close().await.is_ok());
assert!(agent.close().await.is_ok());
assert_eq!(calls.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn close_first_never_touches_session_and_reservation_first_forces_busy() {
let blocker = Arc::new(Blocker::default());
let calls = Arc::new(AtomicUsize::new(0));
let mut hooks = hook::RegistryBuilder::new();
hooks.register(
hook::Registration::new("closing".to_owned()).teardown(Arc::new(BlockingTeardown {
blocker: Arc::clone(&blocker),
calls,
})),
);
let closing = Agent::new(Config::new(Arc::new(Model), empty_tools(), hooks)).unwrap();
let close_handle = {
let closing = closing.clone();
thread::spawn(move || close_on_thread(closing))
};
blocker.wait_until_entered();
let session_touches = AtomicUsize::new(0);
let reservation = closing.reserve_attachment();
if reservation.is_ok() {
session_touches.fetch_add(1, Ordering::SeqCst);
}
assert!(matches!(reservation, Err(StartError::AgentClosed)));
assert_eq!(session_touches.load(Ordering::SeqCst), 0);
blocker.release();
assert!(close_handle.join().unwrap().is_err());
let open = Agent::new(empty_config()).unwrap();
let reservation = open.reserve_attachment().unwrap();
session_touches.fetch_add(1, Ordering::SeqCst);
assert!(matches!(
open.close().await,
Err(CloseError::Busy { active_runs: 1 })
));
drop(reservation);
open.close().await.unwrap();
assert_eq!(session_touches.load(Ordering::SeqCst), 1);
assert!(matches!(
open.reserve_attachment(),
Err(StartError::AgentClosed)
));
}
#[tokio::test]
async fn concurrent_and_repeated_close_wait_for_and_share_one_cached_result() {
let blocker = Arc::new(Blocker::default());
let calls = Arc::new(AtomicUsize::new(0));
let mut hooks = hook::RegistryBuilder::new();
hooks.register(
hook::Registration::new("blocking".to_owned()).teardown(Arc::new(BlockingTeardown {
blocker: Arc::clone(&blocker),
calls: Arc::clone(&calls),
})),
);
let agent = Agent::new(Config::new(Arc::new(Model), empty_tools(), hooks)).unwrap();
let first_agent = agent.clone();
let first = thread::spawn(move || close_on_thread(first_agent));
blocker.wait_until_entered();
const WAITERS: usize = 2;
let probe = agent.install_close_wait_probe(WAITERS);
let waiters = (0..WAITERS)
.map(|_| {
let waiter = agent.clone();
thread::spawn(move || close_on_thread(waiter))
})
.collect::<Vec<_>>();
probe.wait_until_closing_observed();
probe.notify_waiters_one_by_one(&agent);
probe.resume_waiters();
probe.wait_until_closing_rechecked();
blocker.release();
let first = first.join().unwrap().unwrap_err();
let mut waiter_results = waiters
.into_iter()
.map(|waiter| waiter.join().unwrap().unwrap_err());
let second = waiter_results.next().unwrap();
let third = waiter_results.next().unwrap();
assert!(waiter_results.next().is_none());
let fourth = agent.close().await.unwrap_err();
let CloseError::Hooks(first_errors) = &first else {
panic!("close unexpectedly busy");
};
let CloseError::Hooks(second_errors) = &second else {
panic!("close unexpectedly busy");
};
let CloseError::Hooks(third_errors) = &third else {
panic!("close unexpectedly busy");
};
let CloseError::Hooks(fourth_errors) = &fourth else {
panic!("close unexpectedly busy");
};
assert!(std::ptr::eq(
first_errors.entries().as_ptr(),
second_errors.entries().as_ptr()
));
assert!(std::ptr::eq(
second_errors.entries().as_ptr(),
third_errors.entries().as_ptr()
));
assert!(std::ptr::eq(
third_errors.entries().as_ptr(),
fourth_errors.entries().as_ptr()
));
assert_eq!(calls.load(Ordering::SeqCst), 1);
assert_eq!(second.to_string(), "1 Agent OnClose callback(s) failed");
let teardown = second
.source()
.expect("CloseError::Hooks sources TeardownError")
.downcast_ref::<super::agent::TeardownError>()
.expect("the sourced value is TeardownError");
assert!(std::ptr::eq(teardown, second_errors));
let first = teardown
.source()
.expect("TeardownError sources the first AttributedError")
.downcast_ref::<AttributedError>()
.expect("the sourced value is AttributedError");
assert_eq!(first.name(), "blocking");
assert_eq!(first.error().to_string(), "blocked error");
}
#[test]
fn frozen_error_shapes_display_sources_and_clone_without_flattening() {
let simple = [
StartError::AgentClosed,
StartError::SessionBusy,
StartError::Cancelled,
StartError::RunIdExhausted,
];
assert_eq!(
simple.iter().map(ToString::to_string).collect::<Vec<_>>(),
[
"Agent is closed",
"Session already has an active Run",
"Run admission canceled",
"Run identity exhausted"
]
);
assert!(simple.iter().all(|error| error.source().is_none()));
for error in &simple {
let cloned = error.clone();
assert_eq!(cloned.to_string(), error.to_string());
}
let identity = StartError::InvalidIdentity(CoordError::EmptySessionId);
assert_eq!(identity.to_string(), "session ID must not be empty");
assert!(identity.source().is_some());
assert_eq!(identity.clone().to_string(), identity.to_string());
let admission = StartError::Admission(LeaseError::Released);
assert_eq!(admission.to_string(), "session lease released");
assert!(admission.source().is_some());
assert_eq!(admission.clone().to_string(), admission.to_string());
let transcript = StartError::Transcript(store::Error::Poisoned);
assert_eq!(transcript.to_string(), "transcript store lock poisoned");
assert!(transcript.source().is_some());
assert_eq!(transcript.clone().to_string(), transcript.to_string());
let calls = Arc::new(AtomicUsize::new(0));
let order = Arc::new(Mutex::new(Vec::new()));
let mut builder = hook::RegistryBuilder::new();
builder.register(registration(
"prepare-source",
TeardownResult::Error("prepare error"),
&calls,
&order,
));
let agent = Agent::new(Config::new(Arc::new(Model), empty_tools(), builder)).unwrap();
let CloseError::Hooks(teardown) = close_on_thread(agent).unwrap_err() else {
panic!("teardown failure unexpectedly reported as Busy");
};
let failure = teardown.entries()[0].clone();
let prepare = StartError::Prepare(failure);
assert_eq!(prepare.to_string(), "hook prepare-source prepare failed");
assert!(prepare.source().is_some());
assert_eq!(prepare.clone().to_string(), prepare.to_string());
let config = [
ConfigError::Hooks(hook::RegistryError::EmptyName),
ConfigError::InvalidDefaults(model::RequestError::ToolExchangeNotClosed),
ConfigError::RevisionIdExhausted,
];
for error in &config {
let cloned = error.clone();
assert_eq!(cloned.to_string(), error.to_string());
assert_eq!(cloned.source().is_some(), error.source().is_some());
}
}
#[test]
fn config_and_revision_debug_show_everything_but_the_model() {
let mut hooks = hook::RegistryBuilder::new();
hooks.register(hook::Registration::new("recorder".to_owned()));
let mut config = Config::new(Arc::new(Model), empty_tools(), hooks);
config.push_system_instruction(Part::text(SECRET.into()));
config.push_model_extension(
Envelope::try_new(
"example".to_owned(),
"compat".to_owned(),
Payload::copy_from_slice(format!(r#"{{"t":"{SECRET}"}}"#).as_bytes()),
)
.expect("fixture is valid JSON"),
);
config.sampling_mut().stop_sequences = vec![SECRET.into()];
assert_redacted(&config);
assert_states(
&config,
&[
"Config",
r#"hooks: RegistryBuilder { hooks: ["recorder"] }"#,
"text: <21 bytes>",
"stop_sequences: <1 items>",
"..",
],
);
let agent = Agent::new(config).expect("a valid configuration builds");
let revision = agent.revision();
assert_redacted(&revision);
assert_states(
&revision,
&[
"Revision",
r#"hook_names: ["recorder"]"#,
"text: <21 bytes>",
"hooks_require_session_state: false",
"..",
],
);
assert_eq!(
format!("{agent:?}"),
format!("Agent {{ revision: {} }}", revision.id())
);
}