#![cfg(test)]
use std::error;
use std::future::{Future, ready};
use std::pin::Pin;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use std::task::{Context, Poll, Waker};
use std::time::Duration;
use crate::cancel::Token;
use tokio::sync::{Notify, RwLock};
use super::event::ToolEmitter;
use super::tool::{
self, CallDisposition, Descriptor, ExecutionError, Executor, ExecutorError, ExposeAll,
Invocation, MappingError, StepToolSnapshot,
};
use super::tool_batch::{ErrorOrigin, PreparedToolCall, SettledDisposition, execute_prepared};
use crate::opaque::Payload;
use crate::redact_tests::{SECRET, assert_redacted, assert_states};
use crate::transcript::item::{Item, ModelVisibility, System};
use crate::transcript::store::{Snapshot, Version};
use crate::value::tool_call::{self, Id, ToolCall, ToolResult};
fn assert_pending_once<F: Future + ?Sized>(future: Pin<&mut F>) {
let mut context = Context::from_waker(Waker::noop());
assert!(matches!(future.poll(&mut context), Poll::Pending));
}
fn step_transcript() -> Snapshot {
Snapshot::new(
Version::ZERO,
vec![Item::System(System::new(
"step".to_owned(),
"step top".to_owned(),
ModelVisibility::Visible,
))],
)
}
fn local(name: &str) -> tool::Name {
tool::Name::local(name.to_owned()).unwrap()
}
fn descriptor(name: &str) -> Descriptor {
Descriptor::new(
local(name),
"description".to_owned(),
Payload::copy_from_slice(b"{}"),
)
}
fn call(name: &str) -> ToolCall {
ToolCall::new(
Id::new(format!("call-{name}")),
tool_call::Name::new(name.to_owned()),
Payload::copy_from_slice(b"{}"),
)
}
fn snapshot_of(registrations: Vec<tool::Registration>) -> Arc<StepToolSnapshot> {
let mut builder = tool::RegistryBuilder::new();
for registration in registrations {
builder.register(registration);
}
Arc::new(
builder
.build()
.unwrap()
.resolve_step_without_providers(step_transcript(), &Token::new(), &ExposeAll)
.unwrap()
.into_snapshot(),
)
}
fn executable(snapshot: &Arc<StepToolSnapshot>, name: &str) -> PreparedToolCall {
PreparedToolCall::executable(
Arc::clone(snapshot),
Some("session".to_owned()),
call(name),
ToolEmitter::closed(),
)
}
struct DeferredExecutor {
release: Arc<Notify>,
started: Arc<Notify>,
outcome: Mutex<Result<String, ExecutorError>>,
running: Arc<AtomicUsize>,
peak: Arc<AtomicUsize>,
}
impl Executor for DeferredExecutor {
type Output = String;
async fn execute(
&self,
_invocation: &Invocation,
_call: &ToolCall,
_cancellation: Token,
) -> Result<Self::Output, ExecutorError> {
let running = self.running.fetch_add(1, Ordering::SeqCst) + 1;
self.peak.fetch_max(running, Ordering::SeqCst);
self.started.notify_one();
self.release.notified().await;
self.running.fetch_sub(1, Ordering::SeqCst);
self.outcome.lock().unwrap().clone()
}
}
struct DeferredHandles {
release: Arc<Notify>,
started: Arc<Notify>,
}
struct Overlap {
running: Arc<AtomicUsize>,
peak: Arc<AtomicUsize>,
}
impl Overlap {
fn new() -> Self {
Self {
running: Arc::new(AtomicUsize::new(0)),
peak: Arc::new(AtomicUsize::new(0)),
}
}
fn peak(&self) -> usize {
self.peak.load(Ordering::SeqCst)
}
fn deferred(
&self,
name: &str,
outcome: Result<String, ExecutorError>,
) -> (tool::Registration, DeferredHandles) {
let release = Arc::new(Notify::new());
let started = Arc::new(Notify::new());
let registration = tool::Registration::client(
descriptor(name),
Arc::new(DeferredExecutor {
release: Arc::clone(&release),
started: Arc::clone(&started),
outcome: Mutex::new(outcome),
running: Arc::clone(&self.running),
peak: Arc::clone(&self.peak),
}),
Arc::new(|output: String| Ok(ToolResult::success_text(output))),
);
(registration, DeferredHandles { release, started })
}
}
fn text_of(result: &ToolResult) -> String {
result.output().unwrap().user_view().unwrap().to_owned()
}
async fn with_timeout<F: Future>(future: F) -> F::Output {
tokio::time::timeout(Duration::from_secs(5), future)
.await
.expect("the batch engine must join all owned work without hanging")
}
#[tokio::test(flavor = "current_thread")]
async fn executable_slots_genuinely_overlap_rather_than_running_serially() {
let overlap = Overlap::new();
let (first, first_handles) = overlap.deferred("first", Ok("first".to_owned()));
let (second, second_handles) = overlap.deferred("second", Ok("second".to_owned()));
let snapshot = snapshot_of(vec![first, second]);
let entries = vec![
executable(&snapshot, "first"),
executable(&snapshot, "second"),
];
let cancellation = Token::new();
let mut batch = Box::pin(execute_prepared(entries, &cancellation));
assert_pending_once(batch.as_mut());
with_timeout(first_handles.started.notified()).await;
with_timeout(second_handles.started.notified()).await;
assert_eq!(overlap.peak(), 2);
first_handles.release.notify_one();
second_handles.release.notify_one();
let outcomes = with_timeout(batch).await.unwrap();
assert_eq!(overlap.peak(), 2);
assert_eq!(
outcomes
.iter()
.map(|outcome| text_of(outcome.result()))
.collect::<Vec<_>>(),
vec!["first".to_owned(), "second".to_owned()]
);
}
#[tokio::test(flavor = "current_thread")]
async fn reverse_completion_order_still_returns_source_order_with_settled_slots() {
let overlap = Overlap::new();
let (first, first_handles) = overlap.deferred("first", Ok("first".to_owned()));
let (second, second_handles) = overlap.deferred("second", Ok("second".to_owned()));
let snapshot = snapshot_of(vec![first, second]);
let entries = vec![
executable(&snapshot, "first"),
PreparedToolCall::settled(
ToolResult::success_text("short-circuit".to_owned()),
SettledDisposition::ShortCircuited,
),
executable(&snapshot, "second"),
PreparedToolCall::settled(ToolResult::no_output(), SettledDisposition::Blocked),
];
let cancellation = Token::new();
let mut batch = Box::pin(execute_prepared(entries, &cancellation));
assert_pending_once(batch.as_mut());
second_handles.release.notify_one();
with_timeout(second_handles.started.notified()).await;
assert_pending_once(batch.as_mut());
first_handles.release.notify_one();
let outcomes = with_timeout(batch).await.unwrap();
assert_eq!(outcomes.len(), 4);
assert_eq!(text_of(outcomes[0].result()), "first");
assert_eq!(outcomes[0].disposition(), CallDisposition::Executed);
assert_eq!(text_of(outcomes[1].result()), "short-circuit");
assert_eq!(outcomes[1].disposition(), CallDisposition::ShortCircuited);
assert_eq!(text_of(outcomes[2].result()), "second");
assert_eq!(outcomes[2].disposition(), CallDisposition::Executed);
assert!(!outcomes[3].result().has_output());
assert_eq!(outcomes[3].disposition(), CallDisposition::Blocked);
let (result, disposition) = outcomes.into_iter().next().unwrap().into_parts();
assert_eq!(text_of(&result), "first");
assert_eq!(disposition, CallDisposition::Executed);
}
fn immediate(name: &str, outcome: Result<String, ExecutorError>) -> tool::Registration {
tool::Registration::client(
descriptor(name),
Arc::new(ImmediateExecutor(Mutex::new(outcome))),
Arc::new(|output: String| Ok(ToolResult::success_text(output))),
)
}
struct ImmediateExecutor(Mutex<Result<String, ExecutorError>>);
impl Executor for ImmediateExecutor {
type Output = String;
fn execute(
&self,
_invocation: &Invocation,
_call: &ToolCall,
_cancellation: Token,
) -> impl Future<Output = Result<Self::Output, ExecutorError>> + Send {
ready(self.0.lock().unwrap().clone())
}
}
#[tokio::test]
async fn unknown_and_ordinary_error_are_ordered_business_outcomes() {
let snapshot = snapshot_of(vec![
immediate("allowed", Ok("ok".to_owned())),
immediate(
"failing",
Err(ExecutorError::new("ordinary failure".to_owned())),
),
]);
let entries = vec![
executable(&snapshot, "absent"),
executable(&snapshot, "failing"),
executable(&snapshot, "allowed"),
];
let outcomes = with_timeout(execute_prepared(entries, &Token::new()))
.await
.unwrap();
assert_eq!(
outcomes
.iter()
.map(|outcome| (outcome.disposition(), outcome.result().is_business_error()))
.collect::<Vec<_>>(),
vec![
(CallDisposition::Rejected, true),
(CallDisposition::Executed, true),
(CallDisposition::Executed, false),
]
);
assert_eq!(text_of(outcomes[0].result()), "unknown tool: absent");
assert_eq!(text_of(outcomes[1].result()), "ordinary failure");
}
struct FailingMapExecutor;
impl Executor for FailingMapExecutor {
type Output = String;
fn execute(
&self,
_invocation: &Invocation,
_call: &ToolCall,
_cancellation: Token,
) -> impl Future<Output = Result<Self::Output, ExecutorError>> + Send {
ready(Ok("unmappable".to_owned()))
}
}
#[tokio::test]
async fn every_typed_control_path_failure_fails_the_batch_with_no_outcomes() {
let mapping_error = MappingError::Protocol("bad output".to_owned());
let registration = tool::Registration::client(
descriptor("mapping"),
Arc::new(FailingMapExecutor),
Arc::new({
let error = mapping_error.clone();
move |_output: String| Err(error.clone())
}),
);
let snapshot = snapshot_of(vec![registration]);
let entries = vec![
PreparedToolCall::settled(
ToolResult::success_text("settled".to_owned()),
SettledDisposition::ShortCircuited,
),
executable(&snapshot, "mapping"),
];
let failure = with_timeout(execute_prepared(entries, &Token::new()))
.await
.err()
.unwrap();
let expected = ExecutionError::Mapping(mapping_error);
assert_eq!(failure.origin(), ErrorOrigin::Slot(1));
assert_eq!(failure.execution(), &expected);
assert_eq!(
failure.to_string(),
format!("prepared tool batch failed at source index 1: {expected}")
);
let _: &dyn std::error::Error = &failure;
assert_eq!(failure.clone().into_execution(), expected);
assert_deprecated_execution_accessors(&failure);
assert!(format!("{failure:?}").contains("Error"));
}
#[expect(
deprecated,
reason = "the compatibility test intentionally exercises the deprecated aliases"
)]
fn assert_deprecated_execution_accessors(failure: &super::tool_batch::Error) {
assert_eq!(failure.error(), failure.execution());
assert_eq!(
failure.clone().into_error(),
failure.clone().into_execution()
);
}
struct ObservingExecutor(Arc<Observations>);
#[derive(Default)]
struct Observations {
started: Notify,
dropped_with_cancelled_token: AtomicUsize,
dropped: AtomicUsize,
completed: AtomicUsize,
}
struct DropProbe {
observations: Arc<Observations>,
cancellation: Token,
}
impl Drop for DropProbe {
fn drop(&mut self) {
self.observations.dropped.fetch_add(1, Ordering::SeqCst);
if self.cancellation.is_cancelled() {
self.observations
.dropped_with_cancelled_token
.fetch_add(1, Ordering::SeqCst);
}
}
}
impl Executor for ObservingExecutor {
type Output = String;
async fn execute(
&self,
_invocation: &Invocation,
_call: &ToolCall,
cancellation: Token,
) -> Result<Self::Output, ExecutorError> {
let _probe = DropProbe {
observations: Arc::clone(&self.0),
cancellation: cancellation.clone(),
};
self.0.started.notify_one();
cancellation.cancelled().await;
self.0.completed.fetch_add(1, Ordering::SeqCst);
Ok("never materialized".to_owned())
}
}
fn observing(name: &str, observations: &Arc<Observations>) -> tool::Registration {
tool::Registration::client(
descriptor(name),
Arc::new(ObservingExecutor(Arc::clone(observations))),
Arc::new(|output: String| Ok(ToolResult::success_text(output))),
)
}
#[tokio::test(flavor = "current_thread")]
async fn first_fatal_cancels_siblings_that_observe_it_and_all_runs_are_joined() {
let observations = Arc::new(Observations::default());
let snapshot = snapshot_of(vec![
tool::Registration::client(
descriptor("fatal"),
Arc::new(FailingMapExecutor),
Arc::new(|_output: String| Err(MappingError::Protocol("unmappable".to_owned()))),
),
observing("sibling", &observations),
]);
let entries = vec![
executable(&snapshot, "fatal"),
executable(&snapshot, "sibling"),
];
let cancellation = Token::new();
let failure = with_timeout(execute_prepared(entries, &cancellation))
.await
.err()
.unwrap();
assert_eq!(
*failure.execution(),
ExecutionError::Mapping(MappingError::Protocol("unmappable".to_owned()))
);
assert_eq!(failure.origin(), ErrorOrigin::Slot(0));
assert_eq!(observations.dropped.load(Ordering::SeqCst), 1);
assert_eq!(
observations
.dropped_with_cancelled_token
.load(Ordering::SeqCst),
1
);
assert_eq!(observations.completed.load(Ordering::SeqCst), 0);
assert!(!cancellation.is_cancelled());
}
struct TriggeredExecutor(Token);
impl Executor for TriggeredExecutor {
type Output = String;
async fn execute(
&self,
_invocation: &Invocation,
_call: &ToolCall,
_cancellation: Token,
) -> Result<Self::Output, ExecutorError> {
self.0.cancelled().await;
Ok("ready to map".to_owned())
}
}
fn triggered_mapping_failure(
name: &'static str,
trigger: &Token,
mappings: &Arc<AtomicUsize>,
) -> tool::Registration {
let mappings = Arc::clone(mappings);
tool::Registration::client(
descriptor(name),
Arc::new(TriggeredExecutor(trigger.clone())),
Arc::new(move |_output: String| {
mappings.fetch_add(1, Ordering::SeqCst);
Err(MappingError::Protocol(name.to_owned()))
}),
)
}
#[tokio::test(flavor = "current_thread")]
async fn same_poll_pass_fatals_choose_the_lower_source_index() {
let trigger = Token::new();
let mappings = Arc::new(AtomicUsize::new(0));
let snapshot = snapshot_of(vec![
triggered_mapping_failure("earlier", &trigger, &mappings),
triggered_mapping_failure("later", &trigger, &mappings),
]);
let entries = vec![
PreparedToolCall::settled(
ToolResult::success_text("settled".to_owned()),
SettledDisposition::ShortCircuited,
),
executable(&snapshot, "earlier"),
executable(&snapshot, "later"),
];
let cancellation = Token::new();
let mut batch = Box::pin(execute_prepared(entries, &cancellation));
assert_pending_once(batch.as_mut());
trigger.cancel();
let failure = with_timeout(batch).await.err().unwrap();
assert_eq!(mappings.load(Ordering::SeqCst), 2);
assert_eq!(failure.origin(), ErrorOrigin::Slot(1));
assert_eq!(
failure.execution(),
&ExecutionError::Mapping(MappingError::Protocol("earlier".to_owned()))
);
}
struct CancellingExecutor(Token);
impl Executor for CancellingExecutor {
type Output = String;
fn execute(
&self,
_invocation: &Invocation,
_call: &ToolCall,
_cancellation: Token,
) -> impl Future<Output = Result<Self::Output, ExecutorError>> + Send {
self.0.cancel();
ready(Ok("ready to map".to_owned()))
}
}
struct CountingExecutor(Arc<AtomicUsize>);
impl Executor for CountingExecutor {
type Output = String;
fn execute(
&self,
_invocation: &Invocation,
_call: &ToolCall,
_cancellation: Token,
) -> impl Future<Output = Result<Self::Output, ExecutorError>> + Send {
self.0.fetch_add(1, Ordering::SeqCst);
ready(Ok("unexpected execution".to_owned()))
}
}
#[tokio::test(flavor = "current_thread")]
async fn same_poll_pass_fatal_outranks_cancellation() {
let cancellation = Token::new();
let later_calls = Arc::new(AtomicUsize::new(0));
let snapshot = snapshot_of(vec![
tool::Registration::client(
descriptor("fatal"),
Arc::new(CancellingExecutor(cancellation.clone())),
Arc::new(|_output: String| Err(MappingError::Protocol("fatal wins".to_owned()))),
),
tool::Registration::client(
descriptor("cancelled"),
Arc::new(CountingExecutor(Arc::clone(&later_calls))),
Arc::new(|output: String| Ok(ToolResult::success_text(output))),
),
]);
let entries = vec![
PreparedToolCall::settled(
ToolResult::success_text("settled".to_owned()),
SettledDisposition::ShortCircuited,
),
executable(&snapshot, "fatal"),
executable(&snapshot, "cancelled"),
];
let failure = with_timeout(execute_prepared(entries, &cancellation))
.await
.err()
.unwrap();
assert!(cancellation.is_cancelled());
assert_eq!(later_calls.load(Ordering::SeqCst), 0);
assert_eq!(failure.origin(), ErrorOrigin::Slot(1));
assert_eq!(
failure.execution(),
&ExecutionError::Mapping(MappingError::Protocol("fatal wins".to_owned()))
);
}
#[test]
fn latch_prefers_a_real_failure_over_cancellation_and_keeps_the_first_equal_rank() {
let mut latched = None;
super::tool_batch::latch(&mut latched, 3, ExecutionError::Cancelled);
super::tool_batch::latch(
&mut latched,
2,
ExecutionError::Mapping(MappingError::Protocol("first".to_owned())),
);
super::tool_batch::latch(&mut latched, 1, ExecutionError::Hosted("later".to_owned()));
let failure = latched.expect("a failure was latched");
assert_eq!(failure.origin(), ErrorOrigin::Slot(2));
assert_eq!(
failure.execution(),
&ExecutionError::Mapping(MappingError::Protocol("first".to_owned()))
);
}
struct PermanentlyPendingExecutor(Arc<Observations>);
impl Executor for PermanentlyPendingExecutor {
type Output = String;
async fn execute(
&self,
_invocation: &Invocation,
_call: &ToolCall,
_cancellation: Token,
) -> Result<Self::Output, ExecutorError> {
let _probe = DropProbe {
observations: Arc::clone(&self.0),
cancellation: Token::new(),
};
self.0.started.notify_one();
std::future::pending::<()>().await;
unreachable!("a permanently pending executor never completes")
}
}
#[tokio::test(flavor = "current_thread")]
async fn external_cancellation_drops_permanently_pending_runs_without_hanging() {
let observations = Arc::new(Observations::default());
let snapshot = snapshot_of(vec![
tool::Registration::client(
descriptor("pending-a"),
Arc::new(PermanentlyPendingExecutor(Arc::clone(&observations))),
Arc::new(|output: String| Ok(ToolResult::success_text(output))),
),
tool::Registration::client(
descriptor("pending-b"),
Arc::new(PermanentlyPendingExecutor(Arc::clone(&observations))),
Arc::new(|output: String| Ok(ToolResult::success_text(output))),
),
]);
let entries = vec![
PreparedToolCall::settled(
ToolResult::success_text("settled".to_owned()),
SettledDisposition::ShortCircuited,
),
executable(&snapshot, "pending-a"),
executable(&snapshot, "pending-b"),
];
let cancellation = Token::new();
let mut batch = Box::pin(execute_prepared(entries, &cancellation));
assert_pending_once(batch.as_mut());
with_timeout(observations.started.notified()).await;
cancellation.cancel();
let failure = with_timeout(batch).await.err().unwrap();
assert_eq!(*failure.execution(), ExecutionError::Cancelled);
assert_eq!(failure.origin(), ErrorOrigin::Slot(1));
assert_eq!(observations.dropped.load(Ordering::SeqCst), 2);
assert_eq!(observations.completed.load(Ordering::SeqCst), 0);
}
#[tokio::test]
async fn batches_without_executable_slots_return_reserved_positions_in_order() {
let cancellation = Token::new();
assert!(
with_timeout(execute_prepared(Vec::new(), &cancellation))
.await
.unwrap()
.is_empty()
);
let outcomes = with_timeout(execute_prepared(
vec![
PreparedToolCall::settled(
ToolResult::success_text("first".to_owned()),
SettledDisposition::Truncated,
),
PreparedToolCall::settled(
ToolResult::error_text("second".to_owned()),
SettledDisposition::NotExecuted,
),
],
&cancellation,
))
.await
.unwrap();
let cloned = outcomes[0].clone();
assert_eq!(text_of(cloned.result()), "first");
assert_eq!(cloned.disposition(), CallDisposition::Truncated);
assert_eq!(text_of(outcomes[1].result()), "second");
assert!(outcomes[1].result().is_business_error());
assert_eq!(outcomes[1].disposition(), CallDisposition::NotExecuted);
}
#[tokio::test]
async fn pre_cancelled_empty_and_fully_settled_batches_are_cancelled() {
let cancellation = Token::new();
cancellation.cancel();
let empty = execute_prepared(Vec::new(), &cancellation).await.err();
let empty = empty.unwrap();
assert_eq!(*empty.execution(), ExecutionError::Cancelled);
assert_eq!(empty.origin(), ErrorOrigin::Batch);
assert_eq!(
empty.to_string(),
"prepared tool batch failed with no source position: tool run cancelled"
);
let settled = execute_prepared(
vec![
PreparedToolCall::settled(
ToolResult::success_text("first".to_owned()),
SettledDisposition::ShortCircuited,
),
PreparedToolCall::settled(ToolResult::no_output(), SettledDisposition::NotExecuted),
],
&cancellation,
)
.await
.err()
.unwrap();
assert_eq!(*settled.execution(), ExecutionError::Cancelled);
assert_eq!(settled.origin(), ErrorOrigin::Batch);
}
struct SelfCancellingExecutor(Token);
impl Executor for SelfCancellingExecutor {
type Output = String;
fn execute(
&self,
_invocation: &Invocation,
_call: &ToolCall,
_cancellation: Token,
) -> impl Future<Output = Result<Self::Output, ExecutorError>> + Send {
self.0.cancel();
ready(Ok("outran cancellation".to_owned()))
}
}
#[tokio::test]
async fn cancellation_that_every_slot_outran_still_cancels_the_batch() {
let cancellation = Token::new();
let snapshot = snapshot_of(vec![tool::Registration::client(
descriptor("outrunner"),
Arc::new(SelfCancellingExecutor(cancellation.clone())),
Arc::new(|output: String| Ok(ToolResult::success_text(output))),
)]);
let entries = vec![
PreparedToolCall::settled(
ToolResult::success_text("settled".to_owned()),
SettledDisposition::Truncated,
),
executable(&snapshot, "outrunner"),
];
let failure = with_timeout(execute_prepared(entries, &cancellation))
.await
.err()
.unwrap();
assert_eq!(*failure.execution(), ExecutionError::Cancelled);
assert_eq!(failure.origin(), ErrorOrigin::Batch);
}
#[tokio::test(flavor = "current_thread")]
async fn dropping_the_batch_mid_flight_cancels_then_drops_every_started_run() {
let observations = Arc::new(Observations::default());
let snapshot = snapshot_of(vec![
observing("mid-a", &observations),
observing("mid-b", &observations),
]);
let entries = vec![
executable(&snapshot, "mid-a"),
PreparedToolCall::settled(ToolResult::no_output(), SettledDisposition::Blocked),
executable(&snapshot, "mid-b"),
];
let cancellation = Token::new();
let mut batch = Box::pin(execute_prepared(entries, &cancellation));
assert_pending_once(batch.as_mut());
with_timeout(observations.started.notified()).await;
assert_eq!(observations.dropped.load(Ordering::SeqCst), 0);
drop(batch);
assert_eq!(observations.dropped.load(Ordering::SeqCst), 2);
assert_eq!(
observations
.dropped_with_cancelled_token
.load(Ordering::SeqCst),
2
);
assert_eq!(observations.completed.load(Ordering::SeqCst), 0);
assert!(!cancellation.is_cancelled());
}
#[test]
fn error_has_exactly_one_reachable_source_and_it_is_the_trait_method() {
let failure = super::tool_batch::Error::cancelled_batch();
assert_eq!(failure.origin(), ErrorOrigin::Batch);
use std::error::Error as _;
let retained = |cause: Option<&(dyn error::Error + 'static)>| {
cause
.expect("a batch failure retains its typed run failure")
.to_string()
};
let expected = ExecutionError::Cancelled.to_string();
assert_eq!(retained(error::Error::source(&failure)), expected);
assert_eq!(retained(failure.source()), expected);
let dynamic: &dyn error::Error = &failure;
assert_eq!(retained(dynamic.source()), expected);
assert!(
failure
.source()
.and_then(|cause| cause.downcast_ref::<ExecutionError>())
.is_some_and(|cause| *cause == ExecutionError::Cancelled)
);
}
struct Occupancy {
running: AtomicUsize,
peak: AtomicUsize,
peak_beside_exclusive: AtomicUsize,
}
impl Occupancy {
fn new() -> Arc<Self> {
Arc::new(Self {
running: AtomicUsize::new(0),
peak: AtomicUsize::new(0),
peak_beside_exclusive: AtomicUsize::new(0),
})
}
fn enter(&self, exclusive: bool) {
let running = self.running.fetch_add(1, Ordering::SeqCst) + 1;
self.peak.fetch_max(running, Ordering::SeqCst);
if exclusive {
self.peak_beside_exclusive
.fetch_max(running, Ordering::SeqCst);
}
}
fn leave(&self, exclusive: bool) {
if exclusive {
self.peak_beside_exclusive
.fetch_max(self.running.load(Ordering::SeqCst), Ordering::SeqCst);
}
self.running.fetch_sub(1, Ordering::SeqCst);
}
}
struct ExclusionExecutor {
lane: Arc<RwLock<()>>,
exclusive: bool,
label: &'static str,
occupancy: Arc<Occupancy>,
started: Arc<Notify>,
release: Arc<Notify>,
}
impl ExclusionExecutor {
async fn body(&self) -> Result<String, ExecutorError> {
self.occupancy.enter(self.exclusive);
self.started.notify_one();
self.release.notified().await;
self.occupancy.leave(self.exclusive);
Ok(self.label.to_owned())
}
}
impl Executor for ExclusionExecutor {
type Output = String;
async fn execute(
&self,
_invocation: &Invocation,
_call: &ToolCall,
_cancellation: Token,
) -> Result<Self::Output, ExecutorError> {
if self.exclusive {
let _lane = self.lane.write().await;
self.body().await
} else {
let _lane = self.lane.read().await;
self.body().await
}
}
}
struct ExclusionHandles {
started: Arc<Notify>,
release: Arc<Notify>,
}
fn exclusion_tool(
lane: &Arc<RwLock<()>>,
occupancy: &Arc<Occupancy>,
label: &'static str,
exclusive: bool,
) -> (tool::Registration, ExclusionHandles) {
let started = Arc::new(Notify::new());
let release = Arc::new(Notify::new());
let registration = tool::Registration::client(
descriptor(label),
Arc::new(ExclusionExecutor {
lane: Arc::clone(lane),
exclusive,
label,
occupancy: Arc::clone(occupancy),
started: Arc::clone(&started),
release: Arc::clone(&release),
}),
Arc::new(|output: String| Ok(ToolResult::success_text(output))),
);
(registration, ExclusionHandles { started, release })
}
#[tokio::test(flavor = "current_thread")]
async fn a_composition_owned_lane_makes_one_tool_exclusive_without_reordering_the_batch() {
let lane = Arc::new(RwLock::new(()));
let occupancy = Occupancy::new();
let (reader_a, a) = exclusion_tool(&lane, &occupancy, "reader-a", false);
let (reader_b, b) = exclusion_tool(&lane, &occupancy, "reader-b", false);
let (writer, x) = exclusion_tool(&lane, &occupancy, "exclusive", true);
let (reader_c, c) = exclusion_tool(&lane, &occupancy, "reader-c", false);
let snapshot = snapshot_of(vec![reader_a, reader_b, writer, reader_c]);
let entries = vec![
executable(&snapshot, "reader-a"),
executable(&snapshot, "reader-b"),
executable(&snapshot, "exclusive"),
executable(&snapshot, "reader-c"),
];
let cancellation = Token::new();
let mut batch = Box::pin(execute_prepared(entries, &cancellation));
assert_pending_once(batch.as_mut());
with_timeout(a.started.notified()).await;
with_timeout(b.started.notified()).await;
assert!(
occupancy.peak.load(Ordering::SeqCst) >= 2,
"the engine still overlaps the concurrency-safe peers"
);
for handles in [&a, &b, &x, &c] {
handles.release.notify_one();
}
let outcomes = with_timeout(batch).await.unwrap();
assert_eq!(
occupancy.peak_beside_exclusive.load(Ordering::SeqCst),
1,
"nothing else was inside its executor while the exclusive tool was"
);
assert_eq!(
outcomes
.iter()
.map(|outcome| text_of(outcome.result()))
.collect::<Vec<_>>(),
["reader-a", "reader-b", "exclusive", "reader-c"],
"serialising execution does not perturb the source order C14 commits"
);
}
#[test]
fn prepared_call_debug_says_whether_the_position_still_has_to_run() {
let settled = PreparedToolCall::settled(
ToolResult::success_text(SECRET.to_owned()),
SettledDisposition::ShortCircuited,
);
assert_redacted(&settled);
assert_eq!(
format!("{settled:?}"),
"PreparedToolCall { settled: true, disposition: ShortCircuited }"
);
let snapshot = snapshot_of(vec![immediate("search", Ok(SECRET.to_owned()))]);
let executable = executable(&snapshot, "search");
assert_redacted(&executable);
assert_states(
&executable,
&[
"settled: false",
r#"Id("call-search")"#,
"arguments: <2 bytes>",
],
);
}
#[tokio::test]
async fn batch_outcome_debug_keeps_the_disposition_and_redacts_the_result() {
let snapshot = snapshot_of(vec![immediate("search", Ok(SECRET.to_owned()))]);
let outcomes = execute_prepared(
vec![
executable(&snapshot, "search"),
PreparedToolCall::settled(
ToolResult::error_text(SECRET.to_owned()),
SettledDisposition::Truncated,
),
],
&Token::new(),
)
.await
.expect("a successful batch materializes in source order");
for outcome in &outcomes {
assert_redacted(outcome);
}
assert_states(&outcomes[0], &["disposition: Executed", "Success("]);
assert_states(
&outcomes[1],
&["disposition: Truncated", "user_view: Some(<21 bytes>)"],
);
}