#![cfg(test)]
use std::error;
use std::future::{Future, ready};
use std::pin::Pin;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use std::time::Duration;
use crate::cancel::Token;
use tokio::sync::{Notify, RwLock};
use super::event::ToolEmitter;
use super::tool::{
self, Approval, CallDisposition, Descriptor, ExecutionError, Executor, ExecutorError,
ExposeAll, Gate, GateOperationError, GateVerdict, Invocation, MappingError, Requirements,
StepToolSnapshot,
};
use super::tool_batch::{ErrorOrigin, PreparedToolCall, SettledDisposition, execute_prepared};
use crate::opaque::Payload;
use crate::redact_tests::{SECRET, assert_redacted, assert_states};
use crate::transcript::item::{Item, ModelVisibility, System};
use crate::transcript::store::{Snapshot, Version};
use crate::value::tool_call::{self, Id, ToolCall, ToolResult};
fn step_transcript() -> Snapshot {
Snapshot::new(
Version::ZERO,
vec![Item::System(System::new(
"step".to_owned(),
"step top".to_owned(),
ModelVisibility::Visible,
))],
)
}
fn local(name: &str) -> tool::Name {
tool::Name::local(name.to_owned()).unwrap()
}
fn descriptor(name: &str) -> Descriptor {
Descriptor::new(
local(name),
"description".to_owned(),
Payload::copy_from_slice(b"{}"),
)
}
fn call(name: &str) -> ToolCall {
ToolCall::new(
Id::new(format!("call-{name}")),
tool_call::Name::new(name.to_owned()),
Payload::copy_from_slice(b"{}"),
)
}
fn snapshot_of(registrations: Vec<tool::Registration>) -> Arc<StepToolSnapshot> {
let mut builder = tool::RegistryBuilder::new();
for registration in registrations {
builder.register(registration);
}
Arc::new(
builder
.build()
.unwrap()
.resolve_step_without_providers(step_transcript(), &Token::new(), &ExposeAll)
.unwrap()
.into_snapshot(),
)
}
fn executable(
snapshot: &Arc<StepToolSnapshot>,
name: &str,
gate: Option<Arc<dyn Gate>>,
) -> PreparedToolCall {
PreparedToolCall::executable(
Arc::clone(snapshot),
Some("session".to_owned()),
call(name),
ToolEmitter::closed(),
gate,
)
}
struct GatedExecutor {
release: Arc<Notify>,
started: Arc<Notify>,
outcome: Mutex<Result<String, ExecutorError>>,
running: Arc<AtomicUsize>,
peak: Arc<AtomicUsize>,
}
impl Executor for GatedExecutor {
type Output = String;
async fn execute(
&self,
_invocation: &Invocation,
_call: &ToolCall,
_cancellation: Token,
) -> Result<Self::Output, ExecutorError> {
let running = self.running.fetch_add(1, Ordering::SeqCst) + 1;
self.peak.fetch_max(running, Ordering::SeqCst);
self.started.notify_one();
self.release.notified().await;
self.running.fetch_sub(1, Ordering::SeqCst);
self.outcome.lock().unwrap().clone()
}
}
struct GatedHandles {
release: Arc<Notify>,
started: Arc<Notify>,
}
struct Overlap {
running: Arc<AtomicUsize>,
peak: Arc<AtomicUsize>,
}
impl Overlap {
fn new() -> Self {
Self {
running: Arc::new(AtomicUsize::new(0)),
peak: Arc::new(AtomicUsize::new(0)),
}
}
fn peak(&self) -> usize {
self.peak.load(Ordering::SeqCst)
}
fn gated(
&self,
name: &str,
outcome: Result<String, ExecutorError>,
) -> (tool::Registration, GatedHandles) {
let release = Arc::new(Notify::new());
let started = Arc::new(Notify::new());
let registration = tool::Registration::client(
descriptor(name),
Requirements::new(Approval::NotRequired, Vec::new()),
Arc::new(GatedExecutor {
release: Arc::clone(&release),
started: Arc::clone(&started),
outcome: Mutex::new(outcome),
running: Arc::clone(&self.running),
peak: Arc::clone(&self.peak),
}),
Arc::new(|output: String| Ok(ToolResult::success_text(output))),
);
(registration, GatedHandles { release, started })
}
}
fn text_of(result: &ToolResult) -> String {
result.output().unwrap().user_view().unwrap().to_owned()
}
async fn with_timeout<F: Future>(future: F) -> F::Output {
tokio::time::timeout(Duration::from_secs(5), future)
.await
.expect("the batch engine must join all owned work without hanging")
}
#[tokio::test(flavor = "current_thread")]
async fn executable_slots_genuinely_overlap_rather_than_running_serially() {
let overlap = Overlap::new();
let (first, first_handles) = overlap.gated("first", Ok("first".to_owned()));
let (second, second_handles) = overlap.gated("second", Ok("second".to_owned()));
let snapshot = snapshot_of(vec![first, second]);
let entries = vec![
executable(&snapshot, "first", None),
executable(&snapshot, "second", None),
];
let cancellation = Token::new();
let mut batch = Box::pin(execute_prepared(entries, &cancellation));
assert!(
tokio::time::timeout(Duration::from_millis(50), &mut batch)
.await
.is_err()
);
with_timeout(first_handles.started.notified()).await;
with_timeout(second_handles.started.notified()).await;
assert_eq!(overlap.peak(), 2);
first_handles.release.notify_one();
second_handles.release.notify_one();
let outcomes = with_timeout(batch).await.unwrap();
assert_eq!(overlap.peak(), 2);
assert_eq!(
outcomes
.iter()
.map(|outcome| text_of(outcome.result()))
.collect::<Vec<_>>(),
vec!["first".to_owned(), "second".to_owned()]
);
}
#[tokio::test(flavor = "current_thread")]
async fn reverse_completion_order_still_returns_source_order_with_settled_slots() {
let overlap = Overlap::new();
let (first, first_handles) = overlap.gated("first", Ok("first".to_owned()));
let (second, second_handles) = overlap.gated("second", Ok("second".to_owned()));
let snapshot = snapshot_of(vec![first, second]);
let entries = vec![
executable(&snapshot, "first", None),
PreparedToolCall::settled(
ToolResult::success_text("short-circuit".to_owned()),
SettledDisposition::ShortCircuited,
),
executable(&snapshot, "second", None),
PreparedToolCall::settled(ToolResult::no_output(), SettledDisposition::Blocked),
];
let cancellation = Token::new();
let mut batch = Box::pin(execute_prepared(entries, &cancellation));
assert!(
tokio::time::timeout(Duration::from_millis(50), &mut batch)
.await
.is_err()
);
second_handles.release.notify_one();
with_timeout(second_handles.started.notified()).await;
assert!(
tokio::time::timeout(Duration::from_millis(50), &mut batch)
.await
.is_err()
);
first_handles.release.notify_one();
let outcomes = with_timeout(batch).await.unwrap();
assert_eq!(outcomes.len(), 4);
assert_eq!(text_of(outcomes[0].result()), "first");
assert_eq!(outcomes[0].disposition(), CallDisposition::Executed);
assert_eq!(text_of(outcomes[1].result()), "short-circuit");
assert_eq!(outcomes[1].disposition(), CallDisposition::ShortCircuited);
assert_eq!(text_of(outcomes[2].result()), "second");
assert_eq!(outcomes[2].disposition(), CallDisposition::Executed);
assert!(!outcomes[3].result().has_output());
assert_eq!(outcomes[3].disposition(), CallDisposition::Blocked);
let (result, disposition) = outcomes.into_iter().next().unwrap().into_parts();
assert_eq!(text_of(&result), "first");
assert_eq!(disposition, CallDisposition::Executed);
}
struct FixedGate(Result<GateVerdict, GateOperationError>);
impl Gate for FixedGate {
fn decide<'a>(
&'a self,
_call: &'a ToolCall,
_requirements: &'a Requirements,
_cancellation: Token,
) -> Pin<Box<dyn Future<Output = Result<GateVerdict, GateOperationError>> + Send + 'a>> {
Box::pin(ready(self.0.clone()))
}
}
fn immediate(name: &str, outcome: Result<String, ExecutorError>) -> tool::Registration {
tool::Registration::client(
descriptor(name),
Requirements::new(Approval::Required, Vec::new()),
Arc::new(ImmediateExecutor(Mutex::new(outcome))),
Arc::new(|output: String| Ok(ToolResult::success_text(output))),
)
}
struct ImmediateExecutor(Mutex<Result<String, ExecutorError>>);
impl Executor for ImmediateExecutor {
type Output = String;
fn execute(
&self,
_invocation: &Invocation,
_call: &ToolCall,
_cancellation: Token,
) -> impl Future<Output = Result<Self::Output, ExecutorError>> + Send {
ready(self.0.lock().unwrap().clone())
}
}
struct CountingExecutor(Arc<AtomicUsize>);
impl Executor for CountingExecutor {
type Output = String;
fn execute(
&self,
_invocation: &Invocation,
_call: &ToolCall,
_cancellation: Token,
) -> impl Future<Output = Result<Self::Output, ExecutorError>> + Send {
self.0.fetch_add(1, Ordering::SeqCst);
ready(Ok("counted".to_owned()))
}
}
fn counting(name: &str, calls: Arc<AtomicUsize>) -> tool::Registration {
tool::Registration::client(
descriptor(name),
Requirements::new(Approval::Required, Vec::new()),
Arc::new(CountingExecutor(calls)),
Arc::new(|output: String| Ok(ToolResult::success_text(output))),
)
}
#[tokio::test]
async fn unknown_deny_and_ordinary_error_are_ordered_business_outcomes() {
let snapshot = snapshot_of(vec![
immediate("allowed", Ok("ok".to_owned())),
immediate(
"failing",
Err(ExecutorError::new("ordinary failure".to_owned())),
),
immediate("denied", Ok("never".to_owned())),
]);
let allow: Arc<dyn Gate> = Arc::new(FixedGate(Ok(GateVerdict::Allow)));
let deny: Arc<dyn Gate> = Arc::new(FixedGate(Ok(GateVerdict::Deny)));
let entries = vec![
executable(&snapshot, "absent", Some(Arc::clone(&allow))),
executable(&snapshot, "denied", Some(deny)),
executable(&snapshot, "failing", Some(Arc::clone(&allow))),
executable(&snapshot, "allowed", Some(allow)),
];
let outcomes = with_timeout(execute_prepared(entries, &Token::new()))
.await
.unwrap();
assert_eq!(
outcomes
.iter()
.map(|outcome| (outcome.disposition(), outcome.result().is_business_error()))
.collect::<Vec<_>>(),
vec![
(CallDisposition::Rejected, true),
(CallDisposition::Denied, true),
(CallDisposition::Executed, true),
(CallDisposition::Executed, false),
]
);
assert_eq!(text_of(outcomes[0].result()), "unknown tool: absent");
assert_eq!(text_of(outcomes[2].result()), "ordinary failure");
}
struct FailingMapExecutor;
impl Executor for FailingMapExecutor {
type Output = String;
fn execute(
&self,
_invocation: &Invocation,
_call: &ToolCall,
_cancellation: Token,
) -> impl Future<Output = Result<Self::Output, ExecutorError>> + Send {
ready(Ok("unmappable".to_owned()))
}
}
struct FatalCase {
name: &'static str,
registration: tool::Registration,
gate: Option<Arc<dyn Gate>>,
expected: ExecutionError,
}
#[tokio::test]
async fn every_typed_control_path_failure_fails_the_batch_with_no_outcomes() {
let mapping_error = MappingError::Protocol("bad output".to_owned());
let cases = vec![
FatalCase {
name: "mapping",
registration: tool::Registration::client(
descriptor("mapping"),
Requirements::new(Approval::NotRequired, Vec::new()),
Arc::new(FailingMapExecutor),
Arc::new({
let error = mapping_error.clone();
move |_output: String| Err(error.clone())
}),
),
gate: None,
expected: ExecutionError::Mapping(mapping_error),
},
FatalCase {
name: "invalid-verdict",
registration: immediate("invalid-verdict", Ok("never".to_owned())),
gate: Some(Arc::new(FixedGate(Ok(GateVerdict::Invalid)))),
expected: ExecutionError::InvalidGateVerdict,
},
FatalCase {
name: "resolver",
registration: immediate("resolver", Ok("never".to_owned())),
gate: Some(Arc::new(FixedGate(Err(
GateOperationError::ResolverFailed("down".to_owned()),
)))),
expected: ExecutionError::Gate(GateOperationError::ResolverFailed("down".to_owned())),
},
FatalCase {
name: "gate-cancelled",
registration: immediate("gate-cancelled", Ok("never".to_owned())),
gate: Some(Arc::new(FixedGate(Err(GateOperationError::Cancelled)))),
expected: ExecutionError::Cancelled,
},
];
for case in cases {
let snapshot = snapshot_of(vec![case.registration]);
let entries = vec![
PreparedToolCall::settled(
ToolResult::success_text("settled".to_owned()),
SettledDisposition::ShortCircuited,
),
executable(&snapshot, case.name, case.gate),
];
let failure = with_timeout(execute_prepared(entries, &Token::new()))
.await
.err()
.unwrap();
assert_eq!(failure.origin(), ErrorOrigin::Slot(1));
assert_eq!(*failure.error(), case.expected);
assert_eq!(
failure.to_string(),
format!(
"prepared tool batch failed at source index 1: {}",
case.expected
)
);
let _: &dyn std::error::Error = &failure;
assert_eq!(failure.clone().into_error(), case.expected);
assert!(format!("{failure:?}").contains("Error"));
}
}
struct ObservingExecutor(Arc<Observations>);
#[derive(Default)]
struct Observations {
started: Notify,
dropped_with_cancelled_token: AtomicUsize,
dropped: AtomicUsize,
completed: AtomicUsize,
}
struct DropProbe {
observations: Arc<Observations>,
cancellation: Token,
}
impl Drop for DropProbe {
fn drop(&mut self) {
self.observations.dropped.fetch_add(1, Ordering::SeqCst);
if self.cancellation.is_cancelled() {
self.observations
.dropped_with_cancelled_token
.fetch_add(1, Ordering::SeqCst);
}
}
}
impl Executor for ObservingExecutor {
type Output = String;
async fn execute(
&self,
_invocation: &Invocation,
_call: &ToolCall,
cancellation: Token,
) -> Result<Self::Output, ExecutorError> {
let _probe = DropProbe {
observations: Arc::clone(&self.0),
cancellation: cancellation.clone(),
};
self.0.started.notify_one();
cancellation.cancelled().await;
self.0.completed.fetch_add(1, Ordering::SeqCst);
Ok("never materialized".to_owned())
}
}
fn observing(name: &str, observations: &Arc<Observations>) -> tool::Registration {
tool::Registration::client(
descriptor(name),
Requirements::new(Approval::NotRequired, Vec::new()),
Arc::new(ObservingExecutor(Arc::clone(observations))),
Arc::new(|output: String| Ok(ToolResult::success_text(output))),
)
}
#[tokio::test(flavor = "current_thread")]
async fn first_fatal_cancels_siblings_that_observe_it_and_all_runs_are_joined() {
let observations = Arc::new(Observations::default());
let snapshot = snapshot_of(vec![
tool::Registration::client(
descriptor("fatal"),
Requirements::new(Approval::NotRequired, Vec::new()),
Arc::new(FailingMapExecutor),
Arc::new(|_output: String| Err(MappingError::Protocol("unmappable".to_owned()))),
),
observing("sibling", &observations),
]);
let entries = vec![
executable(&snapshot, "fatal", None),
executable(&snapshot, "sibling", None),
];
let cancellation = Token::new();
let failure = with_timeout(execute_prepared(entries, &cancellation))
.await
.err()
.unwrap();
assert_eq!(
*failure.error(),
ExecutionError::Mapping(MappingError::Protocol("unmappable".to_owned()))
);
assert_eq!(failure.origin(), ErrorOrigin::Slot(0));
assert_eq!(observations.dropped.load(Ordering::SeqCst), 1);
assert_eq!(
observations
.dropped_with_cancelled_token
.load(Ordering::SeqCst),
1
);
assert_eq!(observations.completed.load(Ordering::SeqCst), 0);
assert!(!cancellation.is_cancelled());
}
#[tokio::test(flavor = "current_thread")]
async fn a_latched_fatal_still_waits_for_a_sibling_that_ignores_cancellation() {
let executor_calls = Arc::new(AtomicUsize::new(0));
let release = Arc::new(Notify::new());
let snapshot = snapshot_of(vec![
counting("stubborn", Arc::clone(&executor_calls)),
immediate("fatal", Ok("never".to_owned())),
]);
let stubborn: Arc<dyn Gate> = Arc::new(ReleasedGate {
release: Arc::clone(&release),
verdict: GateVerdict::Allow,
});
let fatal: Arc<dyn Gate> = Arc::new(FixedGate(Ok(GateVerdict::Invalid)));
let entries = vec![
executable(&snapshot, "stubborn", Some(stubborn)),
executable(&snapshot, "fatal", Some(fatal)),
];
let cancellation = Token::new();
let mut batch = Box::pin(execute_prepared(entries, &cancellation));
assert!(
tokio::time::timeout(Duration::from_millis(50), &mut batch)
.await
.is_err()
);
release.notify_one();
let failure = with_timeout(batch).await.err().unwrap();
assert_eq!(*failure.error(), ExecutionError::InvalidGateVerdict);
assert_eq!(failure.origin(), ErrorOrigin::Slot(1));
assert_eq!(executor_calls.load(Ordering::SeqCst), 0);
}
struct ReleasedGate {
release: Arc<Notify>,
verdict: GateVerdict,
}
impl Gate for ReleasedGate {
fn decide<'a>(
&'a self,
_call: &'a ToolCall,
_requirements: &'a Requirements,
_cancellation: Token,
) -> Pin<Box<dyn Future<Output = Result<GateVerdict, GateOperationError>> + Send + 'a>> {
Box::pin(async move {
self.release.notified().await;
Ok(self.verdict)
})
}
}
#[tokio::test(flavor = "current_thread")]
async fn latched_fatal_is_never_overwritten_by_later_or_racing_cancellation() {
for cancel_externally in [false, true] {
let observations = Arc::new(Observations::default());
let release = Arc::new(Notify::new());
let snapshot = snapshot_of(vec![
immediate("fatal", Ok("never".to_owned())),
observing("sibling-a", &observations),
observing("sibling-b", &observations),
]);
let gate: Arc<dyn Gate> = Arc::new(ReleasedGate {
release: Arc::clone(&release),
verdict: GateVerdict::Invalid,
});
let entries = vec![
executable(&snapshot, "sibling-a", None),
executable(&snapshot, "fatal", Some(gate)),
executable(&snapshot, "sibling-b", None),
];
let cancellation = Token::new();
let mut batch = Box::pin(execute_prepared(entries, &cancellation));
assert!(
tokio::time::timeout(Duration::from_millis(50), &mut batch)
.await
.is_err()
);
if cancel_externally {
cancellation.cancel();
}
release.notify_one();
let failure = with_timeout(batch).await.err().unwrap();
assert_eq!(*failure.error(), ExecutionError::InvalidGateVerdict);
assert_eq!(failure.origin(), ErrorOrigin::Slot(1));
assert_eq!(observations.dropped.load(Ordering::SeqCst), 2);
assert_eq!(
observations
.dropped_with_cancelled_token
.load(Ordering::SeqCst),
2
);
}
}
struct LateFatalGate;
impl Gate for LateFatalGate {
fn decide<'a>(
&'a self,
_call: &'a ToolCall,
_requirements: &'a Requirements,
cancellation: Token,
) -> Pin<Box<dyn Future<Output = Result<GateVerdict, GateOperationError>> + Send + 'a>> {
Box::pin(async move {
cancellation.cancelled().await;
Err(GateOperationError::ResolverFailed("late".to_owned()))
})
}
}
#[tokio::test(flavor = "current_thread")]
async fn a_real_fatal_displaces_an_already_latched_cancellation() {
let snapshot = snapshot_of(vec![
immediate("cancelled-first", Ok("never".to_owned())),
immediate("late-fatal", Ok("never".to_owned())),
]);
let cancelling: Arc<dyn Gate> = Arc::new(FixedGate(Err(GateOperationError::Cancelled)));
let entries = vec![
executable(&snapshot, "cancelled-first", Some(cancelling)),
executable(&snapshot, "late-fatal", Some(Arc::new(LateFatalGate))),
];
let failure = with_timeout(execute_prepared(entries, &Token::new()))
.await
.err()
.unwrap();
assert_eq!(
*failure.error(),
ExecutionError::Gate(GateOperationError::ResolverFailed("late".to_owned()))
);
assert_eq!(failure.origin(), ErrorOrigin::Slot(1));
}
struct PermanentlyPendingExecutor(Arc<Observations>);
impl Executor for PermanentlyPendingExecutor {
type Output = String;
async fn execute(
&self,
_invocation: &Invocation,
_call: &ToolCall,
_cancellation: Token,
) -> Result<Self::Output, ExecutorError> {
let _probe = DropProbe {
observations: Arc::clone(&self.0),
cancellation: Token::new(),
};
self.0.started.notify_one();
std::future::pending::<()>().await;
unreachable!("a permanently pending executor never completes")
}
}
#[tokio::test(flavor = "current_thread")]
async fn external_cancellation_drops_permanently_pending_runs_without_hanging() {
let observations = Arc::new(Observations::default());
let snapshot = snapshot_of(vec![
tool::Registration::client(
descriptor("pending-a"),
Requirements::new(Approval::NotRequired, Vec::new()),
Arc::new(PermanentlyPendingExecutor(Arc::clone(&observations))),
Arc::new(|output: String| Ok(ToolResult::success_text(output))),
),
tool::Registration::client(
descriptor("pending-b"),
Requirements::new(Approval::NotRequired, Vec::new()),
Arc::new(PermanentlyPendingExecutor(Arc::clone(&observations))),
Arc::new(|output: String| Ok(ToolResult::success_text(output))),
),
]);
let entries = vec![
PreparedToolCall::settled(
ToolResult::success_text("settled".to_owned()),
SettledDisposition::ShortCircuited,
),
executable(&snapshot, "pending-a", None),
executable(&snapshot, "pending-b", None),
];
let cancellation = Token::new();
let mut batch = Box::pin(execute_prepared(entries, &cancellation));
assert!(
tokio::time::timeout(Duration::from_millis(50), &mut batch)
.await
.is_err()
);
with_timeout(observations.started.notified()).await;
cancellation.cancel();
let failure = with_timeout(batch).await.err().unwrap();
assert_eq!(*failure.error(), ExecutionError::Cancelled);
assert_eq!(failure.origin(), ErrorOrigin::Slot(1));
assert_eq!(observations.dropped.load(Ordering::SeqCst), 2);
assert_eq!(observations.completed.load(Ordering::SeqCst), 0);
}
#[tokio::test]
async fn batches_without_executable_slots_return_reserved_positions_in_order() {
let cancellation = Token::new();
assert!(
with_timeout(execute_prepared(Vec::new(), &cancellation))
.await
.unwrap()
.is_empty()
);
let outcomes = with_timeout(execute_prepared(
vec![
PreparedToolCall::settled(
ToolResult::success_text("first".to_owned()),
SettledDisposition::Truncated,
),
PreparedToolCall::settled(
ToolResult::error_text("second".to_owned()),
SettledDisposition::NotExecuted,
),
],
&cancellation,
))
.await
.unwrap();
let cloned = outcomes[0].clone();
assert_eq!(text_of(cloned.result()), "first");
assert_eq!(cloned.disposition(), CallDisposition::Truncated);
assert_eq!(text_of(outcomes[1].result()), "second");
assert!(outcomes[1].result().is_business_error());
assert_eq!(outcomes[1].disposition(), CallDisposition::NotExecuted);
}
#[tokio::test]
async fn pre_cancelled_empty_and_fully_settled_batches_are_cancelled() {
let cancellation = Token::new();
cancellation.cancel();
let empty = execute_prepared(Vec::new(), &cancellation).await.err();
let empty = empty.unwrap();
assert_eq!(*empty.error(), ExecutionError::Cancelled);
assert_eq!(empty.origin(), ErrorOrigin::Batch);
assert_eq!(
empty.to_string(),
"prepared tool batch failed with no source position: tool run cancelled"
);
let settled = execute_prepared(
vec![
PreparedToolCall::settled(
ToolResult::success_text("first".to_owned()),
SettledDisposition::ShortCircuited,
),
PreparedToolCall::settled(ToolResult::no_output(), SettledDisposition::NotExecuted),
],
&cancellation,
)
.await
.err()
.unwrap();
assert_eq!(*settled.error(), ExecutionError::Cancelled);
assert_eq!(settled.origin(), ErrorOrigin::Batch);
}
struct SelfCancellingExecutor(Token);
impl Executor for SelfCancellingExecutor {
type Output = String;
fn execute(
&self,
_invocation: &Invocation,
_call: &ToolCall,
_cancellation: Token,
) -> impl Future<Output = Result<Self::Output, ExecutorError>> + Send {
self.0.cancel();
ready(Ok("outran cancellation".to_owned()))
}
}
#[tokio::test]
async fn cancellation_that_every_slot_outran_still_cancels_the_batch() {
let cancellation = Token::new();
let snapshot = snapshot_of(vec![tool::Registration::client(
descriptor("outrunner"),
Requirements::new(Approval::NotRequired, Vec::new()),
Arc::new(SelfCancellingExecutor(cancellation.clone())),
Arc::new(|output: String| Ok(ToolResult::success_text(output))),
)]);
let entries = vec![
PreparedToolCall::settled(
ToolResult::success_text("settled".to_owned()),
SettledDisposition::Truncated,
),
executable(&snapshot, "outrunner", None),
];
let failure = with_timeout(execute_prepared(entries, &cancellation))
.await
.err()
.unwrap();
assert_eq!(*failure.error(), ExecutionError::Cancelled);
assert_eq!(failure.origin(), ErrorOrigin::Batch);
}
#[tokio::test(flavor = "current_thread")]
async fn dropping_the_batch_mid_flight_cancels_then_drops_every_started_run() {
let observations = Arc::new(Observations::default());
let snapshot = snapshot_of(vec![
observing("mid-a", &observations),
observing("mid-b", &observations),
]);
let entries = vec![
executable(&snapshot, "mid-a", None),
PreparedToolCall::settled(ToolResult::no_output(), SettledDisposition::Blocked),
executable(&snapshot, "mid-b", None),
];
let cancellation = Token::new();
let mut batch = Box::pin(execute_prepared(entries, &cancellation));
assert!(
tokio::time::timeout(Duration::from_millis(50), &mut batch)
.await
.is_err()
);
with_timeout(observations.started.notified()).await;
assert_eq!(observations.dropped.load(Ordering::SeqCst), 0);
drop(batch);
assert_eq!(observations.dropped.load(Ordering::SeqCst), 2);
assert_eq!(
observations
.dropped_with_cancelled_token
.load(Ordering::SeqCst),
2
);
assert_eq!(observations.completed.load(Ordering::SeqCst), 0);
assert!(!cancellation.is_cancelled());
}
struct TriggeredFatalGate {
trigger: Token,
context: &'static str,
}
impl Gate for TriggeredFatalGate {
fn decide<'a>(
&'a self,
_call: &'a ToolCall,
_requirements: &'a Requirements,
_cancellation: Token,
) -> Pin<Box<dyn Future<Output = Result<GateVerdict, GateOperationError>> + Send + 'a>> {
Box::pin(async move {
self.trigger.cancelled().await;
Err(GateOperationError::ResolverFailed(self.context.to_owned()))
})
}
}
#[tokio::test(flavor = "current_thread")]
async fn two_real_fatals_in_one_poll_pass_report_the_lower_source_index() {
let trigger = Token::new();
let snapshot = snapshot_of(vec![
immediate("earlier", Ok("never".to_owned())),
immediate("later", Ok("never".to_owned())),
]);
let earlier: Arc<dyn Gate> = Arc::new(TriggeredFatalGate {
trigger: trigger.clone(),
context: "earlier",
});
let later: Arc<dyn Gate> = Arc::new(TriggeredFatalGate {
trigger: trigger.clone(),
context: "later",
});
let entries = vec![
executable(&snapshot, "earlier", Some(earlier)),
executable(&snapshot, "later", Some(later)),
];
let cancellation = Token::new();
let mut batch = Box::pin(execute_prepared(entries, &cancellation));
assert!(
tokio::time::timeout(Duration::from_millis(50), &mut batch)
.await
.is_err()
);
trigger.cancel();
let failure = with_timeout(batch).await.err().unwrap();
assert_eq!(
*failure.error(),
ExecutionError::Gate(GateOperationError::ResolverFailed("earlier".to_owned()))
);
assert_eq!(failure.origin(), ErrorOrigin::Slot(0));
}
#[test]
fn error_has_exactly_one_reachable_source_and_it_is_the_trait_method() {
let failure = super::tool_batch::Error::cancelled_batch();
assert_eq!(failure.origin(), ErrorOrigin::Batch);
use std::error::Error as _;
let retained = |cause: Option<&(dyn error::Error + 'static)>| {
cause
.expect("a batch failure retains its typed run failure")
.to_string()
};
let expected = ExecutionError::Cancelled.to_string();
assert_eq!(retained(error::Error::source(&failure)), expected);
assert_eq!(retained(failure.source()), expected);
let dynamic: &dyn error::Error = &failure;
assert_eq!(retained(dynamic.source()), expected);
assert!(
failure
.source()
.and_then(|cause| cause.downcast_ref::<ExecutionError>())
.is_some_and(|cause| *cause == ExecutionError::Cancelled)
);
}
struct Occupancy {
running: AtomicUsize,
peak: AtomicUsize,
peak_beside_exclusive: AtomicUsize,
}
impl Occupancy {
fn new() -> Arc<Self> {
Arc::new(Self {
running: AtomicUsize::new(0),
peak: AtomicUsize::new(0),
peak_beside_exclusive: AtomicUsize::new(0),
})
}
fn enter(&self, exclusive: bool) {
let running = self.running.fetch_add(1, Ordering::SeqCst) + 1;
self.peak.fetch_max(running, Ordering::SeqCst);
if exclusive {
self.peak_beside_exclusive
.fetch_max(running, Ordering::SeqCst);
}
}
fn leave(&self, exclusive: bool) {
if exclusive {
self.peak_beside_exclusive
.fetch_max(self.running.load(Ordering::SeqCst), Ordering::SeqCst);
}
self.running.fetch_sub(1, Ordering::SeqCst);
}
}
struct ExclusionExecutor {
lane: Arc<RwLock<()>>,
exclusive: bool,
label: &'static str,
occupancy: Arc<Occupancy>,
started: Arc<Notify>,
release: Arc<Notify>,
}
impl ExclusionExecutor {
async fn body(&self) -> Result<String, ExecutorError> {
self.occupancy.enter(self.exclusive);
self.started.notify_one();
self.release.notified().await;
self.occupancy.leave(self.exclusive);
Ok(self.label.to_owned())
}
}
impl Executor for ExclusionExecutor {
type Output = String;
async fn execute(
&self,
_invocation: &Invocation,
_call: &ToolCall,
_cancellation: Token,
) -> Result<Self::Output, ExecutorError> {
if self.exclusive {
let _lane = self.lane.write().await;
self.body().await
} else {
let _lane = self.lane.read().await;
self.body().await
}
}
}
struct ExclusionHandles {
started: Arc<Notify>,
release: Arc<Notify>,
}
fn exclusion_tool(
lane: &Arc<RwLock<()>>,
occupancy: &Arc<Occupancy>,
label: &'static str,
exclusive: bool,
) -> (tool::Registration, ExclusionHandles) {
let started = Arc::new(Notify::new());
let release = Arc::new(Notify::new());
let registration = tool::Registration::client(
descriptor(label),
Requirements::new(Approval::NotRequired, Vec::new()),
Arc::new(ExclusionExecutor {
lane: Arc::clone(lane),
exclusive,
label,
occupancy: Arc::clone(occupancy),
started: Arc::clone(&started),
release: Arc::clone(&release),
}),
Arc::new(|output: String| Ok(ToolResult::success_text(output))),
);
(registration, ExclusionHandles { started, release })
}
#[tokio::test(flavor = "current_thread")]
async fn a_composition_owned_lane_makes_one_tool_exclusive_without_reordering_the_batch() {
let lane = Arc::new(RwLock::new(()));
let occupancy = Occupancy::new();
let (reader_a, a) = exclusion_tool(&lane, &occupancy, "reader-a", false);
let (reader_b, b) = exclusion_tool(&lane, &occupancy, "reader-b", false);
let (writer, x) = exclusion_tool(&lane, &occupancy, "exclusive", true);
let (reader_c, c) = exclusion_tool(&lane, &occupancy, "reader-c", false);
let snapshot = snapshot_of(vec![reader_a, reader_b, writer, reader_c]);
let entries = vec![
executable(&snapshot, "reader-a", None),
executable(&snapshot, "reader-b", None),
executable(&snapshot, "exclusive", None),
executable(&snapshot, "reader-c", None),
];
let cancellation = Token::new();
let mut batch = Box::pin(execute_prepared(entries, &cancellation));
assert!(
tokio::time::timeout(Duration::from_millis(50), &mut batch)
.await
.is_err()
);
with_timeout(a.started.notified()).await;
with_timeout(b.started.notified()).await;
assert!(
occupancy.peak.load(Ordering::SeqCst) >= 2,
"the engine still overlaps the concurrency-safe peers"
);
for handles in [&a, &b, &x, &c] {
handles.release.notify_one();
}
let outcomes = with_timeout(batch).await.unwrap();
assert_eq!(
occupancy.peak_beside_exclusive.load(Ordering::SeqCst),
1,
"nothing else was inside its executor while the exclusive tool was"
);
assert_eq!(
outcomes
.iter()
.map(|outcome| text_of(outcome.result()))
.collect::<Vec<_>>(),
["reader-a", "reader-b", "exclusive", "reader-c"],
"serialising execution does not perturb the source order C14 commits"
);
}
#[test]
fn prepared_call_debug_says_whether_the_position_still_has_to_run() {
let settled = PreparedToolCall::settled(
ToolResult::success_text(SECRET.to_owned()),
SettledDisposition::ShortCircuited,
);
assert_redacted(&settled);
assert_eq!(
format!("{settled:?}"),
"PreparedToolCall { settled: true, disposition: ShortCircuited }"
);
let snapshot = snapshot_of(vec![immediate("search", Ok(SECRET.to_owned()))]);
let ungated = executable(&snapshot, "search", None);
assert_redacted(&ungated);
assert_states(
&ungated,
&[
"settled: false",
r#"Id("call-search")"#,
"arguments: <2 bytes>",
"gate: <absent>",
],
);
let gated = executable(
&snapshot,
"search",
Some(Arc::new(FixedGate(Ok(GateVerdict::Allow)))),
);
assert_states(&gated, &["gate: <present>"]);
}
#[tokio::test]
async fn batch_outcome_debug_keeps_the_disposition_and_redacts_the_result() {
let snapshot = snapshot_of(vec![immediate("search", Ok(SECRET.to_owned()))]);
let outcomes = execute_prepared(
vec![
executable(
&snapshot,
"search",
Some(Arc::new(FixedGate(Ok(GateVerdict::Allow)))),
),
PreparedToolCall::settled(
ToolResult::error_text(SECRET.to_owned()),
SettledDisposition::Truncated,
),
],
&Token::new(),
)
.await
.expect("an approved batch materializes in source order");
for outcome in &outcomes {
assert_redacted(outcome);
}
assert_states(&outcomes[0], &["disposition: Executed", "Success("]);
assert_states(
&outcomes[1],
&["disposition: Truncated", "user_view: Some(<21 bytes>)"],
);
}