use crate::mcp::budget::ToolBudget;
use crate::mcp::machine_codes;
use crate::mcp::registry;
use crate::mcp::response::{python_json_dumps, sanitize_error, truncate_response, ToolResponse};
use crate::mcp::runtime::{self, MAX_OUTPUT_BYTES, RUNTIME_METRICS};
use serde_json::Value;
use std::sync::atomic::{AtomicBool, Ordering};
#[allow(unused_imports)]
use std::sync::{Arc, Mutex};
use std::time::Duration;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum HandlerPhase {
Queued,
Running,
TimedOutQueued,
TimedOutRunning,
Finished,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum BeginRunning {
Run,
CancelledBeforeStart,
Error,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum TimeoutDisposition {
Queued,
Running,
AlreadyFinished,
}
struct HandlerLifecycle {
phase: Mutex<HandlerPhase>,
}
impl HandlerLifecycle {
fn new() -> Self {
Self {
phase: Mutex::new(HandlerPhase::Queued),
}
}
fn begin_running(&self, metrics: &runtime::RuntimeMetrics) -> BeginRunning {
let mut phase = self.phase.lock().unwrap_or_else(|e| e.into_inner());
match *phase {
HandlerPhase::Queued => {
let current = metrics
.active_blocking_handlers
.fetch_add(1, Ordering::Relaxed)
+ 1;
metrics
.peak_blocking_concurrency
.fetch_max(current, Ordering::Relaxed);
*phase = HandlerPhase::Running;
BeginRunning::Run
}
HandlerPhase::TimedOutQueued => BeginRunning::CancelledBeforeStart,
other => {
debug_assert!(
false,
"begin_running called in unexpected phase: {:?}",
other
);
if !cfg!(debug_assertions) {
eprintln!(
"warning: HandlerLifecycle::begin_running called in unexpected phase: {:?}",
other
);
}
BeginRunning::Error
}
}
}
fn record_timeout(&self, metrics: &runtime::RuntimeMetrics) -> TimeoutDisposition {
let mut phase = self.phase.lock().unwrap_or_else(|e| e.into_inner());
match *phase {
HandlerPhase::Queued => {
*phase = HandlerPhase::TimedOutQueued;
TimeoutDisposition::Queued
}
HandlerPhase::Running => {
metrics.timed_out_handlers.fetch_add(1, Ordering::Relaxed);
*phase = HandlerPhase::TimedOutRunning;
TimeoutDisposition::Running
}
HandlerPhase::TimedOutQueued
| HandlerPhase::TimedOutRunning
| HandlerPhase::Finished => TimeoutDisposition::AlreadyFinished,
}
}
fn finish(&self, metrics: &runtime::RuntimeMetrics) {
let mut phase = self.phase.lock().unwrap_or_else(|e| e.into_inner());
match *phase {
HandlerPhase::Running => {
*phase = HandlerPhase::Finished;
if metrics.active_blocking_handlers.load(Ordering::Relaxed) > 0 {
metrics
.active_blocking_handlers
.fetch_sub(1, Ordering::Relaxed);
} else {
eprintln!(
"warning: active_blocking_handlers underflow prevented in finish(Running)"
);
}
}
HandlerPhase::TimedOutRunning => {
if metrics.timed_out_handlers.load(Ordering::Relaxed) > 0 {
metrics.timed_out_handlers.fetch_sub(1, Ordering::Relaxed);
} else {
eprintln!("warning: timed_out_handlers underflow prevented in finish(TimedOutRunning)");
}
*phase = HandlerPhase::Finished;
if metrics.active_blocking_handlers.load(Ordering::Relaxed) > 0 {
metrics
.active_blocking_handlers
.fetch_sub(1, Ordering::Relaxed);
} else {
eprintln!("warning: active_blocking_handlers underflow prevented in finish(TimedOutRunning)");
}
}
HandlerPhase::TimedOutQueued => {
*phase = HandlerPhase::Finished;
}
HandlerPhase::Finished => {
debug_assert!(false, "double completion detected");
if !cfg!(debug_assertions) {
eprintln!("warning: HandlerLifecycle::finish double completion detected");
}
}
HandlerPhase::Queued => {
*phase = HandlerPhase::Finished;
}
}
}
}
#[cfg(test)]
#[allow(dead_code)]
#[derive(Clone)]
pub(crate) struct BlockingTestGate {
entered: Arc<tokio::sync::Notify>,
state: Arc<(std::sync::Mutex<bool>, std::sync::Condvar)>,
}
#[cfg(test)]
#[allow(dead_code)]
impl BlockingTestGate {
pub fn new() -> Self {
Self {
entered: Arc::new(tokio::sync::Notify::new()),
state: Arc::new((std::sync::Mutex::new(false), std::sync::Condvar::new())),
}
}
pub fn arrive_and_wait(&self) {
self.entered.notify_one();
let (lock, cv) = &*self.state;
let mut released = lock.lock().unwrap();
while !*released {
released = cv.wait(released).unwrap();
}
}
pub async fn wait_until_entered(&self) {
self.entered.notified().await;
}
pub fn release(&self) {
let (lock, cv) = &*self.state;
*lock.lock().unwrap() = true;
cv.notify_all();
}
}
#[cfg(test)]
impl Default for BlockingTestGate {
fn default() -> Self {
Self::new()
}
}
#[cfg(test)]
#[derive(Clone, Default)]
pub(crate) struct ClosureExitSignal {
exited: Arc<tokio::sync::Notify>,
}
#[cfg(test)]
#[allow(dead_code)]
impl ClosureExitSignal {
pub fn new() -> Self {
Self {
exited: Arc::new(tokio::sync::Notify::new()),
}
}
pub fn signal(&self) {
self.exited.notify_one();
}
pub async fn wait(&self) {
self.exited.notified().await;
}
}
#[cfg(test)]
#[allow(dead_code)]
#[derive(Clone, Default)]
pub(crate) struct AsyncTestGate {
entered: Arc<tokio::sync::Notify>,
release: Arc<tokio::sync::Notify>,
}
#[cfg(test)]
#[allow(dead_code)]
impl AsyncTestGate {
pub fn new() -> Self {
Self {
entered: Arc::new(tokio::sync::Notify::new()),
release: Arc::new(tokio::sync::Notify::new()),
}
}
pub async fn arrive_and_wait(&self) {
self.entered.notify_one();
self.release.notified().await;
}
pub async fn wait_until_entered(&self) {
self.entered.notified().await;
}
pub fn release(&self) {
self.release.notify_one();
}
}
#[derive(Clone)]
pub(crate) struct ExecutionHooks {
#[cfg(test)]
pub permit_acquired: Option<Arc<tokio::sync::Notify>>,
#[cfg(test)]
pub before_begin_running: Option<Arc<BlockingTestGate>>,
#[cfg(test)]
pub running_established: Option<Arc<BlockingTestGate>>,
#[cfg(test)]
pub before_handler: Option<Arc<BlockingTestGate>>,
#[cfg(test)]
pub before_timeout_record: Option<Arc<AsyncTestGate>>,
#[cfg(test)]
pub timeout_recorded: Option<Arc<AsyncTestGate>>,
#[cfg(test)]
pub before_finish: Option<Arc<BlockingTestGate>>,
#[cfg(test)]
pub finished: Option<Arc<BlockingTestGate>>,
#[cfg(test)]
pub closure_exited: Option<Arc<ClosureExitSignal>>,
}
impl ExecutionHooks {
#[cfg(test)]
pub fn none() -> Self {
Self {
permit_acquired: None,
before_begin_running: None,
running_established: None,
before_handler: None,
before_timeout_record: None,
timeout_recorded: None,
before_finish: None,
finished: None,
closure_exited: None,
}
}
#[cfg(not(test))]
pub fn none() -> Self {
Self {}
}
}
#[cfg(test)]
const TEST_BLOCK_SLOTS: usize = 16;
#[cfg(test)]
static BLOCK_SLOTS: [AtomicBool; TEST_BLOCK_SLOTS] =
[const { AtomicBool::new(false) }; TEST_BLOCK_SLOTS];
#[cfg(test)]
static RELEASE_SLOTS: [AtomicBool; TEST_BLOCK_SLOTS] =
[const { AtomicBool::new(false) }; TEST_BLOCK_SLOTS];
#[cfg(test)]
static SLOT_IN_USE: [AtomicBool; TEST_BLOCK_SLOTS] =
[const { AtomicBool::new(false) }; TEST_BLOCK_SLOTS];
#[cfg(test)]
struct TestSlotLease {
index: usize,
}
#[cfg(test)]
impl TestSlotLease {
fn index(&self) -> usize {
self.index
}
fn block(&self) {
BLOCK_SLOTS[self.index].store(true, Ordering::SeqCst);
RELEASE_SLOTS[self.index].store(false, Ordering::SeqCst);
}
fn release(&self) {
BLOCK_SLOTS[self.index].store(false, Ordering::SeqCst);
RELEASE_SLOTS[self.index].store(true, Ordering::SeqCst);
}
}
#[cfg(test)]
impl Drop for TestSlotLease {
fn drop(&mut self) {
BLOCK_SLOTS[self.index].store(false, Ordering::SeqCst);
RELEASE_SLOTS[self.index].store(false, Ordering::SeqCst);
SLOT_IN_USE[self.index].store(false, Ordering::Release);
}
}
#[cfg(test)]
fn acquire_test_slot() -> TestSlotLease {
for (index, slot_in_use) in SLOT_IN_USE.iter().enumerate() {
if slot_in_use
.compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
.is_ok()
{
return TestSlotLease { index };
}
}
panic!("no test blocking slots available");
}
#[cfg(test)]
fn test_handler_blocking_slot(args: &Value) -> ToolResponse {
let slot = args
.get("_block_slot")
.and_then(|v| v.as_u64())
.unwrap_or(0) as usize
% TEST_BLOCK_SLOTS;
while BLOCK_SLOTS[slot].load(Ordering::SeqCst) {
if RELEASE_SLOTS[slot].load(Ordering::SeqCst) {
break;
}
std::hint::spin_loop();
}
ToolResponse::success(serde_json::json!("ok"), None)
}
#[cfg(test)]
fn block_slot_args(slot: &TestSlotLease) -> Value {
serde_json::json!({ "_block_slot": slot.index() })
}
#[cfg(test)]
fn test_handler_fast(_args: &Value) -> ToolResponse {
ToolResponse::success(serde_json::json!("ok"), None)
}
pub(crate) struct ExecutionOutcome {
pub tool_response: Result<ToolResponse, tokio::task::JoinError>,
pub timed_out: bool,
}
pub(crate) async fn execute_tool_bounded(
handler: registry::ToolHandler,
args: Value,
tool_name: String,
budget: ToolBudget,
cancel_flag: std::sync::Arc<AtomicBool>,
semaphore: std::sync::Arc<tokio::sync::Semaphore>,
) -> ExecutionOutcome {
execute_tool_bounded_inner(
handler,
args,
tool_name,
budget,
cancel_flag,
semaphore,
ExecutionHooks::none(),
&RUNTIME_METRICS,
)
.await
}
#[cfg(not(debug_assertions))]
fn ensure_calculator_warmed_detached() {
static CALC_WARMUP_STARTED: AtomicBool = AtomicBool::new(false);
if CALC_WARMUP_STARTED.swap(true, Ordering::AcqRel) {
return;
}
let spawned = std::thread::Builder::new()
.name("calc-regex-warmup".into())
.spawn(crate::calc::normalize::warm_calculator_regex_cache);
if let Err(err) = spawned {
CALC_WARMUP_STARTED.store(false, Ordering::Release);
crate::calc::normalize::warm_calculator_regex_cache();
let _ = err;
}
}
#[allow(clippy::too_many_arguments)]
async fn execute_tool_bounded_inner(
handler: registry::ToolHandler,
args: Value,
tool_name: String,
budget: ToolBudget,
cancel_flag: std::sync::Arc<AtomicBool>,
semaphore: std::sync::Arc<tokio::sync::Semaphore>,
_hooks: ExecutionHooks,
metrics: &'static runtime::RuntimeMetrics,
) -> ExecutionOutcome {
#[cfg(not(debug_assertions))]
ensure_calculator_warmed_detached();
let timeout_ms = budget.max_elapsed_ms;
let tool_name_for_timeout = tool_name.clone();
let lifecycle = std::sync::Arc::new(HandlerLifecycle::new());
let lifecycle_for_timeout = lifecycle.clone();
let cancel_flag_for_timeout = cancel_flag.clone();
let result = tokio::time::timeout(Duration::from_millis(timeout_ms), async move {
let permit = match semaphore.clone().acquire_owned().await {
Ok(p) => p,
Err(_) => {
return Ok::<_, tokio::task::JoinError>(ToolResponse::error_with_code(
"internal_error",
machine_codes::INTERNAL_ERROR,
"Tool execution semaphore unavailable (server shutting down)",
None,
None,
));
}
};
#[cfg(test)]
if let Some(ref notify) = _hooks.permit_acquired {
notify.notify_one();
}
let lifecycle_block = lifecycle.clone();
let cancel_flag_block = cancel_flag.clone();
let tool_name_block = tool_name.clone();
#[cfg(test)]
let hooks_for_block = ExecutionHooks {
permit_acquired: None,
before_begin_running: _hooks.before_begin_running.clone(),
running_established: _hooks.running_established.clone(),
before_handler: _hooks.before_handler.clone(),
before_timeout_record: None,
timeout_recorded: None,
before_finish: _hooks.before_finish.clone(),
finished: _hooks.finished.clone(),
closure_exited: _hooks.closure_exited.clone(),
};
tokio::task::spawn_blocking(move || {
#[cfg(test)]
struct ClosureExitGuard(Option<Arc<ClosureExitSignal>>);
#[cfg(test)]
impl Drop for ClosureExitGuard {
fn drop(&mut self) {
if let Some(signal) = self.0.take() {
signal.signal();
}
}
}
#[cfg(test)]
let _closure_exit = ClosureExitGuard(hooks_for_block.closure_exited.clone());
let _permit = permit;
#[cfg(test)]
if let Some(ref gate) = hooks_for_block.before_begin_running {
gate.arrive_and_wait();
}
match lifecycle_block.begin_running(metrics) {
BeginRunning::CancelledBeforeStart => {
return ToolResponse::error_with_code(
"cancelled",
machine_codes::CANCELLED,
&format!(
"Tool '{}' request was cancelled (timed out while queued)",
tool_name_block
),
Some(vec![
"The request was cancelled before execution started".to_string()
]),
Some(&tool_name_block),
);
}
BeginRunning::Run => {}
BeginRunning::Error => {
lifecycle_block.finish(metrics);
return ToolResponse::error_with_code(
"internal_error",
machine_codes::INTERNAL_ERROR,
&format!(
"Tool '{}' lifecycle error: begin_running called in unexpected phase",
tool_name_block
),
None,
Some(&tool_name_block),
);
}
}
#[cfg(test)]
if let Some(ref gate) = hooks_for_block.running_established {
gate.arrive_and_wait();
}
if cancel_flag_block.load(Ordering::Acquire) {
lifecycle_block.finish(metrics);
return ToolResponse::error_with_code(
"cancelled",
machine_codes::CANCELLED,
&format!("Tool '{}' request was cancelled", tool_name_block),
None,
Some(&tool_name_block),
);
}
#[cfg(test)]
if let Some(ref gate) = hooks_for_block.before_handler {
gate.arrive_and_wait();
}
let mut mcp_eval_ctx = crate::calc::EvalContext::mcp_mode();
let cancel_flag_handler = cancel_flag_block.clone();
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
crate::mcp::budget::with_cancel_flag(Some(cancel_flag_handler), || {
crate::mcp::budget::with_eval_context(&mut mcp_eval_ctx, || handler(&args))
})
}));
#[cfg(test)]
if let Some(ref gate) = hooks_for_block.before_finish {
gate.arrive_and_wait();
}
lifecycle_block.finish(metrics);
#[cfg(test)]
if let Some(ref gate) = hooks_for_block.finished {
gate.arrive_and_wait();
}
match result {
Ok(response) => response,
Err(payload) => {
let msg = payload
.downcast_ref::<&str>()
.map(|s| s.to_string())
.or_else(|| payload.downcast_ref::<String>().cloned())
.unwrap_or_else(|| "handler panicked".to_string());
ToolResponse::error_with_code(
"internal_error",
crate::mcp::machine_codes::INTERNAL_ERROR,
&format!("Tool handler panicked: {}", msg),
None,
None,
)
}
}
})
.await
})
.await;
match result {
Ok(Ok(tool_response)) => ExecutionOutcome {
tool_response: Ok(tool_response),
timed_out: false,
},
Ok(Err(join_err)) => ExecutionOutcome {
tool_response: Err(join_err),
timed_out: false,
},
Err(_timeout) => {
cancel_flag_for_timeout.store(true, Ordering::Release);
metrics.total_timeouts.fetch_add(1, Ordering::Relaxed);
#[cfg(test)]
if let Some(ref gate) = _hooks.before_timeout_record {
gate.arrive_and_wait().await;
}
match lifecycle_for_timeout.record_timeout(metrics) {
TimeoutDisposition::Queued | TimeoutDisposition::Running => {}
TimeoutDisposition::AlreadyFinished => {}
}
#[cfg(test)]
if let Some(ref gate) = _hooks.timeout_recorded {
gate.arrive_and_wait().await;
}
ExecutionOutcome {
tool_response: Ok(ToolResponse::error_with_code(
"timeout",
machine_codes::TIMEOUT,
&format!(
"Tool '{}' execution timed out after {}s (budget: {}ms max). The cancel flag was set cooperatively; the handler may continue briefly.",
tool_name_for_timeout,
timeout_ms / 1000,
timeout_ms
),
Some(vec![
"Try a simpler input or shorter text".to_string(),
"The tool handler checks cancellation cooperatively and may not stop immediately".to_string(),
]),
Some(&tool_name_for_timeout),
)),
timed_out: true,
}
}
}
}
#[cfg(test)]
#[allow(clippy::too_many_arguments)]
pub(crate) async fn execute_tool_bounded_with_hooks(
handler: registry::ToolHandler,
args: Value,
tool_name: String,
budget: ToolBudget,
cancel_flag: std::sync::Arc<AtomicBool>,
semaphore: std::sync::Arc<tokio::sync::Semaphore>,
hooks: ExecutionHooks,
metrics: std::sync::Arc<runtime::RuntimeMetrics>,
) -> ExecutionOutcome {
let metrics_ptr: *const runtime::RuntimeMetrics = std::sync::Arc::into_raw(metrics);
let metrics_static: &'static runtime::RuntimeMetrics = unsafe { &*metrics_ptr };
execute_tool_bounded_inner(
handler,
args,
tool_name,
budget,
cancel_flag,
semaphore,
hooks,
metrics_static,
)
.await
}
pub(crate) fn build_tool_response(
outcome: ExecutionOutcome,
tool_name: &str,
budget: &ToolBudget,
id: Option<Value>,
) -> serde_json::Value {
match outcome.tool_response {
Ok(mut response) => {
if outcome.timed_out {
return crate::mcp::response::wrap_tool_response(&response);
}
truncate_response(&mut response, budget);
let output = python_json_dumps(&response);
if output.is_empty() {
crate::mcp::response::wrap_tool_response(&ToolResponse::error_with_code(
"serialization_error",
machine_codes::SERIALIZATION_ERROR,
"Failed to serialize tool response",
None,
Some(tool_name),
))
} else if output.len() > MAX_OUTPUT_BYTES {
crate::mcp::response::wrap_tool_response(&ToolResponse::error_with_code(
"output_too_large",
machine_codes::OUTPUT_TOO_LARGE,
&format!(
"Output exceeds {} bytes and was truncated",
MAX_OUTPUT_BYTES
),
Some(vec![
"Try reducing input size or using a summary/detail option".to_string(),
]),
Some(tool_name),
))
} else {
crate::mcp::response::wrap_tool_response_with_text(&response, output)
}
}
Err(join_err) => crate::mcp::protocol::json_rpc_error(
-32000,
format!(
"Tool execution error: {}",
runtime::truncate_2000(&sanitize_error(&join_err.to_string()))
),
id,
),
}
}
pub(crate) fn build_tool_response_modern(
outcome: ExecutionOutcome,
tool_name: &str,
budget: &ToolBudget,
id: Option<Value>,
) -> serde_json::Value {
use crate::mcp::response::wrap_tool_response_modern;
match outcome.tool_response {
Ok(mut response) => {
if outcome.timed_out {
return wrap_tool_response_modern(&response);
}
truncate_response(&mut response, budget);
let output = python_json_dumps(&response);
if output.is_empty() {
wrap_tool_response_modern(&ToolResponse::error_with_code(
"serialization_error",
machine_codes::SERIALIZATION_ERROR,
"Failed to serialize tool response",
None,
Some(tool_name),
))
} else if output.len() > MAX_OUTPUT_BYTES {
wrap_tool_response_modern(&ToolResponse::error_with_code(
"output_too_large",
machine_codes::OUTPUT_TOO_LARGE,
&format!(
"Output exceeds {} bytes and was truncated",
MAX_OUTPUT_BYTES
),
Some(vec![
"Try reducing input size or using a summary/detail option".to_string(),
]),
Some(tool_name),
))
} else {
crate::mcp::response::wrap_tool_response_modern_with_text(&response, output)
}
}
Err(join_err) => crate::mcp::protocol::json_rpc_error(
-32000,
format!(
"Tool execution error: {}",
runtime::truncate_2000(&sanitize_error(&join_err.to_string()))
),
id,
),
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::mcp::runtime::MetricsSnapshot;
use std::sync::Arc;
fn assert_snapshot_invariant(snap: &MetricsSnapshot) {
assert!(
snap.timed_out_handlers <= snap.active_blocking_handlers,
"INVARIANT VIOLATION: timed_out_handlers ({}) > active_blocking_handlers ({})",
snap.timed_out_handlers,
snap.active_blocking_handlers,
);
}
fn snapshot_from_metrics(m: &runtime::RuntimeMetrics) -> MetricsSnapshot {
MetricsSnapshot {
active_requests: m.active_requests.load(Ordering::Relaxed),
active_blocking_handlers: m.active_blocking_handlers.load(Ordering::Relaxed),
timed_out_handlers: m.timed_out_handlers.load(Ordering::Relaxed),
total_timeouts: m.total_timeouts.load(Ordering::Relaxed),
peak_blocking_concurrency: m.peak_blocking_concurrency.load(Ordering::Relaxed),
}
}
fn new_test_metrics() -> Arc<runtime::RuntimeMetrics> {
Arc::new(runtime::RuntimeMetrics::new_for_test())
}
#[test]
fn slot_leases_are_exclusive_under_parallel_allocation() {
let workers = 8;
let barrier = Arc::new(std::sync::Barrier::new(workers));
let claimed = Arc::new([const { AtomicBool::new(false) }; TEST_BLOCK_SLOTS]);
let mut handles = Vec::new();
for _ in 0..workers {
let barrier = barrier.clone();
let claimed = claimed.clone();
handles.push(std::thread::spawn(move || {
barrier.wait();
for _ in 0..100 {
let lease = acquire_test_slot();
let index = lease.index();
assert!(
claimed[index]
.compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
.is_ok(),
"slot {} was allocated to two parallel test invocations",
index
);
std::thread::yield_now();
assert!(claimed[index].swap(false, Ordering::Release));
drop(lease);
}
}));
}
for handle in handles {
handle
.join()
.expect("slot lease stress thread must not panic");
}
}
static TEST1_HANDLER_RAN: AtomicBool = AtomicBool::new(false);
fn test1_handler(_args: &Value) -> ToolResponse {
TEST1_HANDLER_RAN.store(true, Ordering::SeqCst);
ToolResponse::success(serde_json::json!("done"), None)
}
#[tokio::test]
async fn queued_timeout_smoke_does_not_run_handler() {
let metrics = new_test_metrics();
let semaphore = Arc::new(tokio::sync::Semaphore::new(0));
let cancel_flag = Arc::new(AtomicBool::new(false));
let budget = ToolBudget::CHEAP.with_max_elapsed_ms(10);
TEST1_HANDLER_RAN.store(false, Ordering::SeqCst);
let outcome = execute_tool_bounded_with_hooks(
test1_handler as registry::ToolHandler,
Value::Object(serde_json::Map::new()),
"test_tool".to_string(),
budget,
cancel_flag.clone(),
semaphore.clone(),
ExecutionHooks::none(),
metrics.clone(),
)
.await;
assert!(outcome.timed_out);
assert!(!TEST1_HANDLER_RAN.load(Ordering::SeqCst));
semaphore.add_permits(1);
tokio::time::sleep(Duration::from_millis(50)).await;
assert!(
!TEST1_HANDLER_RAN.load(Ordering::SeqCst),
"handler must not run after queued timeout"
);
assert_snapshot_invariant(&snapshot_from_metrics(&metrics));
}
#[tokio::test]
async fn timeout_smoke_returns_while_handler_continues() {
let metrics = new_test_metrics();
let semaphore = Arc::new(tokio::sync::Semaphore::new(1));
let cancel_flag = Arc::new(AtomicBool::new(false));
let budget = ToolBudget::CHEAP.with_max_elapsed_ms(10);
let outcome = execute_tool_bounded_with_hooks(
|_args| {
std::thread::sleep(Duration::from_millis(200));
ToolResponse::success(serde_json::json!("ok"), None)
},
Value::Object(serde_json::Map::new()),
"test_tool".to_string(),
budget,
cancel_flag.clone(),
semaphore.clone(),
ExecutionHooks::none(),
metrics.clone(),
)
.await;
assert!(outcome.timed_out);
tokio::time::sleep(Duration::from_millis(250)).await;
assert_snapshot_invariant(&snapshot_from_metrics(&metrics));
}
#[tokio::test]
async fn running_timeout_smoke_increments_once() {
let metrics = new_test_metrics();
let semaphore = Arc::new(tokio::sync::Semaphore::new(1));
let cancel_flag = Arc::new(AtomicBool::new(false));
let budget = ToolBudget::CHEAP.with_max_elapsed_ms(10);
let finished = Arc::new(BlockingTestGate::new());
let outcome = execute_tool_bounded_with_hooks(
|_args| {
std::thread::sleep(Duration::from_millis(200));
ToolResponse::success(serde_json::json!("ok"), None)
},
Value::Object(serde_json::Map::new()),
"test_tool".to_string(),
budget,
cancel_flag.clone(),
semaphore.clone(),
ExecutionHooks {
finished: Some(finished.clone()),
..ExecutionHooks::none()
},
metrics.clone(),
)
.await;
assert!(outcome.timed_out);
assert_eq!(
metrics.timed_out_handlers.load(Ordering::Relaxed),
1,
"timed_out_handlers should be exactly 1 while handler is still running"
);
finished.wait_until_entered().await;
assert_eq!(
metrics.timed_out_handlers.load(Ordering::Relaxed),
0,
"timed_out_handlers must return to 0 after handler finishes"
);
assert_snapshot_invariant(&snapshot_from_metrics(&metrics));
finished.release();
}
#[tokio::test]
async fn no_double_completion_smoke() {
let metrics = new_test_metrics();
let semaphore = Arc::new(tokio::sync::Semaphore::new(1));
let cancel_flag = Arc::new(AtomicBool::new(false));
let budget = ToolBudget::CHEAP;
let outcome = execute_tool_bounded_with_hooks(
test_handler_fast as registry::ToolHandler,
Value::Object(serde_json::Map::new()),
"test_tool".to_string(),
budget,
cancel_flag.clone(),
semaphore.clone(),
ExecutionHooks::none(),
metrics.clone(),
)
.await;
assert!(!outcome.timed_out);
assert!(outcome.tool_response.is_ok());
tokio::time::sleep(Duration::from_millis(10)).await;
assert_snapshot_invariant(&snapshot_from_metrics(&metrics));
}
#[tokio::test]
async fn basic_execution_completes_successfully() {
let metrics = new_test_metrics();
let semaphore = Arc::new(tokio::sync::Semaphore::new(1));
let cancel_flag = Arc::new(AtomicBool::new(false));
let tool_budget = ToolBudget::CHEAP;
let closure_exited = Arc::new(ClosureExitSignal::new());
let outcome = execute_tool_bounded_with_hooks(
|_args| ToolResponse::success(serde_json::json!("hello"), None),
Value::Object(serde_json::Map::new()),
"test_tool".to_string(),
tool_budget,
cancel_flag.clone(),
semaphore.clone(),
ExecutionHooks {
closure_exited: Some(closure_exited.clone()),
..ExecutionHooks::none()
},
metrics.clone(),
)
.await;
assert!(!outcome.timed_out);
let resp = outcome.tool_response.unwrap();
assert!(resp.error.is_none());
assert_eq!(resp.result.as_ref().unwrap().as_str().unwrap(), "hello");
assert!(
tokio::time::timeout(Duration::from_secs(5), closure_exited.wait())
.await
.is_ok(),
"normal blocking closure must signal exit"
);
assert_snapshot_invariant(&snapshot_from_metrics(&metrics));
}
#[tokio::test]
async fn panic_cleanup_smoke() {
let metrics = new_test_metrics();
let semaphore = Arc::new(tokio::sync::Semaphore::new(1));
let cancel_flag = Arc::new(AtomicBool::new(false));
let budget = ToolBudget::CHEAP;
let closure_exited = Arc::new(ClosureExitSignal::new());
fn always_panic_handler(_args: &Value) -> ToolResponse {
panic!("intentional test panic");
}
let outcome = execute_tool_bounded_with_hooks(
always_panic_handler as registry::ToolHandler,
Value::Object(serde_json::Map::new()),
"panic_tool".to_string(),
budget,
cancel_flag,
semaphore,
ExecutionHooks {
closure_exited: Some(closure_exited.clone()),
..ExecutionHooks::none()
},
metrics.clone(),
)
.await;
assert!(!outcome.timed_out);
let resp = outcome.tool_response.unwrap();
assert!(!resp.ok, "panicked handler should return ok=false response");
assert_eq!(
resp.machine_code.as_deref(),
Some(machine_codes::INTERNAL_ERROR)
);
assert!(
tokio::time::timeout(Duration::from_secs(5), closure_exited.wait())
.await
.is_ok(),
"caught-panic blocking closure must signal exit"
);
assert_snapshot_invariant(&snapshot_from_metrics(&metrics));
}
#[tokio::test]
async fn timeout_smoke_handler_continues_after_return() {
let metrics = new_test_metrics();
let semaphore = Arc::new(tokio::sync::Semaphore::new(1));
let cancel_flag = Arc::new(AtomicBool::new(false));
let budget = ToolBudget::CHEAP.with_max_elapsed_ms(10);
let outcome = execute_tool_bounded_with_hooks(
|_args| {
std::thread::sleep(Duration::from_millis(100));
ToolResponse::success(serde_json::json!("late"), None)
},
Value::Object(serde_json::Map::new()),
"slow_tool".to_string(),
budget,
cancel_flag,
semaphore,
ExecutionHooks::none(),
metrics.clone(),
)
.await;
assert!(outcome.timed_out);
let resp = outcome.tool_response.unwrap();
assert!(!resp.ok);
tokio::time::sleep(Duration::from_millis(200)).await;
assert_snapshot_invariant(&snapshot_from_metrics(&metrics));
}
}
#[cfg(test)]
mod deterministic_tests {
use super::*;
use crate::mcp::runtime::MetricsSnapshot;
use std::sync::Arc;
fn assert_snapshot_invariant(snap: &MetricsSnapshot) {
assert!(
snap.timed_out_handlers <= snap.active_blocking_handlers,
"INVARIANT VIOLATION: timed_out_handlers ({}) > active_blocking_handlers ({})",
snap.timed_out_handlers,
snap.active_blocking_handlers,
);
}
fn snapshot_from_metrics(m: &runtime::RuntimeMetrics) -> MetricsSnapshot {
MetricsSnapshot {
active_requests: m.active_requests.load(Ordering::Relaxed),
active_blocking_handlers: m.active_blocking_handlers.load(Ordering::Relaxed),
timed_out_handlers: m.timed_out_handlers.load(Ordering::Relaxed),
total_timeouts: m.total_timeouts.load(Ordering::Relaxed),
peak_blocking_concurrency: m.peak_blocking_concurrency.load(Ordering::Relaxed),
}
}
fn new_test_metrics() -> Arc<runtime::RuntimeMetrics> {
Arc::new(runtime::RuntimeMetrics::new_for_test())
}
fn blocking_gate() -> Arc<BlockingTestGate> {
Arc::new(BlockingTestGate::new())
}
fn async_gate() -> Arc<AsyncTestGate> {
Arc::new(AsyncTestGate::new())
}
#[tokio::test]
async fn timeout_after_permit_before_lifecycle_start() {
static HANDLER_RAN: AtomicBool = AtomicBool::new(false);
HANDLER_RAN.store(false, Ordering::SeqCst);
let metrics = new_test_metrics();
let semaphore = Arc::new(tokio::sync::Semaphore::new(1));
let cancel_flag = Arc::new(AtomicBool::new(false));
let budget = ToolBudget::CHEAP.with_max_elapsed_ms(50);
let before_begin_running = blocking_gate();
let closure_exited = Arc::new(ClosureExitSignal::new());
let hooks = ExecutionHooks {
before_begin_running: Some(before_begin_running.clone()),
closure_exited: Some(closure_exited.clone()),
..ExecutionHooks::none()
};
let call = tokio::spawn(execute_tool_bounded_with_hooks(
|_args| {
HANDLER_RAN.store(true, Ordering::SeqCst);
ToolResponse::success(serde_json::json!("done"), None)
},
Value::Object(serde_json::Map::new()),
"test_tool".to_string(),
budget,
cancel_flag,
semaphore,
hooks,
metrics.clone(),
));
before_begin_running.wait_until_entered().await;
let outcome = call.await.unwrap();
assert!(outcome.timed_out, "must time out while closure is gated");
assert_eq!(
metrics.total_timeouts.load(Ordering::Relaxed),
1,
"total_timeouts must be exactly 1"
);
assert_eq!(
metrics.active_blocking_handlers.load(Ordering::Relaxed),
0,
"active_blocking_handlers must be 0 while closure is gated at before_begin_running"
);
assert_eq!(
metrics.timed_out_handlers.load(Ordering::Relaxed),
0,
"timed_out_handlers must be 0 — queued timeout does not increment it"
);
before_begin_running.release();
assert!(
!HANDLER_RAN.load(Ordering::SeqCst),
"handler must not run after queued timeout"
);
assert!(
tokio::time::timeout(Duration::from_secs(5), closure_exited.wait())
.await
.is_ok(),
"blocking closure must signal exit after queued timeout"
);
assert!(!HANDLER_RAN.load(Ordering::SeqCst));
let snap = snapshot_from_metrics(&metrics);
assert_eq!(snap.active_blocking_handlers, 0);
assert_eq!(snap.timed_out_handlers, 0);
assert_eq!(snap.total_timeouts, 1);
assert_snapshot_invariant(&snap);
}
#[tokio::test]
async fn completion_wins_timeout_record_race() {
let metrics = new_test_metrics();
let semaphore = Arc::new(tokio::sync::Semaphore::new(1));
let cancel_flag = Arc::new(AtomicBool::new(false));
let budget = ToolBudget::CHEAP.with_max_elapsed_ms(100);
let before_finish = blocking_gate();
let finished = blocking_gate();
let before_timeout_record = async_gate();
let timeout_recorded = async_gate();
let hooks = ExecutionHooks {
before_finish: Some(before_finish.clone()),
finished: Some(finished.clone()),
before_timeout_record: Some(before_timeout_record.clone()),
timeout_recorded: Some(timeout_recorded.clone()),
..ExecutionHooks::none()
};
let call = tokio::spawn(execute_tool_bounded_with_hooks(
test_handler_fast as registry::ToolHandler,
Value::Object(serde_json::Map::new()),
"test_tool".to_string(),
budget,
cancel_flag,
semaphore,
hooks,
metrics.clone(),
));
before_finish.wait_until_entered().await;
before_timeout_record.wait_until_entered().await;
before_finish.release();
finished.wait_until_entered().await;
finished.release();
before_timeout_record.release();
timeout_recorded.wait_until_entered().await;
timeout_recorded.release();
let outcome = call.await.unwrap();
assert!(outcome.timed_out, "caller must receive timeout");
assert_eq!(
metrics.timed_out_handlers.load(Ordering::Relaxed),
0,
"completion wins: timed_out_handlers must not be incremented"
);
let snap = snapshot_from_metrics(&metrics);
assert_eq!(snap.active_blocking_handlers, 0);
assert_eq!(snap.timed_out_handlers, 0);
assert_eq!(snap.total_timeouts, 1);
assert_snapshot_invariant(&snap);
}
#[tokio::test]
async fn timeout_wins_completion_race() {
let metrics = new_test_metrics();
let semaphore = Arc::new(tokio::sync::Semaphore::new(1));
let cancel_flag = Arc::new(AtomicBool::new(false));
let budget = ToolBudget::CHEAP.with_max_elapsed_ms(100);
let before_finish = blocking_gate();
let finished = blocking_gate();
let before_timeout_record = async_gate();
let timeout_recorded = async_gate();
let hooks = ExecutionHooks {
before_finish: Some(before_finish.clone()),
finished: Some(finished.clone()),
before_timeout_record: Some(before_timeout_record.clone()),
timeout_recorded: Some(timeout_recorded.clone()),
..ExecutionHooks::none()
};
let call = tokio::spawn(execute_tool_bounded_with_hooks(
test_handler_fast as registry::ToolHandler,
Value::Object(serde_json::Map::new()),
"test_tool".to_string(),
budget,
cancel_flag,
semaphore,
hooks,
metrics.clone(),
));
before_finish.wait_until_entered().await;
before_timeout_record.wait_until_entered().await;
before_timeout_record.release();
timeout_recorded.wait_until_entered().await;
timeout_recorded.release();
assert_eq!(
metrics.active_blocking_handlers.load(Ordering::Relaxed),
1,
"active_blocking_handlers must be 1 while handler is still running"
);
assert_eq!(
metrics.timed_out_handlers.load(Ordering::Relaxed),
1,
"timed_out_handlers must be exactly 1"
);
before_finish.release();
finished.wait_until_entered().await;
finished.release();
let snap = snapshot_from_metrics(&metrics);
assert_eq!(snap.active_blocking_handlers, 0);
assert_eq!(snap.timed_out_handlers, 0);
assert_snapshot_invariant(&snap);
let outcome = call.await.unwrap();
assert!(outcome.timed_out, "caller must receive timeout");
}
#[tokio::test]
async fn panic_after_timeout() {
let metrics = new_test_metrics();
let semaphore = Arc::new(tokio::sync::Semaphore::new(1));
let cancel_flag = Arc::new(AtomicBool::new(false));
let budget = ToolBudget::CHEAP.with_max_elapsed_ms(100);
let before_handler = blocking_gate();
let before_finish = blocking_gate();
let finished = blocking_gate();
let before_timeout_record = async_gate();
let timeout_recorded = async_gate();
let closure_exited = Arc::new(ClosureExitSignal::new());
let hooks = ExecutionHooks {
before_handler: Some(before_handler.clone()),
before_finish: Some(before_finish.clone()),
finished: Some(finished.clone()),
before_timeout_record: Some(before_timeout_record.clone()),
timeout_recorded: Some(timeout_recorded.clone()),
closure_exited: Some(closure_exited.clone()),
..ExecutionHooks::none()
};
let call = tokio::spawn(execute_tool_bounded_with_hooks(
|_args| panic!("intentional test panic"),
Value::Object(serde_json::Map::new()),
"test_tool".to_string(),
budget,
cancel_flag,
semaphore,
hooks,
metrics.clone(),
));
before_handler.wait_until_entered().await;
before_timeout_record.wait_until_entered().await;
assert_eq!(
metrics.active_blocking_handlers.load(Ordering::Relaxed),
1,
"active_blocking_handlers must be 1 while handler is still running"
);
assert_eq!(
metrics.timed_out_handlers.load(Ordering::Relaxed),
0,
"timed_out_handlers must be 0 before record_timeout is released"
);
before_timeout_record.release();
timeout_recorded.wait_until_entered().await;
timeout_recorded.release();
assert_eq!(
metrics.timed_out_handlers.load(Ordering::Relaxed),
1,
"timed_out_handlers must be 1 after timeout recording"
);
before_handler.release();
before_finish.release();
finished.wait_until_entered().await;
finished.release();
assert!(
tokio::time::timeout(Duration::from_secs(5), closure_exited.wait())
.await
.is_ok(),
"panicking blocking closure must signal exit"
);
let snap = snapshot_from_metrics(&metrics);
assert_eq!(snap.active_blocking_handlers, 0);
assert_eq!(snap.timed_out_handlers, 0);
assert_snapshot_invariant(&snap);
let outcome = call.await.unwrap();
assert!(outcome.timed_out, "caller must receive timeout");
}
#[tokio::test]
async fn cooperative_cancellation_visibility() {
static COOPERATIVE_CANCEL_OBSERVED: AtomicBool = AtomicBool::new(false);
fn cancel_polling_handler(_args: &Value) -> ToolResponse {
let give_up_at = std::time::Instant::now() + Duration::from_secs(30);
while let Some(flag) = crate::mcp::budget::current_cancel_flag() {
if flag.load(Ordering::Acquire) {
COOPERATIVE_CANCEL_OBSERVED.store(true, Ordering::SeqCst);
return ToolResponse::success(serde_json::json!("cancelled"), None);
}
if std::time::Instant::now() >= give_up_at {
return ToolResponse::success(serde_json::json!("handler-bailout"), None);
}
std::hint::spin_loop();
}
ToolResponse::success(serde_json::json!("ok"), None)
}
let metrics = new_test_metrics();
let semaphore = Arc::new(tokio::sync::Semaphore::new(1));
let cancel_flag = Arc::new(AtomicBool::new(false));
let budget = ToolBudget::CHEAP.with_max_elapsed_ms(50);
let before_finish = blocking_gate();
let finished = blocking_gate();
let hooks = ExecutionHooks {
before_finish: Some(before_finish.clone()),
finished: Some(finished.clone()),
..ExecutionHooks::none()
};
COOPERATIVE_CANCEL_OBSERVED.store(false, Ordering::SeqCst);
let call = tokio::spawn(execute_tool_bounded_with_hooks(
cancel_polling_handler as registry::ToolHandler,
Value::Object(serde_json::Map::new()),
"test_tool".to_string(),
budget,
cancel_flag.clone(),
semaphore,
hooks,
metrics.clone(),
));
let observed = Arc::new(tokio::sync::Notify::new());
let observed_for_poller = observed.clone();
let cancel_poller = std::thread::Builder::new()
.name("cooperative-cancel-poller".into())
.spawn(move || {
let deadline = std::time::Instant::now() + Duration::from_secs(5);
while !COOPERATIVE_CANCEL_OBSERVED.load(Ordering::Acquire) {
if std::time::Instant::now() >= deadline {
return;
}
std::thread::sleep(Duration::from_millis(1));
}
observed_for_poller.notify_one();
})
.expect("failed to spawn cancel poller");
let watchdog = tokio::time::timeout(Duration::from_secs(5), async {
observed.notified().await;
});
let observed_in_time = watchdog.await.is_ok();
cancel_poller
.join()
.expect("cancel poller thread must not panic");
assert!(
observed_in_time,
"handler must observe the cancel flag within 5s"
);
assert!(
cancel_flag.load(Ordering::Relaxed),
"timeout must set the cancel flag"
);
assert!(
COOPERATIVE_CANCEL_OBSERVED.load(Ordering::SeqCst),
"handler must observe the cancel flag"
);
before_finish.wait_until_entered().await;
before_finish.release();
finished.wait_until_entered().await;
finished.release();
let snap = snapshot_from_metrics(&metrics);
assert_eq!(snap.active_blocking_handlers, 0);
assert_eq!(snap.timed_out_handlers, 0);
assert_snapshot_invariant(&snap);
let outcome = call.await.unwrap();
assert!(outcome.timed_out, "caller must receive timeout");
}
#[tokio::test]
async fn one_hundred_exact_interleavings() {
let metrics = new_test_metrics();
let semaphore = Arc::new(tokio::sync::Semaphore::new(1));
let mut timeouts_observed = 0usize;
let mut successes_observed = 0usize;
for iter in 0..100 {
let cancel_flag = Arc::new(AtomicBool::new(false));
let is_timeout_case = iter % 2 == 0;
let before_finish = blocking_gate();
let finished = blocking_gate();
let before_timeout_record = async_gate();
let timeout_recorded = async_gate();
let hooks = ExecutionHooks {
before_finish: Some(before_finish.clone()),
finished: Some(finished.clone()),
before_timeout_record: Some(before_timeout_record.clone()),
timeout_recorded: Some(timeout_recorded.clone()),
..ExecutionHooks::none()
};
let budget = ToolBudget::CHEAP.with_max_elapsed_ms(100);
let call = tokio::spawn(execute_tool_bounded_with_hooks(
test_handler_fast as registry::ToolHandler,
Value::Object(serde_json::Map::new()),
format!("iter_{}", iter),
budget,
cancel_flag,
semaphore.clone(),
hooks,
metrics.clone(),
));
before_finish.wait_until_entered().await;
before_timeout_record.wait_until_entered().await;
if is_timeout_case {
before_timeout_record.release();
timeout_recorded.wait_until_entered().await;
timeout_recorded.release();
assert_eq!(
metrics.timed_out_handlers.load(Ordering::Relaxed),
1,
"iter {}: timed_out_handlers must be 1",
iter
);
before_finish.release();
finished.wait_until_entered().await;
finished.release();
timeouts_observed += 1;
} else {
before_finish.release();
finished.wait_until_entered().await;
finished.release();
before_timeout_record.release();
timeout_recorded.wait_until_entered().await;
timeout_recorded.release();
assert_eq!(
metrics.timed_out_handlers.load(Ordering::Relaxed),
0,
"iter {}: timed_out_handlers must be 0 (completion wins)",
iter
);
successes_observed += 1;
}
let snap = snapshot_from_metrics(&metrics);
assert_eq!(
snap.active_blocking_handlers, 0,
"iter {}: active must be 0",
iter
);
assert_eq!(
snap.timed_out_handlers, 0,
"iter {}: timed_out must be 0",
iter
);
assert_snapshot_invariant(&snap);
let outcome = call.await.unwrap();
assert!(
outcome.timed_out,
"iter {}: caller must receive timeout",
iter
);
}
assert_eq!(
successes_observed, 50,
"exactly 50 successful completions expected"
);
assert_eq!(timeouts_observed, 50, "exactly 50 timeouts expected");
assert_snapshot_invariant(&snapshot_from_metrics(&metrics));
}
#[tokio::test]
#[ignore = "stress test: 500 iterations, run manually"]
async fn five_hundred_exact_interleavings() {
let metrics = new_test_metrics();
let semaphore = Arc::new(tokio::sync::Semaphore::new(1));
let mut timeouts_observed = 0usize;
let mut successes_observed = 0usize;
for iter in 0..500 {
let cancel_flag = Arc::new(AtomicBool::new(false));
let is_timeout_case = iter % 2 == 0;
let before_finish = blocking_gate();
let finished = blocking_gate();
let before_timeout_record = async_gate();
let timeout_recorded = async_gate();
let hooks = ExecutionHooks {
before_finish: Some(before_finish.clone()),
finished: Some(finished.clone()),
before_timeout_record: Some(before_timeout_record.clone()),
timeout_recorded: Some(timeout_recorded.clone()),
..ExecutionHooks::none()
};
let budget = ToolBudget::CHEAP.with_max_elapsed_ms(100);
let call = tokio::spawn(execute_tool_bounded_with_hooks(
test_handler_fast as registry::ToolHandler,
Value::Object(serde_json::Map::new()),
format!("iter_{}", iter),
budget,
cancel_flag,
semaphore.clone(),
hooks,
metrics.clone(),
));
before_finish.wait_until_entered().await;
before_timeout_record.wait_until_entered().await;
if is_timeout_case {
before_timeout_record.release();
timeout_recorded.wait_until_entered().await;
timeout_recorded.release();
before_finish.release();
finished.wait_until_entered().await;
finished.release();
timeouts_observed += 1;
} else {
before_finish.release();
finished.wait_until_entered().await;
finished.release();
before_timeout_record.release();
timeout_recorded.wait_until_entered().await;
timeout_recorded.release();
successes_observed += 1;
}
let snap = snapshot_from_metrics(&metrics);
assert_eq!(
snap.active_blocking_handlers, 0,
"iter {}: active must be 0",
iter
);
assert_eq!(
snap.timed_out_handlers, 0,
"iter {}: timed_out must be 0",
iter
);
assert_snapshot_invariant(&snap);
let outcome = call.await.unwrap();
assert!(
outcome.timed_out,
"iter {}: caller must receive timeout",
iter
);
}
assert_eq!(
successes_observed, 250,
"exactly 250 successful completions expected"
);
assert_eq!(timeouts_observed, 250, "exactly 250 timeouts expected");
assert_snapshot_invariant(&snapshot_from_metrics(&metrics));
}
#[tokio::test]
async fn worker_bound_n_plus_one() {
let max_workers = 3usize;
let metrics = new_test_metrics();
let semaphore = Arc::new(tokio::sync::Semaphore::new(max_workers));
let mut handles = Vec::new();
for i in 0..max_workers {
let sem = semaphore.clone();
let cancel_flag = Arc::new(AtomicBool::new(false));
let budget = ToolBudget::CHEAP.with_max_elapsed_ms(30_000);
let running_established = blocking_gate();
let finished = blocking_gate();
let hooks = ExecutionHooks {
running_established: Some(running_established.clone()),
finished: Some(finished.clone()),
..ExecutionHooks::none()
};
let slot = acquire_test_slot();
slot.block();
let handle = tokio::spawn(execute_tool_bounded_with_hooks(
test_handler_blocking_slot as registry::ToolHandler,
block_slot_args(&slot),
format!("worker_{}", i),
budget,
cancel_flag,
sem,
hooks,
metrics.clone(),
));
handles.push((i, handle, running_established, finished, slot));
}
for (_, _, re, _, _) in &handles {
re.wait_until_entered().await;
re.release();
}
let snap = snapshot_from_metrics(&metrics);
assert_eq!(
snap.active_blocking_handlers, max_workers,
"active_blocking_handlers must equal max_workers"
);
assert_eq!(
snap.peak_blocking_concurrency, max_workers,
"peak_blocking_concurrency must equal max_workers"
);
let fourth_cancel = Arc::new(AtomicBool::new(false));
let fourth_budget = ToolBudget::CHEAP.with_max_elapsed_ms(30_000);
let fourth_running = blocking_gate();
let fourth_finished = blocking_gate();
let fourth_hooks = ExecutionHooks {
running_established: Some(fourth_running.clone()),
finished: Some(fourth_finished.clone()),
..ExecutionHooks::none()
};
let fourth_slot = acquire_test_slot();
fourth_slot.block();
let fourth_handle = tokio::spawn(execute_tool_bounded_with_hooks(
test_handler_blocking_slot as registry::ToolHandler,
block_slot_args(&fourth_slot),
"worker_3".to_string(),
fourth_budget,
fourth_cancel,
semaphore.clone(),
fourth_hooks,
metrics.clone(),
));
let negative = tokio::time::timeout(
Duration::from_millis(50),
fourth_running.wait_until_entered(),
);
assert!(
negative.await.is_err(),
"fourth invocation must not reach running_established while all permits are occupied"
);
handles[0].4.release();
handles[0].3.release();
fourth_running.wait_until_entered().await;
let snap = snapshot_from_metrics(&metrics);
assert_eq!(
snap.active_blocking_handlers, max_workers,
"active concurrency must remain {} after one release, not {}",
max_workers, snap.active_blocking_handlers
);
for (_, _, _, _, slot) in handles.iter().skip(1) {
slot.release();
}
for (_, _, _, fin, _) in &handles {
fin.release();
}
fourth_slot.release();
fourth_running.release();
fourth_finished.release();
for (_, handle, _, _, _) in handles {
let outcome = handle.await;
assert!(outcome.is_ok(), "handler must complete without panic");
}
let fourth_outcome = fourth_handle.await.unwrap();
assert!(!fourth_outcome.timed_out, "fourth handler must succeed");
let snap = snapshot_from_metrics(&metrics);
assert_eq!(snap.active_blocking_handlers, 0);
assert_eq!(snap.timed_out_handlers, 0);
assert_eq!(snap.peak_blocking_concurrency, max_workers);
assert_snapshot_invariant(&snap);
}
#[tokio::test]
async fn deterministic_completion_wins() {
let metrics = new_test_metrics();
let semaphore = Arc::new(tokio::sync::Semaphore::new(1));
let cancel_flag = Arc::new(AtomicBool::new(false));
let budget = ToolBudget::CHEAP.with_max_elapsed_ms(5000);
let finished = blocking_gate();
let hooks = ExecutionHooks {
finished: Some(finished.clone()),
..ExecutionHooks::none()
};
let call = tokio::spawn(execute_tool_bounded_with_hooks(
test_handler_fast as registry::ToolHandler,
Value::Object(serde_json::Map::new()),
"test_tool".to_string(),
budget,
cancel_flag,
semaphore,
hooks,
metrics.clone(),
));
finished.wait_until_entered().await;
finished.release();
let outcome = call.await.unwrap();
assert!(
!outcome.timed_out,
"fast handler must complete before timeout"
);
assert!(outcome.tool_response.is_ok());
let snap = snapshot_from_metrics(&metrics);
assert_eq!(snap.active_blocking_handlers, 0);
assert_eq!(snap.timed_out_handlers, 0);
assert_eq!(snap.peak_blocking_concurrency, 1);
assert_snapshot_invariant(&snap);
}
#[tokio::test]
async fn repeated_single_threaded_100_iterations() {
let metrics = new_test_metrics();
let semaphore = Arc::new(tokio::sync::Semaphore::new(1));
for iter in 0..100 {
let cancel_flag = Arc::new(AtomicBool::new(false));
if iter % 3 == 2 {
let slot = acquire_test_slot();
slot.block();
let closure_exited = Arc::new(ClosureExitSignal::new());
let outcome = execute_tool_bounded_with_hooks(
test_handler_blocking_slot as registry::ToolHandler,
block_slot_args(&slot),
format!("iter_{}", iter),
ToolBudget::CHEAP.with_max_elapsed_ms(5),
cancel_flag,
semaphore.clone(),
ExecutionHooks {
closure_exited: Some(closure_exited.clone()),
..ExecutionHooks::none()
},
metrics.clone(),
)
.await;
assert!(outcome.timed_out, "iter {}: blocking must time out", iter);
slot.release();
assert!(
tokio::time::timeout(Duration::from_secs(5), closure_exited.wait())
.await
.is_ok(),
"iter {}: blocking closure must signal exit",
iter
);
} else {
let outcome = execute_tool_bounded_with_hooks(
test_handler_fast as registry::ToolHandler,
Value::Object(serde_json::Map::new()),
format!("iter_{}", iter),
if iter % 3 == 0 {
ToolBudget::CHEAP.with_max_elapsed_ms(5000)
} else {
ToolBudget::CHEAP.with_max_elapsed_ms(100)
},
cancel_flag,
semaphore.clone(),
ExecutionHooks::none(),
metrics.clone(),
)
.await;
assert!(
!outcome.timed_out,
"iter {}: fast handler must finish",
iter
);
}
assert_snapshot_invariant(&snapshot_from_metrics(&metrics));
}
let snap = snapshot_from_metrics(&metrics);
assert_eq!(snap.active_blocking_handlers, 0);
assert_eq!(snap.timed_out_handlers, 0);
assert_snapshot_invariant(&snap);
}
}
#[cfg(test)]
mod controlled_tests {
use crate::mcp::runtime::snapshot_metrics;
#[test]
fn test_direct_calls_do_not_modify_mcp_metrics() {
let before = snapshot_metrics();
let handler = crate::mcp::registry::tool_handler_for("math_eval").unwrap();
let args = serde_json::json!({"expression": "2 + 2"});
let _ = handler(&args);
let after = snapshot_metrics();
assert_eq!(after.active_requests, before.active_requests);
assert_eq!(
after.active_blocking_handlers,
before.active_blocking_handlers
);
assert_eq!(after.timed_out_handlers, before.timed_out_handlers);
}
}