use super::owned_process_registry::OwnedProcessRegistry;
use super::resolved_tools::ResolvedToolSet;
use super::tool_capacity::{SharedToolCapacity, ToolPermit, TransactionToolCapacity};
use super::tool_handler::{OwnedProcessLease, ToolExecutionControl, ToolKillHandle};
use std::sync::atomic::AtomicU32;
pub struct OrphanToolPermitSet {
orphans: std::sync::Mutex<Vec<OrphanPermit>>,
}
struct OrphanPermit {
#[allow(dead_code)]
permit: ToolPermit,
#[allow(dead_code)]
process_lease: Option<OwnedProcessLease>,
}
impl OrphanToolPermitSet {
pub fn new() -> Self {
Self {
orphans: std::sync::Mutex::new(Vec::new()),
}
}
fn park_orphan_permit(&self, permit: ToolPermit) {
self.park_orphan_permit_with_lease(permit, None);
}
fn park_orphan_permit_with_lease(
&self,
permit: ToolPermit,
process_lease: Option<OwnedProcessLease>,
) {
let mut orphans = self.orphans.lock().unwrap_or_else(|e| e.into_inner());
orphans.push(OrphanPermit {
permit,
process_lease,
});
}
pub fn shutdown_progress(&self) -> usize {
let mut orphans = self.orphans.lock().unwrap_or_else(|e| e.into_inner());
orphans.clear();
0
}
pub fn pending_permits(&self) -> usize {
self.orphans.lock().unwrap_or_else(|e| e.into_inner()).len()
}
pub fn is_empty(&self) -> bool {
self.pending_count() == 0
}
pub fn pending_count(&self) -> usize {
self.pending_permits()
}
}
impl Default for OrphanToolPermitSet {
fn default() -> Self {
Self::new()
}
}
impl Drop for OrphanToolPermitSet {
fn drop(&mut self) {
let _ = std::mem::take(self.orphans.get_mut().unwrap_or_else(|e| e.into_inner()));
}
}
use super::validation::{
validate_tool_completion, validate_tool_input, InputValidationFailure, DEFAULT_MAX_JSON_DEPTH,
};
use monoloop_contracts::{
CanonicalToolError, CanonicalToolOutput, CanonicalToolResult, CanonicalToolResultOutcome,
ExchangeId, SessionKey, ToolActionId, ToolCall, ToolCallContext, ToolCompletion,
ToolExecutionClass, ToolId, ToolLifecycleEvent, ToolName, ToolRuntimeError, ToolStartError,
TransactionId,
};
use std::future::Future;
use std::panic::{catch_unwind, AssertUnwindSafe};
use std::pin::Pin;
use std::sync::Arc;
use std::time::{Duration, Instant};
#[derive(Clone, Debug)]
pub struct DispatchRequest {
pub exchange_id: ExchangeId,
pub tool_action_id: ToolActionId,
pub tool_name: ToolName,
pub provider_tool_call_id: String,
pub request_ordinal: u32,
pub arguments_json: String,
pub transaction_deadline: Instant,
}
#[derive(Clone, Debug)]
pub enum DispatchOutcome {
Canonical {
result: CanonicalToolResult,
lifecycle: Vec<ToolLifecycleEvent>,
},
Rejected {
tool_action_id: ToolActionId,
code: &'static str,
message: String,
lifecycle: Vec<ToolLifecycleEvent>,
},
RuntimeFailed {
tool_action_id: ToolActionId,
tool_id: Option<ToolId>,
code: String,
lifecycle: Vec<ToolLifecycleEvent>,
},
}
#[derive(Clone, Copy, Debug)]
pub struct DispatcherLimits {
pub max_concurrent_tools: usize,
pub max_queued_tools: usize,
pub max_tool_payload_bytes: usize,
pub max_tool_output_bytes: usize,
}
impl Default for DispatcherLimits {
fn default() -> Self {
Self {
max_concurrent_tools: 16,
max_queued_tools: 64,
max_tool_payload_bytes: usize::MAX,
max_tool_output_bytes: usize::MAX,
}
}
}
pub struct TransactionToolDispatcher {
transaction_id: TransactionId,
session_key: std::sync::Mutex<SessionKey>,
tools: ResolvedToolSet,
capacity: Arc<TransactionToolCapacity>,
tool_spill: Arc<OrphanToolPermitSet>,
owned_processes: Arc<AtomicU32>,
process_registry: Arc<OwnedProcessRegistry>,
max_tool_payload_bytes: usize,
max_tool_output_bytes: usize,
max_error_message_bytes: usize,
max_json_depth: u32,
}
impl TransactionToolDispatcher {
pub fn new(
transaction_id: TransactionId,
session_key: SessionKey,
tools: ResolvedToolSet,
shared_capacity: Arc<SharedToolCapacity>,
max_concurrent_tools: usize,
max_queued_tools: usize,
) -> Arc<Self> {
Self::with_limits(
transaction_id,
session_key,
tools,
shared_capacity,
DispatcherLimits {
max_concurrent_tools,
max_queued_tools,
max_tool_payload_bytes: usize::MAX,
max_tool_output_bytes: usize::MAX,
},
)
}
pub fn with_limits(
transaction_id: TransactionId,
session_key: SessionKey,
tools: ResolvedToolSet,
shared_capacity: Arc<SharedToolCapacity>,
limits: DispatcherLimits,
) -> Arc<Self> {
Self::with_limits_and_spill(
transaction_id,
session_key,
tools,
shared_capacity,
limits,
Arc::new(OrphanToolPermitSet::new()),
Arc::new(AtomicU32::new(0)),
Arc::new(OwnedProcessRegistry::new()),
)
}
pub fn with_runtime_spill(
transaction_id: TransactionId,
session_key: SessionKey,
tools: ResolvedToolSet,
shared_capacity: Arc<SharedToolCapacity>,
tool_spill: Arc<OrphanToolPermitSet>,
limits: DispatcherLimits,
) -> Arc<Self> {
Self::with_runtime_resources(
transaction_id,
session_key,
tools,
shared_capacity,
tool_spill,
Arc::new(AtomicU32::new(0)),
Arc::new(OwnedProcessRegistry::new()),
limits,
)
}
#[allow(clippy::too_many_arguments)]
pub fn with_runtime_resources(
transaction_id: TransactionId,
session_key: SessionKey,
tools: ResolvedToolSet,
shared_capacity: Arc<SharedToolCapacity>,
tool_spill: Arc<OrphanToolPermitSet>,
owned_processes: Arc<AtomicU32>,
process_registry: Arc<OwnedProcessRegistry>,
limits: DispatcherLimits,
) -> Arc<Self> {
Self::with_limits_and_spill(
transaction_id,
session_key,
tools,
shared_capacity,
limits,
tool_spill,
owned_processes,
process_registry,
)
}
pub fn limits_from_transaction(
limits: &monoloop_contracts::TransactionLimits,
) -> DispatcherLimits {
DispatcherLimits {
max_concurrent_tools: limits.max_concurrent_tools_per_transaction.max(1),
max_queued_tools: limits.max_queued_tools_per_transaction.max(1),
max_tool_payload_bytes: limits.max_tool_payload_bytes.max(1),
max_tool_output_bytes: limits.max_tool_output_bytes.max(1),
}
}
#[allow(clippy::too_many_arguments)]
fn with_limits_and_spill(
transaction_id: TransactionId,
session_key: SessionKey,
tools: ResolvedToolSet,
shared_capacity: Arc<SharedToolCapacity>,
limits: DispatcherLimits,
tool_spill: Arc<OrphanToolPermitSet>,
owned_processes: Arc<AtomicU32>,
process_registry: Arc<OwnedProcessRegistry>,
) -> Arc<Self> {
let capacity = TransactionToolCapacity::new(
shared_capacity,
limits.max_concurrent_tools,
limits.max_queued_tools,
);
for spec in tools.specs() {
capacity.configure_tool(spec.id.clone(), spec.limits.max_concurrent);
}
Arc::new(Self {
transaction_id,
session_key: std::sync::Mutex::new(session_key),
tools,
capacity,
tool_spill,
owned_processes,
process_registry,
max_tool_payload_bytes: limits.max_tool_payload_bytes.max(1),
max_tool_output_bytes: limits.max_tool_output_bytes.max(1),
max_error_message_bytes: 1024,
max_json_depth: DEFAULT_MAX_JSON_DEPTH,
})
}
pub fn tools(&self) -> &ResolvedToolSet {
&self.tools
}
pub fn transaction_id(&self) -> TransactionId {
self.transaction_id
}
pub fn session_key(&self) -> SessionKey {
self.session_key
.lock()
.unwrap_or_else(|e| e.into_inner())
.clone()
}
pub fn rebind_session(&self, session_key: SessionKey) {
*self.session_key.lock().unwrap_or_else(|e| e.into_inner()) = session_key;
}
pub fn vault_pending_permits(&self) -> usize {
self.tool_spill.pending_permits()
}
pub fn tool_spill(&self) -> &Arc<OrphanToolPermitSet> {
&self.tool_spill
}
pub fn active_tools(&self) -> usize {
self.capacity.active()
}
pub fn queued_tools(&self) -> usize {
self.capacity.queued()
}
pub async fn dispatch(self: &Arc<Self>, request: DispatchRequest) -> DispatchOutcome {
self.dispatch_with_cancel(request, None).await
}
pub async fn dispatch_with_cancel(
self: &Arc<Self>,
request: DispatchRequest,
cancel: Option<Arc<super::sticky_cancel::StickyCancel>>,
) -> DispatchOutcome {
let action = request.tool_action_id.clone();
let Some(resolved) = self.tools.get_by_name(&request.tool_name) else {
return DispatchOutcome::Rejected {
tool_action_id: action,
code: "tool_not_allowed",
message: "tool not in resolved set".into(),
lifecycle: vec![],
};
};
let tool_id = resolved.spec.id.clone();
let spec = resolved.spec.clone();
let handler = Arc::clone(&resolved.handler);
if !self.capacity.try_enqueue() {
return DispatchOutcome::Rejected {
tool_action_id: action,
code: "tool_queue_full",
message: "per-transaction tool queue full".into(),
lifecycle: vec![],
};
}
let max_payload = spec.limits.max_input_bytes.min(self.max_tool_payload_bytes);
let arguments = match validate_tool_input(
&request.arguments_json,
&spec.input_schema,
max_payload,
self.max_json_depth,
) {
Ok(v) => v,
Err(f) => {
self.capacity.dequeue();
return reject_input(action, f);
}
};
let permit = {
let mut acquired = None;
let deadline = Instant::now() + Duration::from_millis(50);
while Instant::now() < deadline {
if let Some(p) = self.capacity.try_acquire(&tool_id) {
acquired = Some(p);
break;
}
tokio::task::yield_now().await;
}
match acquired {
Some(p) => p,
None => {
self.capacity.dequeue();
return DispatchOutcome::Rejected {
tool_action_id: action,
code: "tool_capacity_exceeded",
message: "tool concurrency capacity exceeded".into(),
lifecycle: vec![],
};
}
}
};
let mut dispatch_guard = DispatchGuard {
permit: Some(permit),
kill: None,
process_registry: Arc::clone(&self.process_registry),
};
let mut lifecycle = vec![ToolLifecycleEvent::Started {
tool_action_id: action.clone(),
tool_id: tool_id.clone(),
tool_name: request.tool_name.clone(),
provider_tool_call_id: request.provider_tool_call_id.clone(),
request_ordinal: request.request_ordinal,
}];
let call = ToolCall {
tool_name: request.tool_name.clone(),
tool_id: tool_id.clone(),
provider_tool_call_id: request.provider_tool_call_id.clone(),
arguments,
request_ordinal: request.request_ordinal,
};
let now = Instant::now();
if now >= request.transaction_deadline {
drop(dispatch_guard);
return DispatchOutcome::RuntimeFailed {
tool_action_id: action,
tool_id: Some(tool_id),
code: "transaction_deadline_exceeded".into(),
lifecycle: vec![],
};
}
let tool_cap = now + spec.limits.execution_deadline;
let absolute_tool_deadline = tool_cap.min(request.transaction_deadline);
let context = ToolCallContext {
transaction_id: self.transaction_id,
session_key: self.session_key(),
exchange_id: Some(request.exchange_id),
tool_action_id: action.clone(),
tool_id: tool_id.clone(),
deadline: absolute_tool_deadline,
};
let deadline = absolute_tool_deadline.saturating_duration_since(now);
let policy = spec.execution_class.clone();
let supports_required_termination = match &policy {
ToolExecutionClass::AbortableAtYield { .. } => {
handler.runtime_owns_abortable_drive() && handler.supports_abort()
}
ToolExecutionClass::ProcessIsolated { .. } => {
handler.os_process_isolated() && handler.supports_isolated_kill()
}
ToolExecutionClass::CooperativeInProcess { .. } => true,
};
if !supports_required_termination {
drop(dispatch_guard);
lifecycle.push(ToolLifecycleEvent::RuntimeFailed {
tool_action_id: action.clone(),
tool_id: tool_id.clone(),
code: "missing_kill_handle".into(),
});
return DispatchOutcome::RuntimeFailed {
tool_action_id: action,
tool_id: Some(tool_id),
code: "missing_kill_handle".into(),
lifecycle,
};
}
let start_result = catch_unwind(AssertUnwindSafe(|| handler.start(call, context)));
let handle = match start_result {
Ok(Ok(h)) => h,
Ok(Err(ToolStartError::CapacityExceeded)) => {
drop(dispatch_guard);
return DispatchOutcome::Rejected {
tool_action_id: action,
code: "tool_capacity_exceeded",
message: "handler capacity exceeded".into(),
lifecycle,
};
}
Ok(Err(ToolStartError::Rejected(reason))) => {
drop(dispatch_guard);
lifecycle.push(ToolLifecycleEvent::RuntimeFailed {
tool_action_id: action.clone(),
tool_id: tool_id.clone(),
code: "tool_start_rejected".into(),
});
return DispatchOutcome::RuntimeFailed {
tool_action_id: action,
tool_id: Some(tool_id),
code: format!("start_rejected:{reason}"),
lifecycle,
};
}
Err(_) => {
drop(dispatch_guard);
lifecycle.push(ToolLifecycleEvent::RuntimeFailed {
tool_action_id: action.clone(),
tool_id: tool_id.clone(),
code: "tool_panicked".into(),
});
return DispatchOutcome::RuntimeFailed {
tool_action_id: action,
tool_id: Some(tool_id),
code: "panicked".into(),
lifecycle,
};
}
};
let control = handle.control.clone();
dispatch_guard.kill = handle.kill.clone();
let kill = dispatch_guard.kill.clone();
if let Some(ref k) = kill {
if k.is_process_isolated() {
k.register_owned_process(Arc::clone(&self.owned_processes));
}
}
let ownership_ok = match &policy {
ToolExecutionClass::AbortableAtYield { .. } => {
kill.as_ref().is_some_and(|k| k.is_cancel_only()) && handle.drive.is_some()
}
ToolExecutionClass::ProcessIsolated { .. } => {
kill.as_ref().is_some_and(|k| k.is_process_isolated()) && handle.drive.is_some()
}
ToolExecutionClass::CooperativeInProcess { .. } => true,
};
if !ownership_ok {
control.cancel();
let completion = handle.completion;
let wait = completion.wait();
tokio::pin!(wait);
match tokio::time::timeout(Duration::from_millis(200), &mut wait).await {
Ok(_) => {
dispatch_guard.release_if_idle();
}
Err(_) => {
if let Some(permit) = dispatch_guard.permit.take() {
self.tool_spill.park_orphan_permit(permit);
}
}
}
lifecycle.push(ToolLifecycleEvent::RuntimeFailed {
tool_action_id: action.clone(),
tool_id: tool_id.clone(),
code: "missing_kill_handle".into(),
});
return DispatchOutcome::RuntimeFailed {
tool_action_id: action,
tool_id: Some(tool_id),
code: "missing_kill_handle".into(),
lifecycle,
};
}
let mut drive = handle.drive;
let wait = handle.completion.wait();
tokio::pin!(wait);
let cancel_fut = async {
if let Some(n) = cancel.as_ref() {
n.cancelled().await;
} else {
std::future::pending::<()>().await;
}
};
let completion = if let Some(drive_fut) = drive.take() {
tokio::pin!(drive_fut);
tokio::select! {
biased;
c = &mut wait => c,
_ = &mut drive_fut => wait.await,
_ = cancel_fut => {
await_tool_termination_driven(
&mut wait,
&mut drive_fut,
&control,
kill.as_ref(),
&policy,
)
.await
}
_ = tokio::time::sleep(deadline) => {
await_tool_termination_driven(
&mut wait,
&mut drive_fut,
&control,
kill.as_ref(),
&policy,
)
.await
}
}
} else {
tokio::select! {
biased;
c = &mut wait => c,
_ = cancel_fut => {
await_tool_termination(&mut wait, &control, kill.as_ref(), &policy).await
}
_ = tokio::time::sleep(deadline) => {
await_tool_termination(&mut wait, &control, kill.as_ref(), &policy).await
}
}
};
let cooperative_deadline =
matches!(policy, ToolExecutionClass::CooperativeInProcess { .. })
&& matches!(
&completion,
ToolCompletion::RuntimeFailed(ToolRuntimeError::DeadlineExceeded)
);
if let Some(ref k) = dispatch_guard.kill {
let _ = k.join_timeout(Duration::from_millis(50)).await;
if cooperative_deadline && k.is_cancel_only() {
if let Some(permit) = dispatch_guard.permit.take() {
self.tool_spill.park_orphan_permit(permit);
}
}
} else if cooperative_deadline {
if let Some(permit) = dispatch_guard.permit.take() {
self.tool_spill.park_orphan_permit(permit);
}
}
dispatch_guard.release_if_idle();
let max_output = spec.limits.max_output_bytes.min(self.max_tool_output_bytes);
let validated = match validate_tool_completion(
completion,
&spec.output_contract,
max_output,
self.max_error_message_bytes,
self.max_json_depth,
) {
Ok(c) => c,
Err(_) => {
lifecycle.push(ToolLifecycleEvent::RuntimeFailed {
tool_action_id: action.clone(),
tool_id: tool_id.clone(),
code: "output_contract_violated".into(),
});
return DispatchOutcome::RuntimeFailed {
tool_action_id: action,
tool_id: Some(tool_id),
code: "output_contract_violated".into(),
lifecycle,
};
}
};
match validated {
ToolCompletion::Succeeded(output) => {
let result = CanonicalToolResult {
transaction_id: self.transaction_id,
session_key: self.session_key(),
exchange_id: request.exchange_id,
tool_action_id: action.clone(),
tool_id: tool_id.clone(),
provider_tool_call_id: request.provider_tool_call_id,
request_ordinal: request.request_ordinal,
outcome: CanonicalToolResultOutcome::Succeeded(output),
};
lifecycle.push(ToolLifecycleEvent::Completed {
result: result.clone(),
});
DispatchOutcome::Canonical { result, lifecycle }
}
ToolCompletion::DomainFailed(err) => {
let result = CanonicalToolResult {
transaction_id: self.transaction_id,
session_key: self.session_key(),
exchange_id: request.exchange_id,
tool_action_id: action.clone(),
tool_id: tool_id.clone(),
provider_tool_call_id: request.provider_tool_call_id,
request_ordinal: request.request_ordinal,
outcome: CanonicalToolResultOutcome::DomainFailed(err),
};
lifecycle.push(ToolLifecycleEvent::Completed {
result: result.clone(),
});
DispatchOutcome::Canonical { result, lifecycle }
}
ToolCompletion::RuntimeFailed(e) => {
let code = match e {
ToolRuntimeError::Panicked => "panicked",
ToolRuntimeError::CompletionLost => "completion_lost",
ToolRuntimeError::OutputContractViolated => "output_contract_violated",
ToolRuntimeError::TerminationFailed => "termination_failed",
ToolRuntimeError::DeadlineExceeded => "deadline_exceeded",
};
lifecycle.push(ToolLifecycleEvent::RuntimeFailed {
tool_action_id: action.clone(),
tool_id: tool_id.clone(),
code: code.into(),
});
DispatchOutcome::RuntimeFailed {
tool_action_id: action,
tool_id: Some(tool_id),
code: code.into(),
lifecycle,
}
}
}
}
}
async fn await_tool_termination(
wait: &mut Pin<&mut impl Future<Output = ToolCompletion>>,
control: &ToolExecutionControl,
kill: Option<&ToolKillHandle>,
policy: &ToolExecutionClass,
) -> ToolCompletion {
control.cancel();
let join_grace = Duration::from_millis(200);
match policy {
ToolExecutionClass::AbortableAtYield { .. } => {
if let Some(k) = kill {
k.kill();
if k.join_timeout(join_grace).await.is_err() {
return ToolCompletion::RuntimeFailed(ToolRuntimeError::TerminationFailed);
}
}
match tokio::time::timeout(join_grace, wait).await {
Ok(c) => c,
Err(_) => ToolCompletion::RuntimeFailed(ToolRuntimeError::DeadlineExceeded),
}
}
ToolExecutionClass::CooperativeInProcess { grace } => {
match tokio::time::timeout(*grace, &mut *wait).await {
Ok(c) => c,
Err(_) => ToolCompletion::RuntimeFailed(ToolRuntimeError::DeadlineExceeded),
}
}
ToolExecutionClass::ProcessIsolated {
grace,
kill_deadline,
} => match tokio::time::timeout(*grace, &mut *wait).await {
Ok(c) => c,
Err(_) => {
if let Some(k) = kill {
k.kill();
if k.join_timeout(*kill_deadline).await.is_err() {
return ToolCompletion::RuntimeFailed(ToolRuntimeError::TerminationFailed);
}
}
match tokio::time::timeout(*kill_deadline, wait).await {
Ok(c) => c,
Err(_) => ToolCompletion::RuntimeFailed(ToolRuntimeError::TerminationFailed),
}
}
},
}
}
async fn await_tool_termination_driven(
wait: &mut Pin<&mut impl Future<Output = ToolCompletion>>,
drive: &mut Pin<&mut impl Future<Output = ()>>,
control: &ToolExecutionControl,
kill: Option<&ToolKillHandle>,
policy: &ToolExecutionClass,
) -> ToolCompletion {
control.cancel();
match policy {
ToolExecutionClass::ProcessIsolated {
grace,
kill_deadline,
} => {
tokio::select! {
biased;
c = &mut *wait => c,
_ = &mut *drive => wait.await,
_ = tokio::time::sleep(*grace) => {
if let Some(k) = kill {
k.kill();
}
tokio::select! {
biased;
c = &mut *wait => c,
_ = &mut *drive => wait.await,
_ = tokio::time::sleep(*kill_deadline) => {
if let Some(k) = kill {
let _ = k.join_timeout(Duration::from_millis(50)).await;
}
ToolCompletion::RuntimeFailed(ToolRuntimeError::TerminationFailed)
}
}
}
}
}
ToolExecutionClass::AbortableAtYield { grace }
| ToolExecutionClass::CooperativeInProcess { grace } => {
if let Some(k) = kill {
k.kill();
}
let join_grace = Duration::from_millis(200).max(*grace);
tokio::select! {
biased;
c = &mut *wait => c,
_ = &mut *drive => wait.await,
_ = tokio::time::sleep(join_grace) => {
if let Some(k) = kill {
let _ = k.join_timeout(Duration::from_millis(50)).await;
}
ToolCompletion::RuntimeFailed(ToolRuntimeError::DeadlineExceeded)
}
}
}
}
}
struct DispatchGuard {
permit: Option<ToolPermit>,
kill: Option<ToolKillHandle>,
process_registry: Arc<OwnedProcessRegistry>,
}
impl DispatchGuard {
fn release_if_idle(&mut self) {
if self.kill.as_ref().is_some_and(ToolKillHandle::has_join) {
return;
}
self.kill.take();
drop(self.permit.take());
}
}
impl Drop for DispatchGuard {
fn drop(&mut self) {
if let Some(k) = self.kill.take() {
k.kill(); if k.is_process_isolated() && k.has_join() {
let permit = self.permit.take();
self.process_registry.park(k, permit);
return;
}
}
drop(self.permit.take());
}
}
fn reject_input(action: ToolActionId, f: InputValidationFailure) -> DispatchOutcome {
let (code, message) = match f {
InputValidationFailure::OversizedInput => ("oversized_input", "tool input exceeds limit"),
InputValidationFailure::InvalidJson => {
("invalid_json", "tool arguments are not valid JSON")
}
InputValidationFailure::DepthExceeded => ("json_depth_exceeded", "tool arguments too deep"),
InputValidationFailure::SchemaInvalid => {
("schema_invalid", "tool arguments fail input schema")
}
};
let _ = CanonicalToolError::try_new(code, message, None, 256);
let _ = CanonicalToolOutput::Text(String::new());
DispatchOutcome::Rejected {
tool_action_id: action,
code,
message: message.into(),
lifecycle: vec![],
}
}