use std::{collections::BTreeMap, future::Future, path::PathBuf, sync::Arc, time::Duration};
use tokio::task::JoinSet;
use crate::{
ContentBlock,
agent::{Agent, AgentEvent, AgentStatus},
error::RuntimeError,
runtime::control::{
AfterDecision, BeforeDecision, ExecutionHookSnapshot, HookDecision, PostExecutionContext,
PreExecutionContext, ResultDecision,
},
runtime::{RunOptions, RuntimeHookEvent},
tool::{
ExecutableTool, ParallelToolContext, ResolvedTool, RuntimeToolDescriptor,
ToolAuthorizationOutcome, ToolAuthorizationRequest, ToolCall, ToolCapability, ToolContext,
ToolExecutionCategory, ToolResultContent,
},
};
use super::{
paging::{READ_TOOL_RESULT_TOOL, ToolResultPager},
truncation::{SpillBehavior, ToolOutputLimiter},
};
const PARALLEL_JOIN_POLL_INTERVAL: Duration = Duration::from_millis(10);
pub(crate) struct ToolExecutionOutcome {
pub(crate) results: Vec<ContentBlock>,
pub(crate) successful_task: bool,
pub(crate) end_turn: bool,
pub(crate) details: BTreeMap<String, serde_json::Value>,
}
pub(crate) struct ToolRuntime {
runtime: crate::runtime::handle::RuntimeHandle,
agent_id: String,
tool_calls: usize,
working_directory: Option<PathBuf>,
output_limiter: ToolOutputLimiter,
pager: Option<ToolResultPager>,
}
#[derive(Clone)]
enum ToolCallBatch {
Exclusive(Box<ScheduledToolCall>),
Parallel(Vec<ScheduledToolCall>),
}
struct ToolCallSchedule {
batches: Vec<ToolCallBatch>,
}
#[derive(Clone)]
struct ScheduledToolCall {
call: ToolCall,
tool: ScheduledTool,
execution_category: ToolExecutionCategory,
}
#[derive(Clone)]
enum ScheduledTool {
Resolved(Box<ResolvedTool>),
Unavailable,
Missing,
}
enum HookOutcome {
Proceed { modified: bool },
Refused(String),
}
enum MixedHookOutcome {
Proceed {
modified: bool,
attribution: Option<String>,
},
Refused(String),
}
enum Admission {
Run(Box<AdmittedToolCall>),
Refused(Box<CompletedToolExecution>),
}
struct AdmittedToolCall {
context: ParallelToolContext,
execution_hooks: Option<ExecutionHookSnapshot>,
rewrite_source: Option<String>,
}
struct CompletedToolExecution {
result: ContentBlock,
task_succeeded: bool,
should_end_turn: bool,
terminated: bool,
tool_name: String,
input_json: String,
details: Option<serde_json::Value>,
execution_hooks: Option<ExecutionHookSnapshot>,
}
#[derive(Debug, Clone, Copy, Default)]
struct RoundEffect {
should_end_turn: bool,
terminated: bool,
}
impl ToolRuntime {
pub(crate) fn new(agent: &Agent) -> Self {
let runtime = agent.runtime_handle();
let policy = &runtime.execution.policy;
let spill = if !policy.spill_full_tool_output {
SpillBehavior::Disabled("spill-to-file is disabled by runtime policy")
} else if !runtime.persistence.store.allows_disk_artifacts() {
SpillBehavior::Disabled("the runtime store forbids durable artifacts")
} else {
SpillBehavior::Enabled(agent.config().compaction.transcript_dir.join("tool-output"))
};
let output_limiter = ToolOutputLimiter::new(
policy.max_tool_result_bytes,
policy.max_tool_result_lines,
spill,
);
Self {
runtime,
agent_id: agent.id().to_string(),
tool_calls: 0,
working_directory: None,
output_limiter,
pager: agent.config().tool_result_paging.map(ToolResultPager::new),
}
}
pub(crate) async fn execute_calls(
&mut self,
agent: &mut Agent,
options: &RunOptions,
calls: Vec<ToolCall>,
) -> Result<ToolExecutionOutcome, RuntimeError> {
let mut results = Vec::new();
let mut successful_task = false;
let mut end_turn = false;
let mut details = BTreeMap::new();
let mut batches = ToolCallSchedule::new(self, agent, calls)
.batches
.into_iter();
while let Some(batch) = batches.next() {
options.check_limits()?;
let execution_count = batch.execution_count();
if self.tool_calls + execution_count > options.tool_budget() {
return Err(RuntimeError::ToolBudgetExceeded(options.tool_budget()));
}
self.tool_calls += execution_count;
let executions = match batch {
ToolCallBatch::Exclusive(call) => {
vec![self.execute_one_tool(agent, options, *call).await?]
}
ToolCallBatch::Parallel(calls) => {
self.execute_parallel_batch(agent, options, calls).await?
}
};
let mut terminator = None;
for execution in executions {
successful_task |= execution.task_succeeded;
end_turn |= execution.should_end_turn;
let reviewed = self
.review_result(
&execution.tool_name,
&execution.input_json,
execution.result,
execution.execution_hooks,
)
.await?;
let result = self.page_result(agent, &execution.tool_name, reviewed);
if execution.terminated {
terminator.get_or_insert(execution.tool_name);
}
if let (Some(value), ContentBlock::ToolResult { tool_use_id, .. }) =
(execution.details, &result)
{
details.insert(tool_use_id.clone(), value);
}
results.push(result);
}
if let Some(terminator) = terminator {
for remaining_batch in batches {
for call in remaining_batch.into_calls() {
let result = not_executed_result(&call, &terminator);
results.push(self.page_result(agent, &call.name, result));
}
}
break;
}
}
Ok(ToolExecutionOutcome {
results,
successful_task,
end_turn,
details,
})
}
fn page_result(&self, agent: &Agent, tool_name: &str, result: ContentBlock) -> ContentBlock {
let Some(pager) = self.pager else {
return result;
};
if tool_name == READ_TOOL_RESULT_TOOL {
return result;
}
let ContentBlock::ToolResult {
tool_use_id,
content: mentra_provider::ToolResultContent::Text(text),
is_error,
} = result
else {
return result;
};
let Some(page) = pager.first_page(&tool_use_id, &text) else {
return ContentBlock::ToolResult {
tool_use_id,
content: mentra_provider::ToolResultContent::Text(text),
is_error,
};
};
agent.record_paged_tool_result(&tool_use_id, &text);
ContentBlock::ToolResult {
tool_use_id,
content: mentra_provider::ToolResultContent::Text(page),
is_error,
}
}
fn schedule_call(&self, agent: &Agent, call: ToolCall) -> ScheduledToolCall {
let tool = match agent.resolve_tool(&call.name) {
crate::tool::ToolResolution::Visible(tool) => tool,
crate::tool::ToolResolution::Hidden => {
return ScheduledToolCall {
call,
tool: ScheduledTool::Unavailable,
execution_category: ToolExecutionCategory::ExclusiveLocalMutation,
};
}
crate::tool::ToolResolution::Missing => {
return ScheduledToolCall {
call,
tool: ScheduledTool::Missing,
execution_category: ToolExecutionCategory::ExclusiveLocalMutation,
};
}
};
let (declared, scheduled) =
Self::execution_categories_for_snapshot(&call, &tool.handler, tool.descriptor());
if scheduled != declared {
eprintln!(
"warning: tool '{}' is marked terminal but declared a parallel \
execution category; coercing to exclusive scheduling",
call.name
);
}
ScheduledToolCall {
call,
tool: ScheduledTool::Resolved(tool),
execution_category: scheduled,
}
}
fn execution_categories_for_snapshot(
call: &ToolCall,
tool: &Arc<dyn ExecutableTool>,
descriptor: &RuntimeToolDescriptor,
) -> (ToolExecutionCategory, ToolExecutionCategory) {
let declared = tool.execution_category(&call.input);
let scheduled = if descriptor.terminal && declared.allows_parallel() {
ToolExecutionCategory::ExclusiveLocalMutation
} else {
declared
};
(declared, scheduled)
}
fn note_tool_started(
&mut self,
agent: &mut Agent,
call: &ToolCall,
) -> Result<(), RuntimeError> {
agent.set_status(AgentStatus::ExecutingTool {
id: call.id.clone(),
name: call.name.clone(),
});
agent.emit_event(AgentEvent::ToolExecutionStarted { call: call.clone() });
agent.update_run_state("executing_tool", None)
}
fn emit_tool_runtime_started(&self, call: &ToolCall) -> Result<(), RuntimeError> {
self.runtime
.emit_hook(RuntimeHookEvent::ToolExecutionStarted {
agent_id: self.agent_id.clone(),
tool_name: call.name.clone(),
tool_call_id: call.id.clone(),
})
}
fn emit_tool_runtime_finished(
&self,
call: &ToolCall,
result: &ContentBlock,
details: Option<serde_json::Value>,
) {
let is_error = matches!(result, ContentBlock::ToolResult { is_error: true, .. });
let output_preview = match result {
ContentBlock::ToolResult { content, .. } => content.to_display_string(),
_ => String::new(),
};
let error = is_error.then_some(output_preview.clone());
let _ = self
.runtime
.emit_hook(RuntimeHookEvent::ToolExecutionFinished {
agent_id: self.agent_id.clone(),
tool_name: call.name.clone(),
tool_call_id: call.id.clone(),
is_error,
error,
output_preview,
details,
});
}
fn emit_tool_authorization_started(
&self,
call: &ToolCall,
preview: crate::tool::ToolAuthorizationPreview,
) -> Result<(), RuntimeError> {
self.runtime
.emit_hook(RuntimeHookEvent::ToolAuthorizationStarted {
agent_id: self.agent_id.clone(),
tool_name: call.name.clone(),
tool_call_id: call.id.clone(),
preview,
})
}
fn emit_tool_authorization_finished(
&self,
call: &ToolCall,
outcome: ToolAuthorizationOutcome,
reason: Option<String>,
) -> Result<(), RuntimeError> {
self.runtime
.emit_hook(RuntimeHookEvent::ToolAuthorizationFinished {
agent_id: self.agent_id.clone(),
tool_name: call.name.clone(),
tool_call_id: call.id.clone(),
outcome,
reason,
})
}
fn emit_tool_authorization_blocked(
&self,
call: &ToolCall,
outcome: ToolAuthorizationOutcome,
reason: Option<String>,
) -> Result<(), RuntimeError> {
self.runtime
.emit_hook(RuntimeHookEvent::ToolAuthorizationBlocked {
agent_id: self.agent_id.clone(),
tool_name: call.name.clone(),
tool_call_id: call.id.clone(),
outcome,
reason,
})
}
async fn run_pre_hooks(&mut self, call: &ToolCall) -> Result<HookDecision, RuntimeError> {
let hooks = self
.runtime
.pre_hooks()
.snapshot(self.runtime.tool_audience());
let context = PreExecutionContext {
agent_id: self.agent_id.clone(),
tool_name: call.name.clone(),
tool_call_id: call.id.clone(),
input_json: serde_json::to_string(&call.input).unwrap_or_default(),
working_directory: self.working_directory(),
};
hooks.run(&context).await
}
async fn review_result(
&mut self,
tool_name: &str,
input_json: &str,
result: ContentBlock,
execution_hooks: Option<ExecutionHookSnapshot>,
) -> Result<ContentBlock, RuntimeError> {
let (result, run_legacy) = match execution_hooks {
Some(hooks) => {
self.run_execution_after(&hooks, tool_name, input_json, result)
.await?
}
None => (result, true),
};
if run_legacy {
self.run_post_hooks(tool_name, input_json, result).await
} else {
Ok(result)
}
}
async fn run_execution_after(
&mut self,
hooks: &ExecutionHookSnapshot,
tool_name: &str,
input_json: &str,
result: ContentBlock,
) -> Result<(ContentBlock, bool), RuntimeError> {
let ContentBlock::ToolResult {
tool_use_id,
content,
is_error,
} = result
else {
return Ok((result, true));
};
let context = PostExecutionContext {
agent_id: self.agent_id.clone(),
tool_name: tool_name.to_string(),
tool_call_id: tool_use_id.clone(),
input_json: input_json.to_string(),
working_directory: self.working_directory(),
content,
is_error,
};
Ok(match hooks.after(&context).await? {
AfterDecision::Continue => (
ContentBlock::ToolResult {
tool_use_id,
content: context.content,
is_error: context.is_error,
},
true,
),
AfterDecision::Deny(reason) => (
ContentBlock::ToolResult {
tool_use_id,
content: ToolResultContent::text(reason),
is_error: true,
},
false,
),
AfterDecision::Replace {
content, is_error, ..
} => (
ContentBlock::ToolResult {
tool_use_id,
content,
is_error: is_error.unwrap_or(context.is_error),
},
true,
),
})
}
async fn run_post_hooks(
&mut self,
tool_name: &str,
input_json: &str,
result: ContentBlock,
) -> Result<ContentBlock, RuntimeError> {
let hooks = self
.runtime
.post_hooks()
.snapshot(self.runtime.tool_audience());
if hooks.is_empty() {
return Ok(result);
}
let ContentBlock::ToolResult {
tool_use_id,
content,
is_error,
} = result
else {
return Ok(result);
};
let context = PostExecutionContext {
agent_id: self.agent_id.clone(),
tool_name: tool_name.to_string(),
tool_call_id: tool_use_id.clone(),
input_json: input_json.to_string(),
working_directory: self.working_directory(),
content,
is_error,
};
Ok(match hooks.run(&context).await? {
ResultDecision::Keep => ContentBlock::ToolResult {
tool_use_id,
content: context.content,
is_error: context.is_error,
},
ResultDecision::Replace { content, is_error } => ContentBlock::ToolResult {
tool_use_id,
content,
is_error,
},
})
}
async fn apply_pre_hooks(&mut self, call: &mut ToolCall) -> Result<HookOutcome, RuntimeError> {
match self.run_pre_hooks(call).await? {
HookDecision::Allow => Ok(HookOutcome::Proceed { modified: false }),
HookDecision::Deny(reason) => Ok(HookOutcome::Refused(reason)),
HookDecision::Modify { input_json, .. } => {
match serde_json::from_str(&input_json) {
Ok(input) => {
call.input = input;
Ok(HookOutcome::Proceed { modified: true })
}
Err(error) => Ok(HookOutcome::Refused(format!(
"pre-execution hook returned invalid JSON for '{}': {error}",
call.name
))),
}
}
}
}
async fn apply_execution_before(
&mut self,
call: &mut ToolCall,
hooks: &ExecutionHookSnapshot,
) -> Result<MixedHookOutcome, RuntimeError> {
let context = PreExecutionContext {
agent_id: self.agent_id.clone(),
tool_name: call.name.clone(),
tool_call_id: call.id.clone(),
input_json: serde_json::to_string(&call.input).unwrap_or_default(),
working_directory: self.working_directory(),
};
match hooks.before(&context).await? {
BeforeDecision::Continue => Ok(MixedHookOutcome::Proceed {
modified: false,
attribution: None,
}),
BeforeDecision::Deny(reason) => Ok(MixedHookOutcome::Refused(reason)),
BeforeDecision::Modify {
input_json,
attribution,
} => match serde_json::from_str(&input_json) {
Ok(input) => {
call.input = input;
Ok(MixedHookOutcome::Proceed {
modified: true,
attribution,
})
}
Err(error) => Ok(MixedHookOutcome::Refused(format!(
"{} returned invalid JSON for '{}': {error}",
mixed_rewrite_source(attribution.as_deref()),
call.name
))),
},
}
}
async fn admit_call(
&mut self,
agent: &mut Agent,
options: &RunOptions,
call: &mut ToolCall,
tool: &Arc<dyn ExecutableTool>,
descriptor: &RuntimeToolDescriptor,
execution_category: ToolExecutionCategory,
) -> Result<Admission, RuntimeError> {
let legacy_modified = match self.apply_pre_hooks(call).await? {
HookOutcome::Proceed { modified } => modified,
HookOutcome::Refused(reason) => {
return Ok(Admission::Refused(Box::new(
self.hook_blocked_execution(agent, call, descriptor, &reason),
)));
}
};
let execution_hooks = self
.runtime
.execution_hooks()
.snapshot(self.runtime.tool_audience());
let mut mixed_modified = false;
let mut mixed_attribution = None;
if !execution_hooks.is_empty() {
match self.apply_execution_before(call, &execution_hooks).await? {
MixedHookOutcome::Proceed {
modified,
attribution,
} => {
mixed_modified = modified;
mixed_attribution = attribution;
}
MixedHookOutcome::Refused(reason) => {
return Ok(Admission::Refused(Box::new(
self.mixed_hook_blocked_execution(agent, call, descriptor, &reason),
)));
}
}
}
let rewrite_source = if mixed_modified {
Some(mixed_rewrite_source(mixed_attribution.as_deref()))
} else if legacy_modified {
Some("pre-execution hook".to_string())
} else {
None
};
let authorization_category = if let Some(rewrite_source) = rewrite_source.as_deref() {
let (_, rewritten_category) =
Self::execution_categories_for_snapshot(call, tool, descriptor);
if execution_category.allows_parallel() && !rewritten_category.allows_parallel() {
let reason = format!(
"{} changed '{}' from a parallel call into {:?}; refusing to \
run mutating work in the parallel lane",
rewrite_source, call.name, rewritten_category
);
let execution = if mixed_modified {
self.mixed_hook_blocked_execution(agent, call, descriptor, &reason)
} else {
self.hook_blocked_execution(agent, call, descriptor, &reason)
};
return Ok(Admission::Refused(Box::new(execution)));
}
rewritten_category
} else {
execution_category
};
if let Some(error) = self.schema_violation(call, descriptor) {
return Ok(Admission::Refused(Box::new(
if let Some(rewrite_source) = rewrite_source.as_deref() {
let reason = format!(
"{} rewrote '{}' into input that does not fit its schema: {error}",
rewrite_source, call.name
);
if mixed_modified {
self.mixed_hook_blocked_execution(agent, call, descriptor, &reason)
} else {
self.hook_blocked_execution(agent, call, descriptor, &reason)
}
} else {
self.schema_violation_execution(agent, call, error)
},
)));
}
let ctx = self.parallel_tool_context(agent, options, call);
if let Some(result) = self
.authorize_tool_call(
call,
tool,
&ctx,
authorization_category,
rewrite_source.as_deref(),
)
.await?
{
let execution = self.completed_execution(
agent,
call,
descriptor,
result,
RoundEffect::default(),
None,
);
return Ok(Admission::Refused(Box::new(execution)));
}
Ok(Admission::Run(Box::new(AdmittedToolCall {
context: ctx,
execution_hooks: (!execution_hooks.is_empty()).then_some(execution_hooks),
rewrite_source,
})))
}
fn hook_blocked_execution(
&self,
agent: &Agent,
call: &ToolCall,
descriptor: &RuntimeToolDescriptor,
reason: &str,
) -> CompletedToolExecution {
self.emit_tool_execution_blocked(call, reason);
let result = ContentBlock::ToolResult {
tool_use_id: call.id.clone(),
content: format!("Blocked by pre-execution hook: {reason}").into(),
is_error: true,
};
self.completed_execution(
agent,
call,
descriptor,
result,
RoundEffect::default(),
None,
)
}
fn mixed_hook_blocked_execution(
&self,
agent: &Agent,
call: &ToolCall,
descriptor: &RuntimeToolDescriptor,
reason: &str,
) -> CompletedToolExecution {
self.emit_tool_execution_blocked(call, reason);
let result = ContentBlock::ToolResult {
tool_use_id: call.id.clone(),
content: format!("Blocked by mixed execution hook: {reason}").into(),
is_error: true,
};
self.completed_execution(
agent,
call,
descriptor,
result,
RoundEffect::default(),
None,
)
}
fn schema_violation_execution(
&self,
agent: &Agent,
call: &ToolCall,
error: String,
) -> CompletedToolExecution {
let result = ContentBlock::ToolResult {
tool_use_id: call.id.clone(),
content: format!("Invalid input for '{}': {error}", call.name).into(),
is_error: true,
};
agent.emit_event(AgentEvent::ToolExecutionFinished {
result: result.clone(),
});
CompletedToolExecution {
result,
task_succeeded: false,
should_end_turn: false,
terminated: false,
tool_name: call.name.clone(),
input_json: serde_json::to_string(&call.input).unwrap_or_default(),
details: None,
execution_hooks: None,
}
}
fn emit_tool_execution_blocked(&self, call: &ToolCall, reason: &str) {
let _ = self
.runtime
.emit_hook(RuntimeHookEvent::ToolExecutionBlocked {
agent_id: self.agent_id.clone(),
tool_name: call.name.clone(),
tool_call_id: call.id.clone(),
reason: reason.to_string(),
});
}
fn schema_violation(
&self,
call: &ToolCall,
descriptor: &RuntimeToolDescriptor,
) -> Option<String> {
if descriptor.terminal {
return None;
}
crate::tool::schema::validate_tool_input(&descriptor.provider.input_schema, &call.input)
.err()
.map(|error| error.to_string())
}
fn unavailable_tool_result(&self, call: ToolCall) -> ContentBlock {
ContentBlock::ToolResult {
tool_use_id: call.id,
content: format!("Tool '{}' is not available for this agent", call.name).into(),
is_error: true,
}
}
fn unavailable_tool_execution(&self, agent: &Agent, call: ToolCall) -> CompletedToolExecution {
let result = self.unavailable_tool_result(call.clone());
agent.emit_event(AgentEvent::ToolExecutionFinished {
result: result.clone(),
});
CompletedToolExecution {
result,
task_succeeded: false,
should_end_turn: false,
terminated: false,
tool_name: call.name,
input_json: serde_json::to_string(&call.input).unwrap_or_default(),
details: None,
execution_hooks: None,
}
}
fn missing_tool_execution(&self, agent: &Agent, call: ToolCall) -> CompletedToolExecution {
let result = ContentBlock::ToolResult {
tool_use_id: call.id.clone(),
content: "Tool not found".into(),
is_error: true,
};
agent.emit_event(AgentEvent::ToolExecutionFinished {
result: result.clone(),
});
CompletedToolExecution {
result,
task_succeeded: false,
should_end_turn: false,
terminated: false,
tool_name: call.name,
input_json: serde_json::to_string(&call.input).unwrap_or_default(),
details: None,
execution_hooks: None,
}
}
fn blocked_tool_result(&self, call: &ToolCall, error: RuntimeError) -> ContentBlock {
ContentBlock::ToolResult {
tool_use_id: call.id.clone(),
content: format!("Tool execution blocked: {error}").into(),
is_error: true,
}
}
fn blocked_authorization_result(
&self,
call: &ToolCall,
outcome: ToolAuthorizationOutcome,
reason: Option<String>,
rewrite_source: Option<&str>,
) -> ContentBlock {
let content = match outcome {
ToolAuthorizationOutcome::Allow => "Tool execution blocked by authorizer".to_string(),
ToolAuthorizationOutcome::Prompt => reason
.map(|reason| format!("Tool execution requires approval: {reason}"))
.unwrap_or_else(|| "Tool execution requires approval".to_string()),
ToolAuthorizationOutcome::Deny => match rewrite_source {
Some(_) => format!(
"Tool execution denied: {}",
rewritten_call_failure(
rewrite_source,
reason.unwrap_or_else(|| "denied by authorizer".to_string()),
)
),
None => reason
.map(|reason| format!("Tool execution denied: {reason}"))
.unwrap_or_else(|| "Tool execution denied by authorizer".to_string()),
},
};
ContentBlock::ToolResult {
tool_use_id: call.id.clone(),
content: content.into(),
is_error: true,
}
}
async fn tool_output_block(
&self,
call: &ToolCall,
output: Result<crate::tool::ToolOutput, String>,
rewrite_source: Option<&str>,
) -> (ContentBlock, Option<serde_json::Value>, bool) {
match output {
Ok(output) => (
ContentBlock::ToolResult {
tool_use_id: call.id.clone(),
content: self.output_limiter.apply(output.content).await,
is_error: false,
},
output.details,
output.terminate,
),
Err(content) => (
ContentBlock::ToolResult {
tool_use_id: call.id.clone(),
content: self
.output_limiter
.apply(mentra_provider::ToolResultContent::Text(
rewritten_call_failure(rewrite_source, content),
))
.await,
is_error: true,
},
None,
false,
),
}
}
fn completed_execution(
&self,
agent: &Agent,
call: &ToolCall,
descriptor: &RuntimeToolDescriptor,
result: ContentBlock,
effect: RoundEffect,
details: Option<serde_json::Value>,
) -> CompletedToolExecution {
self.emit_tool_runtime_finished(call, &result, details.clone());
agent.emit_event(AgentEvent::ToolExecutionFinished {
result: result.clone(),
});
let task_succeeded = matches!(
&result,
ContentBlock::ToolResult {
is_error: false,
..
}
) && descriptor
.capabilities
.iter()
.any(|capability| matches!(capability, ToolCapability::TaskMutation));
CompletedToolExecution {
result,
task_succeeded,
should_end_turn: effect.should_end_turn,
terminated: effect.terminated,
tool_name: call.name.clone(),
input_json: serde_json::to_string(&call.input).unwrap_or_default(),
details,
execution_hooks: None,
}
}
fn working_directory(&mut self) -> std::path::PathBuf {
if let Some(path) = &self.working_directory {
return path.clone();
}
let path = self
.runtime
.resolve_working_directory(&self.agent_id, None)
.unwrap_or_else(|_| self.runtime.default_working_directory(&self.agent_id));
self.working_directory = Some(path.clone());
path
}
fn parallel_tool_context(
&mut self,
agent: &Agent,
options: &RunOptions,
call: &ToolCall,
) -> ParallelToolContext {
ParallelToolContext {
agent_id: self.agent_id.clone(),
tool_call_id: call.id.clone(),
tool_name: call.name.clone(),
working_directory: self.working_directory(),
runtime: self.runtime.clone(),
subagent_template: agent.disposable_subagent_template(),
agent_name: agent.name().to_string(),
model: agent.model().to_string(),
history_len: agent.history().len(),
tasks: agent.tasks().to_vec(),
event_tx: agent.event_sender(),
run_options: options.clone(),
}
}
async fn authorize_tool_call(
&self,
call: &ToolCall,
tool: &Arc<dyn ExecutableTool>,
ctx: &ParallelToolContext,
execution_category: ToolExecutionCategory,
rewrite_source: Option<&str>,
) -> Result<Option<ContentBlock>, RuntimeError> {
let Some(authorizer) = self.runtime.execution.tool_authorizer.clone() else {
return Ok(None);
};
let preview = match tool.authorization_preview(ctx, &call.input) {
Ok(preview) => preview,
Err(error) => {
return Ok(Some(self.blocked_authorization_result(
call,
ToolAuthorizationOutcome::Deny,
Some(error),
rewrite_source,
)));
}
};
let preview = crate::tool::ToolAuthorizationPreview {
execution_category,
..preview
};
self.emit_tool_authorization_started(call, preview.clone())?;
let request = ToolAuthorizationRequest {
agent_id: self.agent_id.clone(),
agent_name: ctx.agent_name().to_string(),
model: ctx.model().to_string(),
history_len: ctx.history_len(),
tool_call_id: call.id.clone(),
tool_name: call.name.clone(),
preview,
};
ctx.run_options.check_limits()?;
let timeout = authorizer.timeout();
let authorization = authorizer.authorize(&request);
let timeout_wait = async move {
match timeout {
Some(timeout) => tokio::time::sleep(timeout).await,
None => std::future::pending().await,
}
};
let hard_limit = wait_for_hard_run_limit(&ctx.run_options);
tokio::pin!(authorization, timeout_wait, hard_limit);
let result = tokio::select! {
result = &mut authorization => result,
() = &mut timeout_wait => {
let timeout = timeout.expect("a disabled timeout never completes");
return self.handle_authorization_block(
call,
ToolAuthorizationOutcome::Deny,
Some(format!(
"authorizer timed out after {}",
format_duration(timeout)
)),
rewrite_source,
);
}
error = &mut hard_limit => return Err(error),
};
match result {
Ok(decision) => match decision.outcome {
ToolAuthorizationOutcome::Allow => {
self.emit_tool_authorization_finished(call, decision.outcome, decision.reason)?;
Ok(None)
}
outcome => {
self.handle_authorization_block(call, outcome, decision.reason, rewrite_source)
}
},
Err(error) => self.handle_authorization_block(
call,
ToolAuthorizationOutcome::Deny,
Some(error.to_string()),
rewrite_source,
),
}
}
fn handle_authorization_block(
&self,
call: &ToolCall,
outcome: ToolAuthorizationOutcome,
reason: Option<String>,
rewrite_source: Option<&str>,
) -> Result<Option<ContentBlock>, RuntimeError> {
self.emit_tool_authorization_finished(call, outcome, reason.clone())?;
self.emit_tool_authorization_blocked(call, outcome, reason.clone())?;
Ok(Some(self.blocked_authorization_result(
call,
outcome,
reason,
rewrite_source,
)))
}
async fn execute_one_tool(
&mut self,
agent: &mut Agent,
options: &RunOptions,
scheduled: ScheduledToolCall,
) -> Result<CompletedToolExecution, RuntimeError> {
let ScheduledToolCall {
call,
tool,
execution_category,
} = scheduled;
self.note_tool_started(agent, &call)?;
match tool {
ScheduledTool::Unavailable => Ok(self.unavailable_tool_execution(agent, call)),
ScheduledTool::Missing => Ok(self.missing_tool_execution(agent, call)),
ScheduledTool::Resolved(tool) => Ok(self
.execute_registered_tool(agent, options, call, *tool, execution_category)
.await),
}
}
async fn execute_parallel_batch(
&mut self,
agent: &mut Agent,
options: &RunOptions,
calls: Vec<ScheduledToolCall>,
) -> Result<Vec<CompletedToolExecution>, RuntimeError> {
let len = calls.len();
let mut results = (0..len).map(|_| None).collect::<Vec<_>>();
let mut join_set = JoinSet::new();
for (index, scheduled) in calls.into_iter().enumerate() {
let ScheduledToolCall {
mut call,
tool,
execution_category,
} = scheduled;
if let Err(error) = self.note_tool_started(agent, &call) {
join_set.abort_all();
return Err(error);
}
let resolved = match tool {
ScheduledTool::Resolved(tool) => *tool,
ScheduledTool::Unavailable => {
results[index] = Some(self.unavailable_tool_execution(agent, call));
continue;
}
ScheduledTool::Missing => {
results[index] = Some(self.missing_tool_execution(agent, call));
continue;
}
};
let descriptor = resolved.descriptor().clone();
let tool = resolved.handler;
let admitted = match self
.admit_call(
agent,
options,
&mut call,
&tool,
&descriptor,
execution_category,
)
.await?
{
Admission::Run(admitted) => *admitted,
Admission::Refused(execution) => {
results[index] = Some(*execution);
continue;
}
};
let AdmittedToolCall {
context: ctx,
execution_hooks,
rewrite_source,
} = admitted;
if let Err(error) = self.emit_tool_runtime_started(&call) {
let result = self.blocked_tool_result(&call, error);
let execution = self.completed_execution(
agent,
&call,
&descriptor,
result,
RoundEffect::default(),
None,
);
results[index] = Some(execution);
continue;
}
join_set.spawn(async move {
let output = execute_tool_future(
&call.name,
descriptor.execution_timeout,
tool.execute_output(ctx, call.input.clone()),
)
.await;
(
index,
call,
descriptor,
output,
execution_hooks,
rewrite_source,
)
});
}
while !join_set.is_empty() {
if let Err(error) = options.check_limits() {
join_set.abort_all();
return Err(error);
}
match tokio::time::timeout(PARALLEL_JOIN_POLL_INTERVAL, join_set.join_next()).await {
Ok(Some(Ok((
index,
call,
descriptor,
output,
execution_hooks,
rewrite_source,
)))) => {
let (result, details, terminate) = self
.tool_output_block(&call, output, rewrite_source.as_deref())
.await;
let (result, details) = if terminate {
eprintln!(
"warning: tool '{}' requested termination from a parallel \
execution; rejecting as a misuse error, run continues",
call.name
);
(parallel_termination_rejected(&call), None)
} else {
(result, details)
};
let mut execution = self.completed_execution(
agent,
&call,
&descriptor,
result,
RoundEffect::default(),
details,
);
execution.execution_hooks = execution_hooks;
results[index] = Some(execution);
}
Ok(Some(Err(error))) => {
join_set.abort_all();
return Err(RuntimeError::Store(format!(
"parallel tool task failed: {error}"
)));
}
Ok(None) => break,
Err(_) => continue,
}
}
if let Err(error) = options.check_limits() {
join_set.abort_all();
return Err(error);
}
let mut ordered = Vec::with_capacity(len);
for result in results {
ordered.push(result.ok_or_else(|| {
RuntimeError::Store("parallel tool batch lost a result".to_string())
})?);
}
Ok(ordered)
}
async fn execute_registered_tool(
&mut self,
agent: &mut Agent,
options: &RunOptions,
mut call: ToolCall,
resolved: ResolvedTool,
execution_category: ToolExecutionCategory,
) -> CompletedToolExecution {
let descriptor = resolved.descriptor().clone();
let tool = resolved.handler;
let admitted = match self
.admit_call(
agent,
options,
&mut call,
&tool,
&descriptor,
execution_category,
)
.await
{
Ok(Admission::Run(admitted)) => *admitted,
Ok(Admission::Refused(execution)) => return *execution,
Err(error) => {
let result = self.blocked_tool_result(&call, error);
return self.completed_execution(
agent,
&call,
&descriptor,
result,
RoundEffect::default(),
None,
);
}
};
let AdmittedToolCall {
context: authorization_ctx,
execution_hooks,
rewrite_source,
} = admitted;
if let Err(error) = self.emit_tool_runtime_started(&call) {
let result = self.blocked_tool_result(&call, error);
return self.completed_execution(
agent,
&call,
&descriptor,
result,
RoundEffect::default(),
None,
);
}
let working_directory = authorization_ctx.working_directory.clone();
let runtime = authorization_ctx.runtime.clone();
let event_tx = agent.event_sender();
let (result, details, terminate) = self
.tool_output_block(
&call,
execute_tool_future(
&call.name,
descriptor.execution_timeout,
tool.execute_mut_output(
ToolContext {
agent_id: self.agent_id.clone(),
tool_call_id: call.id.clone(),
tool_name: call.name.clone(),
working_directory,
runtime,
agent,
event_tx,
run_options: options.clone(),
},
call.input.clone(),
),
)
.await,
rewrite_source.as_deref(),
)
.await;
let effect = RoundEffect {
should_end_turn: agent.take_idle_requested() || terminate,
terminated: terminate,
};
let mut execution =
self.completed_execution(agent, &call, &descriptor, result, effect, details);
execution.execution_hooks = execution_hooks;
execution
}
}
fn mixed_rewrite_source(attribution: Option<&str>) -> String {
match attribution {
Some(attribution) => format!("mixed execution hooks ({attribution})"),
None => "mixed execution hooks".to_string(),
}
}
fn rewritten_call_failure(rewrite_source: Option<&str>, failure: String) -> String {
match rewrite_source {
Some(rewrite_source) => {
format!("{rewrite_source} rewrote this call; the rewritten call then failed: {failure}")
}
None => failure,
}
}
impl ToolCallSchedule {
fn new(runtime: &ToolRuntime, agent: &Agent, calls: Vec<ToolCall>) -> Self {
let mut batches = Vec::new();
let mut pending_parallel = Vec::new();
for call in calls {
let scheduled = runtime.schedule_call(agent, call);
match scheduled.execution_category {
ToolExecutionCategory::ReadOnlyParallel => pending_parallel.push(scheduled),
ToolExecutionCategory::ExclusiveLocalMutation
| ToolExecutionCategory::ExclusivePersistentMutation
| ToolExecutionCategory::BackgroundJob
| ToolExecutionCategory::Delegation => {
if !pending_parallel.is_empty() {
batches.push(ToolCallBatch::Parallel(std::mem::take(
&mut pending_parallel,
)));
}
batches.push(ToolCallBatch::Exclusive(Box::new(scheduled)));
}
}
}
if !pending_parallel.is_empty() {
batches.push(ToolCallBatch::Parallel(pending_parallel));
}
Self { batches }
}
}
impl ToolCallBatch {
fn execution_count(&self) -> usize {
match self {
ToolCallBatch::Exclusive(_) => 1,
ToolCallBatch::Parallel(calls) => calls.len(),
}
}
fn into_calls(self) -> Vec<ToolCall> {
match self {
ToolCallBatch::Exclusive(call) => vec![call.call],
ToolCallBatch::Parallel(calls) => {
calls.into_iter().map(|scheduled| scheduled.call).collect()
}
}
}
}
fn not_executed_result(call: &ToolCall, terminated_by: &str) -> ContentBlock {
ContentBlock::ToolResult {
tool_use_id: call.id.clone(),
content: format!("not executed: run terminated by '{terminated_by}'").into(),
is_error: true,
}
}
fn parallel_termination_rejected(call: &ToolCall) -> ContentBlock {
ContentBlock::ToolResult {
tool_use_id: call.id.clone(),
content: format!(
"not honored: tool '{}' requested termination from a parallel execution; \
termination is only honored from an exclusive execution",
call.name
)
.into(),
is_error: true,
}
}
async fn wait_for_hard_run_limit(options: &RunOptions) -> RuntimeError {
if options.cancellation.is_none() && options.deadline.is_none() {
return std::future::pending().await;
}
loop {
if let Err(error) = options.check_limits() {
return error;
}
tokio::time::sleep(PARALLEL_JOIN_POLL_INTERVAL).await;
}
}
async fn execute_tool_future<F, T>(
tool_name: &str,
execution_timeout: Option<Duration>,
future: F,
) -> Result<T, String>
where
F: Future<Output = Result<T, String>>,
{
match execution_timeout {
Some(timeout) => match tokio::time::timeout(timeout, future).await {
Ok(result) => result,
Err(_) => Err(format!(
"Tool '{tool_name}' timed out after {}",
format_duration(timeout)
)),
},
None => future.await,
}
}
fn format_duration(duration: Duration) -> String {
if duration.as_secs() > 0 && duration.subsec_nanos() == 0 {
format!("{}s", duration.as_secs())
} else if duration.as_millis() > 0 {
format!("{}ms", duration.as_millis())
} else if duration.as_micros() > 0 {
format!("{}us", duration.as_micros())
} else {
format!("{}ns", duration.as_nanos())
}
}