1use anyhow::{Context, Result, anyhow};
4use serde_json::{Value, json};
5use std::borrow::Cow;
6use std::future::Future;
7use std::time::{Duration, Instant};
8use tracing::{trace, warn};
9use vtcode_commons::ErrorCategory;
10
11use crate::config::constants::tools;
12use crate::core::agent::harness_kernel::PreparedToolCall;
13use crate::core::memory_pool::SizeRecommendation;
14use crate::tool_policy::ToolExecutionDecision;
15use crate::tools::error_messages::agent_execution;
16use crate::tools::request_response::{ToolCallRequest, ToolCallResponse};
17use crate::tools::tool_intent;
18use crate::tools::unified_error::UnifiedErrorKind;
19use crate::tools::unified_error::UnifiedToolError;
20use crate::ui::search::fuzzy_match;
21
22use super::assembly::public_tool_name_candidates;
23use super::execution_kernel;
24use super::reentrancy::ToolReentrancyGuard;
25use super::{
26 ExecSettlementMode, ExecutionPolicySnapshot, ToolErrorType, ToolExecutionError, ToolExecutionOutcome,
27 ToolExecutionRecord, ToolExecutionRequest, ToolRegistry,
28};
29use vtcode_config::constants::execution::{LOOP_THROTTLE_MAX_MS, LOOP_THROTTLE_REGISTRY_BASE_MS};
30
31const LOOP_HARD_BLOCK_REPEAT_COUNT: usize = 5;
35
36impl ToolRegistry {
37 fn annotate_timeout_error_payload(
38 payload: &mut Value,
39 timeout_category: &str,
40 timeout_ms: u64,
41 circuit_breaker: bool,
42 ) {
43 if let Some(obj) = payload.get_mut("error").and_then(|value| value.as_object_mut()) {
44 obj.insert("timeout_category".into(), Value::String(timeout_category.to_string()));
45 obj.insert("timeout_ms".into(), Value::from(timeout_ms));
46 obj.insert("circuit_breaker".into(), Value::Bool(circuit_breaker));
47 }
48 }
49
50 pub fn safety_gateway(&self) -> std::sync::Arc<crate::tools::safety_gateway::SafetyGateway> {
51 std::sync::Arc::clone(&self.safety_gateway)
52 }
53
54 pub fn execute_public_tool_request(
57 &self,
58 request: ToolExecutionRequest,
59 ) -> impl Future<Output = ToolExecutionOutcome> + '_ {
60 self.execute_tool_request_internal(request)
61 }
62
63 pub async fn execute_prepared_public_tool_request(
64 &self,
65 prepared: &PreparedToolCall,
66 policy: ExecutionPolicySnapshot,
67 ) -> ToolExecutionOutcome {
68 let request = ToolExecutionRequest::new(prepared.canonical_name.clone(), prepared.effective_args.clone())
69 .with_policy(
70 policy
71 .with_prevalidated(prepared.already_preflighted)
72 .with_safety_prevalidated(false),
73 );
74 self.execute_tool_request_internal(request).await
75 }
76
77 async fn should_skip_loop_detection_for_exec_continuation(&self, tool_name: &str, args: &Value) -> bool {
78 if tool_name == tools::WRITE_STDIN {
79 return matches!(
80 crate::tools::command_args::write_stdin_dispatch(args),
81 Ok(crate::tools::command_args::WriteStdinDispatch::Poll
82 | crate::tools::command_args::WriteStdinDispatch::Wait,)
83 );
84 }
85
86 if tool_name != tools::UNIFIED_EXEC {
87 return false;
88 }
89
90 if !tool_intent::command_session_action_in(args, &["poll", "continue"]) {
91 return false;
92 }
93 if tool_intent::command_session_action_is(args, "continue")
94 && crate::tools::command_args::interactive_input_text(args).is_some()
95 {
96 return false;
97 }
98
99 let Some(session_id) = crate::tools::command_args::session_id_text(args) else {
100 return false;
101 };
102
103 matches!(self.exec_session_completed(session_id).await, Ok(None))
104 }
105
106 async fn public_tool_catalog_for_error(&self, requested_name: &str) -> (Vec<String>, Vec<String>) {
107 let mut tool_names = self.available_tools().await;
108 tool_names.sort_unstable();
109 tool_names.dedup();
110
111 let requested_candidates = public_tool_name_candidates(requested_name);
112 let mut similar_tools = Vec::new();
113
114 if let Ok(resolved) = self.resolve_public_tool_name_sync(requested_name)
115 && tool_names.iter().any(|tool| tool == &resolved)
116 {
117 similar_tools.push(resolved);
118 }
119
120 for tool in &tool_names {
121 if similar_tools.len() >= 3 {
122 break;
123 }
124
125 if similar_tools.iter().any(|candidate| candidate == tool) {
126 continue;
127 }
128
129 if requested_candidates.iter().any(|candidate| fuzzy_match(candidate, tool)) {
130 similar_tools.push(tool.clone());
131 }
132 }
133
134 (tool_names, similar_tools)
135 }
136
137 pub fn preflight_validate_call(&self, name: &str, args: &Value) -> Result<super::ToolPreflightOutcome> {
138 execution_kernel::preflight_validate_call(self, name, args)
139 }
140
141 pub fn preflight_validate_harness_call(&self, name: &str, args: &Value) -> Result<super::ToolPreflightOutcome> {
145 execution_kernel::preflight_validate_call_with_mode(self, name, args, execution_kernel::DispatchMode::Harness)
146 }
147
148 pub fn admit_public_tool_call(&self, name: &str, args: &Value) -> Result<PreparedToolCall> {
149 let preflight = self.preflight_validate_harness_call(name, args)?;
150 Ok(PreparedToolCall::new(
151 preflight.normalized_tool_name,
152 preflight.readonly_classification,
153 preflight.parallel_safe_after_preflight,
154 preflight.effective_args,
155 ))
156 }
157
158 pub async fn execute_tool(&self, name: &str, args: Value) -> Result<Value> {
159 self.execute_tool_ref(name, &args).await
160 }
161
162 pub async fn execute_public_tool_ref(&self, name: &str, args: &Value) -> Result<Value> {
164 self.execute_public_tool_ref_internal(name, args, false).await
165 }
166
167 pub async fn execute_tool_ref(&self, name: &str, args: &Value) -> Result<Value> {
170 self.execute_tool_ref_internal(name, args, false, ExecSettlementMode::Manual)
171 .await
172 }
173
174 pub async fn execute_tool_ref_prevalidated(&self, name: &str, args: &Value) -> Result<Value> {
179 self.execute_tool_ref_internal(name, args, true, ExecSettlementMode::Manual)
180 .await
181 }
182
183 pub async fn execute_public_tool_ref_prevalidated(&self, name: &str, args: &Value) -> Result<Value> {
185 self.execute_public_tool_ref_prevalidated_with_mode(name, args, ExecSettlementMode::Manual)
186 .await
187 }
188
189 #[doc(hidden)]
190 pub async fn execute_public_tool_ref_prevalidated_with_mode(
191 &self,
192 name: &str,
193 args: &Value,
194 exec_settlement_mode: ExecSettlementMode,
195 ) -> Result<Value> {
196 self.execute_public_tool_ref_internal_with_mode(name, args, true, exec_settlement_mode)
197 .await
198 }
199
200 pub async fn execute_prepared_public_tool_ref_with_mode(
201 &self,
202 prepared: &PreparedToolCall,
203 exec_settlement_mode: ExecSettlementMode,
204 ) -> Result<Value> {
205 self.execute_public_tool_ref_dispatch(
209 prepared.canonical_name.as_str(),
210 &prepared.effective_args,
211 prepared.already_preflighted,
212 execution_kernel::DispatchMode::Harness,
213 exec_settlement_mode,
214 )
215 .await
216 }
217
218 async fn execute_public_tool_ref_internal(&self, name: &str, args: &Value, prevalidated: bool) -> Result<Value> {
219 self.execute_public_tool_ref_dispatch(
220 name,
221 args,
222 prevalidated,
223 execution_kernel::DispatchMode::ModelPublic,
224 ExecSettlementMode::Manual,
225 )
226 .await
227 }
228
229 async fn execute_public_tool_ref_internal_with_mode(
230 &self,
231 name: &str,
232 args: &Value,
233 prevalidated: bool,
234 exec_settlement_mode: ExecSettlementMode,
235 ) -> Result<Value> {
236 self.execute_public_tool_ref_dispatch(
237 name,
238 args,
239 prevalidated,
240 execution_kernel::DispatchMode::ModelPublic,
241 exec_settlement_mode,
242 )
243 .await
244 }
245
246 pub(super) async fn execute_public_tool_ref_dispatch(
256 &self,
257 name: &str,
258 args: &Value,
259 prevalidated: bool,
260 dispatch_mode: execution_kernel::DispatchMode,
261 exec_settlement_mode: ExecSettlementMode,
262 ) -> Result<Value> {
263 let routed_name = execution_kernel::resolve_dispatch_target(self, name, dispatch_mode)
264 .map_err(|err| anyhow!(err.to_string()))?;
265 let effective_args = execution_kernel::remap_public_file_operation_alias_args(name, routed_name.as_str(), args)
266 .or_else(|| execution_kernel::remap_consolidated_action_alias_args(name, routed_name.as_str(), args));
267 self.execute_tool_ref_internal(
268 routed_name.as_str(),
269 effective_args.as_ref().unwrap_or(args),
270 prevalidated,
271 exec_settlement_mode,
272 )
273 .await
274 }
275
276 async fn execute_tool_ref_internal(
277 &self,
278 name: &str,
279 args: &Value,
280 prevalidated: bool,
281 exec_settlement_mode: ExecSettlementMode,
282 ) -> Result<Value> {
283 let resolved = self.resolve_tool_name_with_display(name);
284 self.enforce_matrix_role(&resolved.canonical)?;
285 if let Err(error) = crate::core::agent::snapshots::declare_prompt_edit(
286 self.harness_context_snapshot().session_id,
287 name.to_owned(),
288 args.clone(),
289 )
290 .await
291 {
292 tracing::warn!(
293 tool = %name,
294 error = %error,
295 "Checkpoint pre-image capture failed; rewind may not restore this edit"
296 );
297 }
298 let _pool_guard = if self.optimization_config.memory_pool.enabled {
300 Some(self.memory_pool.get_string())
301 } else {
302 None
303 };
304
305 if self.optimization_config.memory_pool.enabled {
307 let recommendation = self.memory_pool.auto_tune(&self.optimization_config.memory_pool);
308
309 if !matches!(
311 (
312 recommendation.string_size_recommendation,
313 recommendation.value_size_recommendation,
314 recommendation.vec_size_recommendation
315 ),
316 (SizeRecommendation::Maintain, SizeRecommendation::Maintain, SizeRecommendation::Maintain)
317 ) {
318 tracing::debug!(
319 "Memory pool tuning recommendation: string={:?}, value={:?}, vec={:?}, allocations_avoided={}",
320 recommendation.string_size_recommendation,
321 recommendation.value_size_recommendation,
322 recommendation.vec_size_recommendation,
323 recommendation.total_allocations_avoided
324 );
325 }
326 }
327
328 let resolved_name = self.resolve_tool_name_with_display(name);
329 let tool_name = resolved_name.canonical;
330 let tool_name_owned = tool_name.clone();
331 let display_name = resolved_name.display;
332
333 let cached_tool = if self.optimization_config.tool_registry.use_optimized_registry {
339 let cache = self.hot_tool_cache.read();
340 cache.peek(&tool_name).cloned()
341 } else {
342 None
343 };
344
345 if let Some(tool_arc) = cached_tool.as_ref()
347 && self.optimization_config.tool_registry.use_optimized_registry
348 && tool_name != name
349 {
350 self.hot_tool_cache.write().put(tool_name.clone(), tool_arc.clone());
352 }
353
354 let execution_args = self.prepare_execution_args(&tool_name, args)?;
355 let is_verification_command = execution_args.is_verification_command;
356 let max_output_tokens = execution_args.max_output_tokens;
357 let args = execution_args.handler_args.as_ref();
358 let requested_name = name.to_string();
359
360 let args_for_recording = args.clone();
362 let context_snapshot = self.harness_context_snapshot();
364 let record_failure = |tool_name: String,
365 is_mcp_tool: bool,
366 mcp_provider: Option<String>,
367 args: Value,
368 error_msg: String,
369 timeout_category: Option<String>,
370 base_timeout_ms: Option<u64>,
371 adaptive_timeout_ms: Option<u64>,
372 effective_timeout_ms: Option<u64>,
373 circuit_breaker: bool| {
374 self.execution_history.add_record(ToolExecutionRecord::failure(
375 tool_name,
376 requested_name.clone(),
377 is_mcp_tool,
378 mcp_provider,
379 args,
380 error_msg,
381 context_snapshot.clone(),
382 timeout_category,
383 base_timeout_ms,
384 adaptive_timeout_ms,
385 effective_timeout_ms,
386 circuit_breaker,
387 ));
388 };
389
390 let allow_parallel_sibling = prevalidated && tool_intent::is_parallel_safe_call(&tool_name, args);
391 let _reentrancy_guard = match ToolReentrancyGuard::enter(&tool_name, allow_parallel_sibling) {
392 Ok(guard) => guard,
393 Err(violation) => {
394 let reentry_count = violation.tool_reentry_count + 1;
395 let error_message = format!(
396 "Reentrancy guard: tool '{}' is already running in this call stack, so this recursive call was blocked. \
397 Repeating the same call is blocked the same way; change the control flow or use a different tool.\n\
398 Current stack depth: {}. Re-entry count for this tool in the current task: {}.\n\
399 Stack trace: {}",
400 display_name, violation.stack_depth, reentry_count, violation.stack_trace
401 );
402 let error = ToolExecutionError::new(
403 tool_name_owned.clone(),
404 ToolErrorType::PolicyViolation,
405 error_message.clone(),
406 );
407 let mut payload = error.to_json_value();
408 if let Some(obj) = payload.as_object_mut() {
409 obj.insert("reentrant_call_blocked".into(), json!(true));
410 obj.insert("stack_depth".into(), json!(violation.stack_depth));
411 obj.insert("reentry_count".into(), json!(reentry_count));
412 obj.insert("tool".into(), json!(display_name));
413 obj.insert("stack_trace".into(), json!(violation.stack_trace));
414 }
415 record_failure(
416 tool_name_owned.clone(),
417 false,
418 None,
419 args_for_recording.clone(),
420 error_message.clone(),
421 None,
422 None,
423 None,
424 None,
425 false,
426 );
427 return Err(anyhow!(error_message).context("tool reentrancy blocked"));
428 }
429 };
430
431 let (intent, readonly_classification) = if prevalidated {
437 #[cfg(debug_assertions)]
438 {
439 if let Err(err) = execution_kernel::preflight_validate_resolved_call(self, &tool_name, args)
440 && !agent_execution::is_planning_active_denial(&err.to_string())
441 {
442 debug_assert!(false, "prevalidated execution received invalid call for '{tool_name}': {err}");
443 }
444 }
445 let intent = tool_intent::classify_tool_intent(&tool_name, args);
446 (intent, !intent.mutating)
447 } else {
448 match execution_kernel::preflight_validate_resolved_call(self, &tool_name, args) {
449 Ok(outcome) => (outcome.intent, outcome.readonly_classification),
450 Err(err) => {
451 let err_msg = err.to_string();
452 record_failure(
453 tool_name_owned.clone(),
454 false,
455 None,
456 args_for_recording.clone(),
457 err_msg,
458 None,
459 None,
460 None,
461 None,
462 false,
463 );
464 return Err(err);
465 }
466 }
467 };
468
469 if readonly_classification {
470 trace!(tool = %tool_name, "Validation classified tool as read-only");
471 }
472
473 if self.is_planning_active() && !self.is_planning_active_allowed_with_intent(&tool_name, args, &intent) {
477 let error_msg = agent_execution::planning_workflow_denial_message(&display_name);
478 record_failure(
479 tool_name_owned.clone(),
480 false,
481 None,
482 args_for_recording.clone(),
483 error_msg.clone(),
484 None,
485 None,
486 None,
487 None,
488 false,
489 );
490 return Err(anyhow!(error_msg).context(agent_execution::PLANNING_DENIED_CONTEXT));
491 }
492
493 let shared_circuit_breaker = self.shared_circuit_breaker();
494 if let Some(breaker) = shared_circuit_breaker.as_ref()
495 && !breaker.allow_request_for_tool(&tool_name)
496 {
497 let diagnostics = breaker.get_diagnostics(&tool_name);
498 let retry_after = diagnostics
499 .remaining_backoff
500 .map(|backoff| format!(" retry_after={}s.", backoff.as_secs()))
501 .unwrap_or_default();
502 let error_msg = format!(
503 "Tool '{display_name}' is temporarily disabled due to high failure rate (Circuit Breaker OPEN).{retry_after}"
504 );
505 self.execution_history.add_record(
506 ToolExecutionRecord::failure(
507 tool_name_owned.clone(),
508 requested_name.clone(),
509 false,
510 None,
511 args_for_recording.clone(),
512 error_msg.clone(),
513 context_snapshot.clone(),
514 None,
515 None,
516 None,
517 None,
518 true,
519 )
520 .with_circuit_breaker_state(format!("{:?}", diagnostics.status))
521 .with_retry_after(diagnostics.remaining_backoff),
522 );
523 return Err(anyhow!(error_msg).context("tool denied by circuit breaker"));
524 }
525
526 let timeout_category = self.timeout_category_for_args(&tool_name, args).await;
527
528 if let Some(backoff) = self.should_circuit_break(timeout_category) {
529 warn!(
530 tool = %tool_name,
531 category = %timeout_category.label(),
532 delay_ms = %backoff.as_millis(),
533 "Circuit breaker active for tool category; backing off before execution"
534 );
535 tokio::time::sleep(backoff).await;
536 }
537
538 let execution_span = tracing::debug_span!(
539 "tool_execution",
540 tool = %tool_name,
541 requested = %name,
542 session_id = %context_snapshot.session_id,
543 task_id = %context_snapshot.task_id.as_deref().unwrap_or("")
544 );
545 let _span_guard = execution_span.enter();
546
547 trace!(
548 tool = %tool_name,
549 session_id = %context_snapshot.session_id,
550 task_id = %context_snapshot.task_id.as_deref().unwrap_or(""),
551 "Executing tool with harness context"
552 );
553
554 if tool_name != name {
555 trace!(
556 requested = %name,
557 canonical = %tool_name,
558 "Resolved tool alias to canonical name"
559 );
560 }
561
562 let base_timeout_ms = self
563 .timeout_policy
564 .read()
565 .ceiling_for(timeout_category)
566 .map(|d| d.as_millis() as u64);
567 let adaptive_timeout_ms = self
568 .resiliency
569 .lock()
570 .adaptive_timeout_ceiling
571 .get(&timeout_category)
572 .filter(|d| d.as_millis() > 0)
573 .map(|d| d.as_millis() as u64);
574 let timeout_category_label = Some(timeout_category.label().to_string());
575
576 if let Some(rate_limit) = self.execution_history.rate_limit_per_minute() {
577 let calls_last_minute = self.execution_history.calls_in_window(Duration::from_secs(60));
578 if calls_last_minute >= rate_limit {
579 warn!(
580 tool = %tool_name_owned,
581 requested = %requested_name,
582 calls_last_minute,
583 rate_limit,
584 "Execution history rate-limit threshold exceeded (observability-only)"
585 );
586 }
587 }
588
589 let fresh_patch_read = self.consume_patch_recovery_read(&tool_name, args);
590 let matrix_verification = self
593 .matrix_worker
594 .read()
595 .as_ref()
596 .is_some_and(|worker| worker.assignment.phase == crate::exec::events::matrix::MatrixPhase::Verify);
597 let reusable_result = readonly_classification
598 && !matrix_verification
599 && !matches!(tool_name.as_str(), tools::RECORD_DECISION | tools::TASK_TRACKER | tools::MATRIX);
600 let skip_loop_detection = self.should_skip_loop_detection_for_exec_continuation(&tool_name, args).await;
601 if skip_loop_detection {
602 trace!(
603 tool = %tool_name,
604 "Skipping identical-call loop detection for stateful exec continuation"
605 );
606 }
607
608 if reusable_result && !is_verification_command && !skip_loop_detection && !fresh_patch_read {
623 let fast_reuse_max_age = Duration::from_secs(60);
624 let fast_reused = self
625 .execution_history
626 .find_recent_spooled_result(&tool_name, args, fast_reuse_max_age)
627 .or_else(|| {
628 self.execution_history
629 .find_recent_successful_result(&tool_name, args, fast_reuse_max_age)
630 });
631 if let Some(mut reused_value) = fast_reused {
632 if let Some(obj) = reused_value.as_object_mut() {
633 obj.insert("reused_recent_result".into(), json!(true));
634 obj.insert("tool".into(), json!(display_name));
635 let reused_spooled = obj.get("spool_path").and_then(|v| v.as_str()).is_some();
636 let note = if reused_spooled {
637 "Reusing a recent spooled output for this identical read-only call. Continue from the spool file instead of re-running the tool."
638 } else {
639 "Reusing a recent successful output for this identical read-only call."
640 };
641 obj.insert("reused_result_note".into(), json!(note));
642 }
643 self.execution_history.add_record(ToolExecutionRecord::success(
659 tool_name.clone(),
660 requested_name.clone(),
661 false,
662 None,
663 args_for_recording.clone(),
664 reused_value.clone(),
665 context_snapshot.clone(),
666 timeout_category_label.clone(),
667 base_timeout_ms,
668 adaptive_timeout_ms,
669 None,
670 false,
671 ));
672 trace!(
673 tool = %tool_name,
674 "Fast-reusing recent successful read-only result"
675 );
676 self.observe_source_output(&tool_name, args, &reused_value).await;
677 return Ok(reused_value);
678 }
679 }
680
681 let loop_limit = if skip_loop_detection {
683 0
684 } else {
685 self.execution_history.loop_limit_for(&tool_name, args)
686 };
687 let loop_result = if skip_loop_detection {
688 crate::tools::registry::execution_history::LoopDetectionResult {
689 detected: false,
690 repeat_count: 0,
691 tool_name: tool_name.clone(),
692 }
693 } else {
694 self.execution_history.detect_loop(&tool_name, args)
695 };
696 if loop_result.detected && loop_result.repeat_count > 1 {
697 let delay_ms = (LOOP_THROTTLE_REGISTRY_BASE_MS * loop_result.repeat_count as u64).min(LOOP_THROTTLE_MAX_MS);
698 if delay_ms > 0 {
699 tokio::time::sleep(Duration::from_millis(delay_ms)).await;
700 }
701 }
702 if loop_limit > 0 && loop_result.detected {
703 warn!(
704 tool = %tool_name,
705 repeats = loop_result.repeat_count,
706 "Loop detected: agent calling same tool with identical parameters {} times",
707 loop_result.repeat_count
708 );
709 if loop_result.repeat_count >= loop_limit {
710 let hard_block = loop_result.repeat_count >= LOOP_HARD_BLOCK_REPEAT_COUNT;
715
716 if reusable_result && !is_verification_command && !hard_block && !fresh_patch_read {
717 let reuse_max_age = Duration::from_secs(120);
718 let reused = self
719 .execution_history
720 .find_recent_spooled_result(&tool_name, args, reuse_max_age)
721 .or_else(|| {
722 self.execution_history
723 .find_recent_successful_result(&tool_name, args, reuse_max_age)
724 });
725 if let Some(mut reused_value) = reused {
726 if let Some(obj) = reused_value.as_object_mut() {
727 obj.insert("reused_recent_result".into(), json!(true));
728 obj.insert("loop_detected".into(), json!(true));
729 obj.insert("repeat_count".into(), json!(loop_result.repeat_count));
730 obj.insert("limit".into(), json!(loop_limit));
731 obj.insert("tool".into(), json!(display_name));
732 let reused_spooled = obj.get("spool_path").and_then(|v| v.as_str()).is_some();
733 let note = if reused_spooled {
734 "Loop detected: this identical read-only call has been repeated, so the earlier result was reused. The full output is in the spool file and the conversation history; further repeats return an error instead."
735 } else {
736 "Loop detected: this identical read-only call has been repeated with no new information, so the earlier result was reused. It is already in the conversation history; further repeats return an error instead."
737 };
738 obj.insert("loop_detected_note".into(), json!(note));
739 }
740 self.observe_source_output(&tool_name, args, &reused_value).await;
741 return Ok(reused_value);
742 }
743 }
744
745 let delay_ms =
746 (LOOP_THROTTLE_REGISTRY_BASE_MS * loop_result.repeat_count as u64).min(LOOP_THROTTLE_MAX_MS);
747 if delay_ms > 0 {
748 tokio::time::sleep(Duration::from_millis(delay_ms)).await;
749 }
750
751 let error = ToolExecutionError::new(
752 tool_name_owned.clone(),
753 ToolErrorType::PolicyViolation,
754 agent_execution::loop_detection_block_message(&display_name, loop_result.repeat_count as u64, None),
755 );
756 let mut payload = error.to_json_value();
757 if let Some(obj) = payload.as_object_mut() {
758 obj.insert("loop_detected".into(), json!(true));
759 obj.insert("repeat_count".into(), json!(loop_result.repeat_count));
760 obj.insert("limit".into(), json!(loop_limit));
761 obj.insert("tool".into(), json!(display_name));
762 obj.insert(
763 "next_action".into(),
764 json!("Identical calls to this tool are blocked. Use the data already in the conversation history; for different information, change the arguments or use another tool."),
765 );
766 }
767
768 record_failure(
769 tool_name_owned,
770 false,
771 None,
772 args_for_recording,
773 "Tool call blocked due to repeated identical invocations".to_string(),
774 timeout_category_label.clone(),
775 base_timeout_ms,
776 adaptive_timeout_ms,
777 None,
778 false,
779 );
780
781 return Ok(payload);
782 }
783 }
784
785 let full_auto_denied = {
786 let gateway = self.policy_gateway.clone();
787 let tool_name_ref = &tool_name;
788 async move { gateway.is_denied_in_full_auto(tool_name_ref).await }
789 };
790 let full_auto_denied = full_auto_denied.await;
791 if full_auto_denied {
792 let _error = ToolExecutionError::new(
793 tool_name_owned.clone(),
794 ToolErrorType::PolicyViolation,
795 format!("Tool '{display_name}' is not permitted while full-auto permission review is active"),
796 );
797
798 record_failure(
799 tool_name_owned.clone(),
800 false,
801 None,
802 args_for_recording.clone(),
803 "Tool execution denied by policy".to_string(),
804 timeout_category_label.clone(),
805 base_timeout_ms,
806 adaptive_timeout_ms,
807 None,
808 false,
809 );
810
811 return Err(anyhow!("Tool '{display_name}' is not permitted while full-auto permission review is active")
812 .context("tool denied by full-auto allowlist"));
813 }
814
815 let skip_policy_prompt = self.policy_gateway.take_preapproved(&tool_name).await;
816
817 let decision = if skip_policy_prompt {
818 ToolExecutionDecision::Allowed
819 } else {
820 self.policy_gateway.should_execute_tool(&tool_name).await?
821 };
822
823 if !decision.is_allowed() {
824 let error_msg = match decision {
825 ToolExecutionDecision::DeniedWithFeedback(feedback) => {
826 format!("Tool '{display_name}' denied by user: {feedback}")
827 }
828 _ => format!("Tool '{display_name}' execution denied by policy"),
829 };
830
831 let _error =
832 ToolExecutionError::new(tool_name_owned.clone(), ToolErrorType::PolicyViolation, error_msg.clone());
833
834 record_failure(
835 tool_name_owned.clone(),
836 false,
837 None,
838 args_for_recording.clone(),
839 error_msg.clone(),
840 timeout_category_label.clone(),
841 base_timeout_ms,
842 adaptive_timeout_ms,
843 None,
844 false,
845 );
846
847 return Err(anyhow!("{error_msg}").context("tool denied by policy"));
848 }
849
850 let gateway = self.policy_gateway.clone();
851 let constrained_result = gateway.apply_policy_constraints(&tool_name, args).await;
852 let mut args = match constrained_result {
853 Ok(processed_args) => processed_args,
854 Err(err) => {
855 let error = ToolExecutionError::with_original_error(
856 tool_name_owned.clone(),
857 ToolErrorType::InvalidParameters,
858 "Failed to apply policy constraints".to_string(),
859 err.to_string(),
860 );
861
862 record_failure(
863 tool_name_owned,
864 false,
865 None,
866 args_for_recording,
867 format!("Failed to apply policy constraints: {err}"),
868 timeout_category_label.clone(),
869 base_timeout_ms,
870 adaptive_timeout_ms,
871 None,
872 false,
873 );
874
875 return Err(anyhow!(error.to_json_value()).context("tool denied by policy constraints"));
876 }
877 };
878
879 let super::execution_stages::ExecutionRoute {
880 route:
881 super::execution_stages::ToolRoute {
882 needs_pty,
883 tool_exists,
884 is_mcp: is_mcp_tool,
885 mcp_provider,
886 mcp_tool_name,
887 },
888 mcp_lookup_error,
889 } = self.resolve_execution_route(name, &tool_name).await;
890
891 if !tool_exists {
893 if let Some(err) = mcp_lookup_error {
894 let error = ToolExecutionError::with_original_error(
895 tool_name_owned.clone(),
896 ToolErrorType::ExecutionError,
897 format!("Failed to resolve MCP tool '{display_name}': {err}"),
898 err.to_string(),
899 );
900
901 record_failure(
902 tool_name_owned,
903 is_mcp_tool,
904 mcp_provider.clone(),
905 args_for_recording,
906 format!("Failed to resolve MCP tool '{display_name}': {err}"),
907 timeout_category_label.clone(),
908 base_timeout_ms,
909 adaptive_timeout_ms,
910 None,
911 false,
912 );
913
914 return Ok(error.to_json_value());
915 }
916
917 let (all_tool_names, similar_tools) = self.public_tool_catalog_for_error(name).await;
918 let suggestion = if !similar_tools.is_empty() {
919 format!(" Did you mean: {}?", similar_tools.join(", "))
920 } else {
921 String::new()
922 };
923 let available_tool_list = all_tool_names.join(", ");
924 let message = format!("Unknown tool: {display_name}. Available tools: {available_tool_list}.{suggestion}");
925 let error = ToolExecutionError::new(tool_name_owned.clone(), ToolErrorType::ToolNotFound, message.clone());
926
927 record_failure(
928 tool_name_owned,
929 is_mcp_tool,
930 mcp_provider.clone(),
931 args_for_recording,
932 message,
933 timeout_category_label.clone(),
934 base_timeout_ms,
935 adaptive_timeout_ms,
936 None,
937 false,
938 );
939
940 return Ok(error.to_json_value());
941 }
942
943 if is_mcp_tool && !self.mcp_circuit_breaker.allow_request() {
945 let diag = self.mcp_circuit_breaker.diagnostics();
946 let error = ToolExecutionError::new(
947 tool_name_owned.clone(),
948 ToolErrorType::ExecutionError,
949 format!("MCP circuit breaker {:?}; skipping execution", diag.status),
950 );
951 let payload = json!({
952 "error": error.to_json_value(),
953 "circuit_breaker_state": format!("{:?}", diag.status),
954 "consecutive_failures": diag.consecutive_failures,
955 "note": "MCP provider circuit breaker open; execution skipped",
956 "last_failed_at_ago_ms": diag.last_failure_time
957 .map(|ts| ts.elapsed().as_millis() as u64),
958 "current_timeout_seconds": diag.current_timeout.as_secs(),
959 "mcp_provider": mcp_provider,
960 });
961 warn!(
962 tool = %tool_name_owned,
963 payload = %payload,
964 "Skipping MCP tool execution due to circuit breaker"
965 );
966 self.execution_history.add_record(
967 ToolExecutionRecord::failure(
968 tool_name_owned,
969 requested_name.clone(),
970 is_mcp_tool,
971 mcp_provider.clone(),
972 args_for_recording,
973 format!("MCP circuit breaker {:?}; execution skipped", diag.status),
974 context_snapshot.clone(),
975 timeout_category_label.clone(),
976 base_timeout_ms,
977 adaptive_timeout_ms,
978 None,
979 false,
980 )
981 .with_circuit_breaker_state(format!("{:?}", diag.status))
982 .with_retry_after(diag.retry_after),
983 );
984 return Ok(payload);
985 }
986
987 trace!(
988 tool = %tool_name,
989 requested = %name,
990 is_mcp = is_mcp_tool,
991 uses_pty = needs_pty,
992 alias = %if tool_name == name { "" } else { name },
993 mcp_provider = %mcp_provider.as_deref().unwrap_or(""),
994 "Resolved tool route"
995 );
996
997 let _pty_guard = if needs_pty {
999 match self.start_pty_session() {
1000 Ok(guard) => Some(guard),
1001 Err(err) => {
1002 let error = ToolExecutionError::with_original_error(
1003 tool_name_owned.clone(),
1004 ToolErrorType::ExecutionError,
1005 "Failed to start PTY session".to_string(),
1006 err.to_string(),
1007 );
1008
1009 record_failure(
1010 tool_name_owned,
1011 is_mcp_tool,
1012 mcp_provider.clone(),
1013 args_for_recording,
1014 "Failed to start PTY session".to_string(),
1015 timeout_category_label.clone(),
1016 base_timeout_ms,
1017 adaptive_timeout_ms,
1018 None,
1019 false,
1020 );
1021
1022 return Ok(error.to_json_value());
1023 }
1024 }
1025 } else {
1026 None
1027 };
1028
1029 let execution_started_at = Instant::now();
1032 let effective_timeout = self.effective_timeout_for_call(timeout_category, &args);
1036 let effective_timeout_ms = effective_timeout.map(|d| d.as_millis() as u64);
1037
1038 let fail_open = self.optimization_config.tool_registry.middleware_fail_open;
1039 let middleware_req = ToolCallRequest {
1040 id: requested_name.clone(),
1041 tool_name: tool_name.as_str().into(),
1042 args: args.clone(),
1043 metadata: None,
1044 };
1045 if let Err(err) = self.middleware.before_execute_opt(&middleware_req, fail_open).await {
1046 if !fail_open {
1047 let error_msg = format!("Middleware denied execution: {err}");
1048 record_failure(
1049 tool_name_owned.clone(),
1050 is_mcp_tool,
1051 mcp_provider.clone(),
1052 args_for_recording.clone(),
1053 error_msg.clone(),
1054 timeout_category_label.clone(),
1055 base_timeout_ms,
1056 adaptive_timeout_ms,
1057 None,
1058 false,
1059 );
1060 return Err(anyhow!(error_msg).context("tool denied by middleware"));
1061 }
1062 }
1063
1064 if fresh_patch_read
1067 && !tool_intent::is_command_run_tool_call(&tool_name, &args)
1068 && let Some(object) = args.as_object_mut()
1069 {
1070 object.insert(
1071 crate::tools::file_ops::PATCH_READ_CACHE_NONCE.to_string(),
1072 json!(uuid::Uuid::new_v4().to_string()),
1073 );
1074 }
1075 let exec_future = async {
1076 if is_mcp_tool {
1077 let mcp_name = mcp_tool_name
1078 .as_deref()
1079 .context("MCP tool routing inconsistency: resolved MCP tool name missing")?;
1080 self.execute_mcp_tool(mcp_name, args).await
1081 } else if exec_settlement_mode.settle_noninteractive()
1082 && matches!(tool_name.as_str(), tools::UNIFIED_EXEC | tools::EXEC_COMMAND | tools::EXEC_PTY_CMD)
1083 {
1084 let exec_args = match tool_name.as_str() {
1085 tools::EXEC_COMMAND => super::executors::normalize_command_session_run_alias_args(&args, false)?,
1086 tools::EXEC_PTY_CMD => super::executors::normalize_command_session_run_alias_args(&args, true)?,
1087 _ => args.clone(),
1088 };
1089 if self.optimization_config.memory_pool.enabled {
1090 let _execution_guard = self.memory_pool.get_value();
1091 let _string_guard = self.memory_pool.get_string();
1092 let _vec_guard = self.memory_pool.get_vec();
1093 self.execute_command_session_internal(exec_args, exec_settlement_mode).await
1094 } else {
1095 self.execute_command_session_internal(exec_args, exec_settlement_mode).await
1096 }
1097 } else if exec_settlement_mode.settle_noninteractive() && tool_name == tools::WRITE_STDIN {
1098 self.execute_write_stdin(args, exec_settlement_mode).await
1099 } else if let Some(registration) = self.inventory.registration_for(&tool_name) {
1100 self.execute_registered_handler(&tool_name, ®istration, args, cached_tool.as_ref())
1101 .await
1102 } else {
1103 let (tool_names, similar_tools) = self.public_tool_catalog_for_error(&requested_name).await;
1106 let available_tool_list = tool_names.join(", ");
1107
1108 let suggestion = if !similar_tools.is_empty() {
1109 format!(" Did you mean: {}?", similar_tools.join(", "))
1110 } else {
1111 String::new()
1112 };
1113
1114 let error_msg = format!(
1115 "Tool '{display_name}' not found in registry. Available tools: {available_tool_list}.{suggestion}"
1116 );
1117
1118 let error =
1119 ToolExecutionError::new(tool_name_owned.clone(), ToolErrorType::ToolNotFound, error_msg.clone());
1120
1121 record_failure(
1122 tool_name_owned.clone(),
1123 is_mcp_tool,
1124 mcp_provider.clone(),
1125 args_for_recording.clone(),
1126 error_msg,
1127 timeout_category_label.clone(),
1128 base_timeout_ms,
1129 adaptive_timeout_ms,
1130 effective_timeout_ms,
1131 false,
1132 );
1133
1134 Ok(error.to_json_value())
1135 }
1136 };
1137
1138 let result = if let Some(limit) = effective_timeout {
1139 trace!(
1140 tool = %tool_name_owned,
1141 category = %timeout_category.label(),
1142 timeout_ms = %limit.as_millis(),
1143 "Executing tool with effective timeout"
1144 );
1145 match tokio::time::timeout(limit, exec_future).await {
1146 Ok(res) => res,
1147 Err(_) => {
1148 let timeout_ms = limit.as_millis() as u64;
1149 let tripped = self.record_tool_failure(timeout_category);
1150 if tripped {
1151 warn!(
1152 tool = %tool_name_owned,
1153 category = %timeout_category.label(),
1154 "Tool circuit breaker tripped after consecutive timeout failures"
1155 );
1156 }
1157 let retry_after = self.should_circuit_break(timeout_category);
1158
1159 let mut timeout_error = ToolExecutionError::new(
1160 tool_name_owned.clone(),
1161 ToolErrorType::Timeout,
1162 format!(
1163 "Operation '{}' exceeded the {} timeout ceiling ({}s)",
1164 tool_name_owned,
1165 timeout_category.label(),
1166 limit.as_secs()
1167 ),
1168 )
1169 .with_tool_call_context(&tool_name_owned, &args_for_recording)
1170 .with_surface("tool_registry")
1171 .with_debug_metadata("timeout_category", timeout_category.label())
1172 .with_debug_metadata("timeout_ms", timeout_ms.to_string());
1173
1174 if tool_name_owned == tools::UNIFIED_EXEC {
1175 timeout_error.recovery_suggestions = vec![
1176 Cow::Borrowed("Use write_stdin with empty chars to poll command progress"),
1177 Cow::Borrowed("Use exec_command with a fresh command if the original session is stale"),
1178 Cow::Borrowed("Ask for manual cleanup if a stale session is still active"),
1179 ];
1180 }
1181
1182 if let Some(delay) = retry_after {
1183 timeout_error.retry_after_ms = Some(delay.as_millis().min(u128::from(u64::MAX)) as u64);
1184 }
1185
1186 let mut timeout_payload = timeout_error.to_json_value();
1187 Self::annotate_timeout_error_payload(
1188 &mut timeout_payload,
1189 timeout_category.label(),
1190 timeout_ms,
1191 tripped,
1192 );
1193
1194 if let Some(breaker) = shared_circuit_breaker.as_ref() {
1195 breaker.record_failure_category_for_tool(&tool_name_owned, ErrorCategory::Timeout);
1196 }
1197 if is_mcp_tool {
1198 self.mcp_circuit_breaker.record_failure_category(ErrorCategory::Timeout);
1199 }
1200 record_failure(
1201 tool_name_owned,
1202 is_mcp_tool,
1203 mcp_provider,
1204 args_for_recording,
1205 timeout_error.user_message(),
1206 timeout_category_label.clone(),
1207 base_timeout_ms,
1208 adaptive_timeout_ms,
1209 Some(timeout_ms),
1210 tripped,
1211 );
1212 return Ok(timeout_payload);
1213 }
1214 }
1215 } else {
1216 exec_future.await
1217 };
1218
1219 match result {
1224 Ok(value) => {
1225 if let Some(breaker) = shared_circuit_breaker.as_ref() {
1226 breaker.record_success_for_tool(&tool_name_owned);
1227 }
1228 if is_mcp_tool {
1229 self.mcp_circuit_breaker.record_success();
1230 }
1231 self.reset_tool_failure(timeout_category);
1232 let should_decay = {
1233 let mut state = self.resiliency.lock();
1234 let success_streak = state.adaptive_tuning.success_streak;
1235 if let Some(counter) = state.success_trackers.get_mut(&timeout_category) {
1236 *counter = counter.saturating_add(1);
1237 let counter_val = *counter;
1238 if counter_val >= success_streak {
1239 *counter = 0;
1240 true
1241 } else {
1242 false
1243 }
1244 } else {
1245 false
1246 }
1247 };
1248 if should_decay {
1249 self.decay_adaptive_timeout(timeout_category);
1250 }
1251 self.record_tool_latency(timeout_category, execution_started_at.elapsed());
1252 let super::execution_results::ExecutionOutput { normalized_value, structured_error } = self
1253 .prepare_execution_output(
1254 &tool_name_owned,
1255 &args_for_recording,
1256 value,
1257 is_mcp_tool,
1258 max_output_tokens,
1259 )
1260 .await;
1261
1262 if !readonly_classification {
1263 self.invalidate_mutated_reads(&tool_name_owned, &args_for_recording);
1264 }
1265
1266 if let Some(error_msg) = structured_error {
1267 self.execution_history.add_record(ToolExecutionRecord::failure(
1268 tool_name_owned,
1269 requested_name,
1270 is_mcp_tool,
1271 mcp_provider,
1272 args_for_recording,
1273 error_msg,
1274 context_snapshot.clone(),
1275 timeout_category_label.clone(),
1276 base_timeout_ms,
1277 adaptive_timeout_ms,
1278 effective_timeout_ms,
1279 false,
1280 ));
1281 } else {
1282 self.execution_history.add_record(ToolExecutionRecord::success(
1283 tool_name_owned,
1284 requested_name,
1285 is_mcp_tool,
1286 mcp_provider,
1287 args_for_recording,
1288 normalized_value.clone(),
1289 context_snapshot.clone(),
1290 timeout_category_label.clone(),
1291 base_timeout_ms,
1292 adaptive_timeout_ms,
1293 effective_timeout_ms,
1294 false,
1295 ));
1296 }
1297
1298 let _ = self
1299 .middleware
1300 .after_execute(
1301 &middleware_req,
1302 &ToolCallResponse {
1303 id: middleware_req.id.clone(),
1304 success: true,
1305 result: Some(normalized_value.clone()),
1306 error: None,
1307 duration_ms: Some(execution_started_at.elapsed().as_millis() as u64),
1308 cache_hit: None,
1309 },
1310 )
1311 .await;
1312
1313 Ok(normalized_value)
1314 }
1315 Err(err) => {
1316 if err.to_string().contains("tool reentrancy blocked") {
1320 return Err(err);
1321 }
1322
1323 let error = ToolExecutionError::from_anyhow(
1324 tool_name_owned.clone(),
1325 &err,
1326 0,
1327 false,
1328 false,
1329 Some("tool_registry"),
1330 )
1331 .with_tool_call_context(&tool_name_owned, &args_for_recording);
1332 self.grant_patch_recovery_read(&error).await;
1333 let error_category = error.category;
1334 if error.circuit_breaker_impact
1335 && let Some(breaker) = shared_circuit_breaker.as_ref()
1336 {
1337 breaker.record_failure_category_for_tool(&tool_name_owned, error_category);
1338 }
1339 if error.circuit_breaker_impact && is_mcp_tool {
1340 self.mcp_circuit_breaker.record_failure_category(error_category);
1341 }
1342
1343 let tripped = if error.circuit_breaker_impact {
1344 let tripped = self.record_tool_failure(timeout_category);
1345 if tripped {
1346 warn!(
1347 tool = %tool_name_owned,
1348 category = %timeout_category.label(),
1349 "Tool circuit breaker tripped after consecutive failures"
1350 );
1351 }
1352 tripped
1353 } else {
1354 false
1355 };
1356
1357 let mut payload = error.to_json_value();
1358 Self::annotate_timeout_error_payload(
1359 &mut payload,
1360 timeout_category.label(),
1361 effective_timeout_ms.unwrap_or(0),
1362 tripped,
1363 );
1364
1365 record_failure(
1366 tool_name_owned,
1367 is_mcp_tool,
1368 mcp_provider,
1369 args_for_recording,
1370 format!("Tool execution failed: {err}"),
1371 timeout_category_label.clone(),
1372 base_timeout_ms,
1373 adaptive_timeout_ms,
1374 effective_timeout_ms,
1375 tripped,
1376 );
1377
1378 let _ = self
1379 .middleware
1380 .on_error(
1381 &middleware_req,
1382 &UnifiedToolError::new(
1383 UnifiedErrorKind::from(vtcode_commons::classify_anyhow_error(&err)),
1384 err.to_string(),
1385 ),
1386 )
1387 .await;
1388
1389 Ok(payload)
1390 }
1391 }
1392 }
1393}