Skip to main content

microclaw_engine/
tool_executor.rs

1//! Parallel tool execution within a single agent turn.
2//!
3//! When the LLM returns multiple tool_use blocks, this module partitions them into
4//! execution "waves" based on their concurrency class and runs safe subsets in parallel
5//! via `tokio::JoinSet`.
6
7use std::collections::HashMap;
8use std::sync::Arc;
9
10use serde_json::Value;
11use tracing::{info, warn};
12
13use crate::hooks::HookOutcome;
14use crate::internal::observability::traces::{kv, new_span_id, now_unix_nano, SpanData};
15use crate::internal::tool_runtime::runtime::{
16    parse_concurrency_class, tool_concurrency_class, ToolConcurrencyClass, ToolResult,
17};
18use crate::runtime::AppState;
19use crate::tools::ToolAuthContext;
20use microclaw_core::llm_types::ContentBlock;
21use opentelemetry_proto::tonic::trace::v1::Status;
22
23use crate::agent_engine::AgentEvent;
24use crate::approval_broker::ApprovalDecision;
25use crate::internal::observability::traces::OtlpTraceExporter;
26use microclaw_core::runtime_event::{RuntimePlanStep, RuntimePlanStepStatus, RuntimeProcessKind};
27use tokio::sync::mpsc::UnboundedSender;
28
29/// A single pending tool call extracted from the LLM response.
30#[derive(Clone, Debug)]
31pub struct PendingToolCall {
32    pub id: String,
33    pub name: String,
34    pub input: Value,
35}
36
37/// Mutable context accumulated across tool executions within a turn.
38pub struct ToolBatchContext {
39    pub failed_tools: std::collections::BTreeSet<String>,
40    pub failed_tool_details: Vec<String>,
41    pub seen_failed_tool_details: std::collections::HashSet<String>,
42    pub consecutive_send_message_calls: usize,
43    pub skill_env_files: Vec<String>,
44    pub tool_auth: ToolAuthContext,
45    pub waiting_for_user_approval: bool,
46    pub waiting_approval_tool: Option<String>,
47    /// Human-readable preview of the operation requesting approval (e.g., the
48    /// exact bash command). Surfaced to the operator so they can confirm
49    /// without having to scroll back through tool-call JSON.
50    pub waiting_approval_preview: Option<String>,
51}
52
53/// Metrics counters for tool execution.
54pub struct ToolMetrics {
55    pub tool_calls: i64,
56    pub tool_errors: i64,
57}
58
59/// Resolve the effective concurrency class for a tool, respecting config overrides.
60fn resolve_concurrency_class(
61    name: &str,
62    overrides: &HashMap<String, String>,
63) -> ToolConcurrencyClass {
64    if let Some(override_str) = overrides.get(name) {
65        if let Some(class) = parse_concurrency_class(override_str) {
66            return class;
67        }
68        warn!(
69            "Invalid tool_concurrency_override for '{}': '{}'; using default",
70            name, override_str
71        );
72    }
73    tool_concurrency_class(name)
74}
75
76/// Partition tool calls into execution waves based on concurrency classes.
77///
78/// Rules:
79/// 1. All ReadOnly -> single wave, all parallel
80/// 2. Mixed ReadOnly + others -> wave 1 = ReadOnly parallel, then rest sequentially
81/// 3. Exclusive -> runs alone in its own wave
82fn partition_into_waves(
83    calls: &[PendingToolCall],
84    overrides: &HashMap<String, String>,
85) -> Vec<Vec<usize>> {
86    if calls.is_empty() {
87        return Vec::new();
88    }
89    if calls.len() == 1 {
90        return vec![vec![0]];
91    }
92
93    let classes: Vec<ToolConcurrencyClass> = calls
94        .iter()
95        .map(|c| resolve_concurrency_class(&c.name, overrides))
96        .collect();
97
98    let mut readonly_indices = Vec::new();
99    let mut sideeffect_indices = Vec::new();
100    let mut exclusive_indices = Vec::new();
101
102    for (i, class) in classes.iter().enumerate() {
103        match class {
104            ToolConcurrencyClass::ReadOnly => readonly_indices.push(i),
105            ToolConcurrencyClass::SideEffect => sideeffect_indices.push(i),
106            ToolConcurrencyClass::Exclusive => exclusive_indices.push(i),
107        }
108    }
109
110    let mut waves = Vec::new();
111
112    // Wave 1: all ReadOnly tools in parallel
113    if !readonly_indices.is_empty() {
114        waves.push(readonly_indices);
115    }
116
117    // Then: SideEffect tools, one per wave (sequential)
118    for idx in sideeffect_indices {
119        waves.push(vec![idx]);
120    }
121
122    // Then: Exclusive tools, one per wave
123    for idx in exclusive_indices {
124        waves.push(vec![idx]);
125    }
126
127    waves
128}
129
130/// Execute a batch of tool calls, optionally in parallel.
131///
132/// When `parallel_enabled` is false, all tools execute sequentially (existing behavior).
133/// When true, tools are partitioned into waves and ReadOnly tools within a wave run concurrently.
134#[expect(clippy::too_many_arguments)]
135pub async fn execute_tool_batch(
136    state: &AppState,
137    calls: &[PendingToolCall],
138    ctx: &mut ToolBatchContext,
139    metrics: &mut ToolMetrics,
140    event_tx: Option<&UnboundedSender<AgentEvent>>,
141    chat_id: i64,
142    iteration: usize,
143    caller_channel: &str,
144    explicit_user_approval: bool,
145    trace_id: &[u8],
146    parent_span_id: &[u8],
147) -> Vec<ContentBlock> {
148    let max_concurrency = state.config.parallel_tool_max_concurrency;
149
150    if calls.len() <= 1 {
151        // Sequential execution (original behavior)
152        return execute_sequentially(
153            state,
154            calls,
155            ctx,
156            metrics,
157            event_tx,
158            chat_id,
159            iteration,
160            caller_channel,
161            explicit_user_approval,
162            trace_id,
163            parent_span_id,
164        )
165        .await;
166    }
167
168    let waves = partition_into_waves(calls, &state.config.tool_concurrency_overrides);
169    let mut all_results: Vec<(usize, ContentBlock)> = Vec::with_capacity(calls.len());
170
171    for (wave_idx, wave) in waves.iter().enumerate() {
172        if let Some(tx) = event_tx {
173            let _ = tx.send(AgentEvent::ToolWaveStart {
174                wave: wave_idx + 1,
175                tool_count: wave.len(),
176            });
177        }
178
179        if wave.len() == 1 {
180            // Single tool in wave — execute inline (no spawn overhead)
181            let idx = wave[0];
182            let result = execute_single_tool(
183                state,
184                &calls[idx],
185                ctx,
186                metrics,
187                event_tx,
188                chat_id,
189                iteration,
190                caller_channel,
191                explicit_user_approval,
192                trace_id,
193                parent_span_id,
194            )
195            .await;
196            all_results.push((idx, result));
197        } else {
198            // Multiple tools — run in parallel with JoinSet
199            let results = execute_wave_parallel(
200                state,
201                calls,
202                wave,
203                ctx,
204                metrics,
205                event_tx,
206                chat_id,
207                iteration,
208                caller_channel,
209                explicit_user_approval,
210                trace_id,
211                parent_span_id,
212                max_concurrency,
213            )
214            .await;
215            all_results.extend(results);
216        }
217
218        if let Some(tx) = event_tx {
219            let _ = tx.send(AgentEvent::ToolWaveComplete { wave: wave_idx + 1 });
220        }
221    }
222
223    // Sort by original index to preserve order
224    all_results.sort_by_key(|(idx, _)| *idx);
225    all_results.into_iter().map(|(_, block)| block).collect()
226}
227
228/// Execute all tools sequentially (the original behavior).
229#[expect(clippy::too_many_arguments)]
230async fn execute_sequentially(
231    state: &AppState,
232    calls: &[PendingToolCall],
233    ctx: &mut ToolBatchContext,
234    metrics: &mut ToolMetrics,
235    event_tx: Option<&UnboundedSender<AgentEvent>>,
236    chat_id: i64,
237    iteration: usize,
238    caller_channel: &str,
239    explicit_user_approval: bool,
240    trace_id: &[u8],
241    parent_span_id: &[u8],
242) -> Vec<ContentBlock> {
243    let mut results = Vec::with_capacity(calls.len());
244    for call in calls {
245        let block = execute_single_tool(
246            state,
247            call,
248            ctx,
249            metrics,
250            event_tx,
251            chat_id,
252            iteration,
253            caller_channel,
254            explicit_user_approval,
255            trace_id,
256            parent_span_id,
257        )
258        .await;
259        results.push(block);
260    }
261    results
262}
263
264/// Execute a single tool call with all the hooks, tracing, and guardrail logic.
265#[expect(clippy::too_many_arguments)]
266async fn execute_single_tool(
267    state: &AppState,
268    call: &PendingToolCall,
269    ctx: &mut ToolBatchContext,
270    metrics: &mut ToolMetrics,
271    event_tx: Option<&UnboundedSender<AgentEvent>>,
272    chat_id: i64,
273    iteration: usize,
274    caller_channel: &str,
275    explicit_user_approval: bool,
276    trace_id: &[u8],
277    parent_span_id: &[u8],
278) -> ContentBlock {
279    let id = &call.id;
280    let name = &call.name;
281    let input = &call.input;
282
283    // Malformed tool name check
284    if name.trim().is_empty() {
285        warn!(
286            chat_id,
287            iteration,
288            tool_use_id = %id,
289            "Skipping malformed tool call with empty tool name"
290        );
291        return ContentBlock::ToolResult {
292            tool_use_id: id.clone(),
293            content: "Malformed tool call: missing tool name. Retry with a valid registered tool."
294                .to_string(),
295            is_error: Some(true),
296        };
297    }
298
299    // send_message consecutive call guardrail
300    if name != "send_message" {
301        ctx.consecutive_send_message_calls = 0;
302    } else if ctx.consecutive_send_message_calls >= 3 {
303        warn!(
304            chat_id,
305            iteration, "Guardrail: blocking repeated send_message loop"
306        );
307        let content = "send_message blocked: too many consecutive send_message calls in one request. Use normal assistant reply for final output instead of repeatedly calling send_message.".to_string();
308        ctx.failed_tools.insert(name.clone());
309        let detail = format!("send_message: {content}");
310        if ctx.seen_failed_tool_details.insert(detail.clone()) {
311            ctx.failed_tool_details.push(detail);
312        }
313        return ContentBlock::ToolResult {
314            tool_use_id: id.clone(),
315            content,
316            is_error: Some(true),
317        };
318    }
319
320    // Feishu send_message text restriction
321    let is_feishu_turn = caller_channel.starts_with("feishu") || caller_channel.starts_with("lark");
322    let send_message_has_attachment = input
323        .get("attachment_path")
324        .and_then(|v| v.as_str())
325        .map(|v| !v.trim().is_empty())
326        .unwrap_or(false);
327    if name == "send_message" && is_feishu_turn && !send_message_has_attachment {
328        return ContentBlock::ToolResult {
329            tool_use_id: id.clone(),
330            content: "send_message text replies are disabled for Feishu runtime turns; return the final assistant text directly so channel reaction/text delivery can be handled correctly. If you need to send a file, call send_message with attachment_path.".to_string(),
331            is_error: Some(true),
332        };
333    }
334
335    // Before-tool hook
336    let mut effective_input = input.clone();
337    if let Ok(hook_outcome) = state
338        .hooks
339        .run_before_tool(chat_id, caller_channel, iteration, name, &effective_input)
340        .await
341    {
342        match hook_outcome {
343            HookOutcome::Block { reason } => {
344                return ContentBlock::ToolResult {
345                    tool_use_id: id.clone(),
346                    content: if reason.trim().is_empty() {
347                        format!("tool '{}' blocked by policy hook", name)
348                    } else {
349                        reason
350                    },
351                    is_error: Some(true),
352                };
353            }
354            HookOutcome::Allow { patches } => {
355                for patch in patches {
356                    if let Some(v) = patch.get("tool_input") {
357                        effective_input = v.clone();
358                    }
359                }
360            }
361        }
362    }
363
364    // Emit ToolStart event
365    if let Some(tx) = event_tx {
366        let _ = tx.send(AgentEvent::ToolStart {
367            call_id: id.clone(),
368            name: name.clone(),
369            input: effective_input.clone(),
370        });
371    }
372    info!(chat_id, tool = %name, iteration, "Executing tool");
373
374    let started = std::time::Instant::now();
375    let mut executed_input = effective_input.clone();
376    let tool_span_id = new_span_id();
377    let tool_start = now_unix_nano();
378    metrics.tool_calls += 1;
379
380    // Execute tool
381    let mut result = state
382        .tools
383        .execute_with_auth(name, executed_input.clone(), &ctx.tool_auth)
384        .await;
385
386    // Trace span
387    if let Some(exp) = &state.trace_exporter {
388        emit_tool_span(
389            exp,
390            trace_id,
391            tool_span_id,
392            parent_span_id,
393            name,
394            &executed_input,
395            &result,
396            tool_start,
397            "tool_execution",
398        );
399    }
400
401    // Approval gate: retry with the approval marker when approval is not
402    // required, already given, or granted in-run through a registered
403    // approval broker; otherwise pause the turn for a text reply.
404    if result.is_error && result.error_type.as_deref() == Some("approval_required") {
405        let confirmation_required = state.config.high_risk_tool_user_confirmation_required;
406        let approved = if !confirmation_required {
407            info!("Auto-retrying tool '{}' after approval gate", name);
408            true
409        } else if explicit_user_approval {
410            info!("Retrying tool '{}' after explicit user approval", name);
411            true
412        } else {
413            let preview = approval_preview(name, &executed_input);
414            match await_in_run_approval(
415                state,
416                event_tx,
417                caller_channel,
418                chat_id,
419                name,
420                preview.as_deref(),
421            )
422            .await
423            {
424                Some(ApprovalDecision::ApproveOnce) => {
425                    info!("Retrying tool '{}' after in-run approval", name);
426                    true
427                }
428                Some(ApprovalDecision::ApproveAlways) => {
429                    info!("Retrying tool '{}' after in-run standing approval", name);
430                    record_standing_grant(state, chat_id, name).await;
431                    true
432                }
433                Some(ApprovalDecision::Deny) => {
434                    info!("Tool '{}' denied in-run", name);
435                    result = ToolResult::error(format!(
436                        "The user denied this {name} call; it was not executed. Do not retry it \
437                         unchanged. Continue without it, or explain what you need and why."
438                    ))
439                    .with_error_type("approval_denied");
440                    false
441                }
442                Some(ApprovalDecision::Defer) | None => {
443                    ctx.waiting_for_user_approval = true;
444                    ctx.waiting_approval_tool = Some(name.clone());
445                    ctx.waiting_approval_preview = preview;
446                    false
447                }
448            }
449        };
450        if approved {
451            executed_input = with_high_risk_approval_marker(&effective_input);
452            let retry_span_id = new_span_id();
453            let retry_start = now_unix_nano();
454            metrics.tool_calls += 1;
455
456            result = state
457                .tools
458                .execute_with_auth(name, executed_input.clone(), &ctx.tool_auth)
459                .await;
460
461            if let Some(exp) = &state.trace_exporter {
462                emit_tool_span(
463                    exp,
464                    trace_id,
465                    retry_span_id,
466                    parent_span_id,
467                    name,
468                    &executed_input,
469                    &result,
470                    retry_start,
471                    "tool_execution_retry",
472                );
473            }
474        }
475    }
476
477    // activate_skill metadata handling
478    if name == "activate_skill" && !result.is_error {
479        if let Some(meta) = &result.metadata {
480            if let Some(path) = meta.get("skill_env_file").and_then(|v| v.as_str()) {
481                let path_str = path.to_string();
482                if !ctx.skill_env_files.contains(&path_str) {
483                    ctx.skill_env_files.push(path_str);
484                    ctx.tool_auth.env_files = ctx.skill_env_files.clone();
485                }
486                if let Ok(files_json) = serde_json::to_string(&ctx.skill_env_files) {
487                    let db = state.db.clone();
488                    let _ = crate::internal::storage::db::call_blocking(db, move |db| {
489                        db.save_session_skill_envs(chat_id, &files_json)
490                    })
491                    .await;
492                }
493            }
494        }
495    }
496
497    // After-tool hook
498    if let Ok(hook_outcome) = state
499        .hooks
500        .run_after_tool(
501            chat_id,
502            caller_channel,
503            iteration,
504            name,
505            &executed_input,
506            &result,
507        )
508        .await
509    {
510        match hook_outcome {
511            HookOutcome::Block { reason } => {
512                result.is_error = true;
513                if !reason.trim().is_empty() {
514                    result.content = reason;
515                }
516                if result.error_type.is_none() {
517                    result.error_type = Some("hook_blocked".to_string());
518                }
519            }
520            HookOutcome::Allow { patches } => {
521                for patch in patches {
522                    if let Some(v) = patch.get("content").and_then(|v| v.as_str()) {
523                        result.content = v.to_string();
524                    }
525                    if let Some(v) = patch.get("is_error").and_then(|v| v.as_bool()) {
526                        result.is_error = v;
527                    }
528                    if let Some(v) = patch
529                        .get("error_type")
530                        .and_then(|v| v.as_str())
531                        .map(str::to_string)
532                    {
533                        result.error_type = Some(v);
534                    }
535                    if let Some(v) = patch
536                        .get("status_code")
537                        .and_then(|v| v.as_i64())
538                        .map(|x| x as i32)
539                    {
540                        result.status_code = Some(v);
541                    }
542                }
543            }
544        }
545    }
546
547    // Error tracking
548    if result.is_error && result.error_type.as_deref() != Some("approval_required") {
549        let suppress = result
550            .error_type
551            .as_deref()
552            .map(|t| t == "feishu_reaction_protocol_text" || t == "feishu_send_message_disabled")
553            .unwrap_or(false);
554        if !suppress {
555            ctx.failed_tools.insert(name.clone());
556            let detail = format_failed_action_for_user(name, &executed_input, &result.content);
557            if ctx.seen_failed_tool_details.insert(detail.clone()) {
558                ctx.failed_tool_details.push(detail);
559            }
560        }
561        let preview = truncate_for_log(&result.content, 300);
562        warn!(
563            chat_id,
564            tool = %name,
565            iteration,
566            error_type = ?result.error_type,
567            "Tool execution failed: {}",
568            preview
569        );
570    }
571
572    // Emit ToolResult event
573    if let Some(tx) = event_tx {
574        let preview = truncate_for_log(&result.content, 160);
575        let duration_ms = result
576            .duration_ms
577            .unwrap_or_else(|| started.elapsed().as_millis());
578        let transport_duration_ms = u64::try_from(duration_ms).unwrap_or(u64::MAX);
579        let _ = tx.send(AgentEvent::ToolResult {
580            call_id: id.clone(),
581            name: name.clone(),
582            is_error: result.is_error,
583            preview,
584            duration_ms: transport_duration_ms,
585            status_code: result.status_code,
586            bytes: result.bytes,
587            error_type: result.error_type.clone(),
588        });
589        emit_file_diff_event(tx, &result);
590        emit_plan_update_event(tx, name, &executed_input, result.is_error);
591        emit_process_output_event(tx, id, name, &executed_input, &result, duration_ms);
592    }
593    if result.is_error {
594        metrics.tool_errors += 1;
595    }
596    if name == "send_message" {
597        ctx.consecutive_send_message_calls += 1;
598    }
599
600    // Spill oversized tool results to an artifact and truncate the inline copy
601    // so the message history stays cheap to replay. The agent can fetch the
602    // full body via `fetch_artifact`. `fetch_artifact` itself is exempted to
603    // avoid recursive truncation when the agent reads a slice.
604    let content = if name == "fetch_artifact" {
605        result.content
606    } else {
607        maybe_spill_to_artifact(state, chat_id, name, result.content).await
608    };
609
610    ContentBlock::ToolResult {
611        tool_use_id: id.clone(),
612        content,
613        is_error: if result.is_error { Some(true) } else { None },
614    }
615}
616
617/// Spill `content` to a `tool_result_artifacts` row when it exceeds the
618/// configured threshold; the returned string keeps the head + tail and a
619/// pointer the agent can pass to `fetch_artifact`. Falls back to the
620/// untouched content on any storage error so a transient DB failure never
621/// drops a tool result.
622async fn maybe_spill_to_artifact(
623    state: &AppState,
624    chat_id: i64,
625    tool_name: &str,
626    content: String,
627) -> String {
628    let threshold = state.config.tool_result_truncation_threshold_chars;
629    if threshold == 0 {
630        return content;
631    }
632    let total_chars = content.chars().count();
633    if total_chars <= threshold {
634        return content;
635    }
636
637    let head_chars = state.config.tool_result_truncation_head_chars;
638    let tail_chars = state.config.tool_result_truncation_tail_chars;
639    let ttl_hours = state.config.tool_result_artifact_ttl_hours;
640
641    // Guard against pathological config: head + tail must still leave room
642    // for the truncation marker, otherwise truncation is pointless.
643    if head_chars + tail_chars >= total_chars {
644        return content;
645    }
646
647    let artifact_id = format!("art_{}", uuid::Uuid::new_v4().simple());
648    let expires_at = (chrono::Utc::now() + chrono::Duration::hours(ttl_hours as i64)).to_rfc3339();
649
650    let db = state.db.clone();
651    let artifact_id_for_save = artifact_id.clone();
652    let tool_name_for_save = tool_name.to_string();
653    let content_for_save = content.clone();
654    let saved = crate::internal::storage::db::call_blocking(db, move |db| {
655        db.save_tool_artifact(
656            &artifact_id_for_save,
657            chat_id,
658            &tool_name_for_save,
659            &content_for_save,
660            &expires_at,
661        )
662    })
663    .await;
664
665    if let Err(e) = saved {
666        warn!(
667            chat_id,
668            tool = %tool_name,
669            "Failed to persist tool artifact; returning untruncated content: {}",
670            e
671        );
672        return content;
673    }
674
675    let head: String = content.chars().take(head_chars).collect();
676    let tail: String = content.chars().skip(total_chars - tail_chars).collect();
677    let hidden = total_chars - head_chars - tail_chars;
678    format!(
679        "{head}\n\n[--- {hidden} of {total_chars} characters truncated. \
680         Full result saved as artifact `{artifact_id}` for {ttl_hours}h. \
681         Call fetch_artifact(artifact_id=\"{artifact_id}\", offset=N, length=N) \
682         to read more. ---]\n\n{tail}"
683    )
684}
685
686/// Execute a wave of tools in parallel using futures::join_all.
687///
688/// Before-tool hooks run sequentially first, then tools execute concurrently,
689/// then after-tool hooks are handled per-tool. This preserves hook ordering.
690#[expect(clippy::too_many_arguments)]
691async fn execute_wave_parallel(
692    state: &AppState,
693    calls: &[PendingToolCall],
694    wave_indices: &[usize],
695    ctx: &mut ToolBatchContext,
696    metrics: &mut ToolMetrics,
697    event_tx: Option<&UnboundedSender<AgentEvent>>,
698    chat_id: i64,
699    iteration: usize,
700    caller_channel: &str,
701    _explicit_user_approval: bool,
702    _trace_id: &[u8],
703    _parent_span_id: &[u8],
704    _max_concurrency: usize,
705) -> Vec<(usize, ContentBlock)> {
706    info!(
707        chat_id,
708        wave_size = wave_indices.len(),
709        "Executing tool wave in parallel"
710    );
711
712    // Pre-process: run before_tool hooks sequentially and prepare inputs.
713    struct PreparedCall {
714        idx: usize,
715        name: String,
716        effective_input: Value,
717        skipped: Option<ContentBlock>,
718    }
719
720    let mut prepared = Vec::with_capacity(wave_indices.len());
721    for &idx in wave_indices {
722        let call = &calls[idx];
723        let name = &call.name;
724        let mut effective_input = call.input.clone();
725
726        if let Some(tx) = event_tx {
727            let _ = tx.send(AgentEvent::ToolStart {
728                call_id: call.id.clone(),
729                name: name.clone(),
730                input: effective_input.clone(),
731            });
732        }
733
734        let mut skipped = None;
735        if let Ok(hook_outcome) = state
736            .hooks
737            .run_before_tool(chat_id, caller_channel, iteration, name, &effective_input)
738            .await
739        {
740            match hook_outcome {
741                HookOutcome::Block { reason } => {
742                    skipped = Some(ContentBlock::ToolResult {
743                        tool_use_id: call.id.clone(),
744                        content: if reason.trim().is_empty() {
745                            format!("tool '{}' blocked by policy hook", name)
746                        } else {
747                            reason
748                        },
749                        is_error: Some(true),
750                    });
751                }
752                HookOutcome::Allow { patches } => {
753                    for patch in patches {
754                        if let Some(v) = patch.get("tool_input") {
755                            effective_input = v.clone();
756                        }
757                    }
758                }
759            }
760        }
761
762        prepared.push(PreparedCall {
763            idx,
764            name: name.clone(),
765            effective_input,
766            skipped,
767        });
768    }
769
770    // Execute non-skipped tools in parallel using futures::join_all.
771    // This borrows state rather than requiring 'static, which avoids the Clone issue.
772    let tool_auth = ctx.tool_auth.clone();
773    let futures: Vec<_> = prepared
774        .iter()
775        .filter(|p| p.skipped.is_none())
776        .map(|prep| {
777            let tool_auth = tool_auth.clone();
778            let name = prep.name.clone();
779            let input = prep.effective_input.clone();
780            let idx = prep.idx;
781            async move {
782                let started = std::time::Instant::now();
783                let result = state
784                    .tools
785                    .execute_with_auth(&name, input.clone(), &tool_auth)
786                    .await;
787                let duration = started.elapsed().as_millis();
788                (idx, name, input, result, duration)
789            }
790        })
791        .collect();
792
793    let parallel_results = futures_util::future::join_all(futures).await;
794
795    let mut results: Vec<(usize, ContentBlock)> = Vec::with_capacity(prepared.len());
796
797    // Add skipped results first
798    for prep in &prepared {
799        if let Some(block) = &prep.skipped {
800            results.push((prep.idx, block.clone()));
801        }
802    }
803
804    // Process parallel results
805    for (idx, name, input, result, _duration) in parallel_results {
806        let call_id = &calls[idx].id;
807        metrics.tool_calls += 1;
808        if result.is_error {
809            metrics.tool_errors += 1;
810            ctx.failed_tools.insert(name.clone());
811        }
812        if name == "send_message" {
813            ctx.consecutive_send_message_calls += 1;
814        }
815
816        // Emit ToolResult event
817        if let Some(tx) = event_tx {
818            let preview = truncate_for_log(&result.content, 160);
819            let duration_ms = result.duration_ms.unwrap_or(_duration);
820            let transport_duration_ms = u64::try_from(duration_ms).unwrap_or(u64::MAX);
821            let _ = tx.send(AgentEvent::ToolResult {
822                call_id: call_id.clone(),
823                name: name.clone(),
824                is_error: result.is_error,
825                preview,
826                duration_ms: transport_duration_ms,
827                status_code: result.status_code,
828                bytes: result.bytes,
829                error_type: result.error_type.clone(),
830            });
831            emit_file_diff_event(tx, &result);
832            emit_plan_update_event(tx, &name, &input, result.is_error);
833            emit_process_output_event(tx, call_id, &name, &input, &result, duration_ms);
834        }
835
836        results.push((
837            idx,
838            ContentBlock::ToolResult {
839                tool_use_id: call_id.clone(),
840                content: result.content,
841                is_error: if result.is_error { Some(true) } else { None },
842            },
843        ));
844    }
845
846    results
847}
848
849fn emit_plan_update_event(
850    tx: &UnboundedSender<AgentEvent>,
851    tool_name: &str,
852    input: &Value,
853    is_error: bool,
854) {
855    if tool_name != "todo_write" || is_error {
856        return;
857    }
858    let Some(todos) = input.get("todos").and_then(Value::as_array) else {
859        return;
860    };
861    let steps = todos
862        .iter()
863        .filter_map(|todo| {
864            let title = todo.get("task")?.as_str()?.trim();
865            if title.is_empty() {
866                return None;
867            }
868            let status = match todo.get("status")?.as_str()? {
869                "pending" => RuntimePlanStepStatus::Pending,
870                "in_progress" => RuntimePlanStepStatus::InProgress,
871                "completed" => RuntimePlanStepStatus::Completed,
872                _ => return None,
873            };
874            Some(RuntimePlanStep {
875                title: title.to_string(),
876                status,
877            })
878        })
879        .collect();
880    let _ = tx.send(AgentEvent::PlanUpdated { steps });
881}
882
883const MAX_PROCESS_OUTPUT_BYTES: usize = 16 * 1024;
884
885fn emit_process_output_event(
886    tx: &UnboundedSender<AgentEvent>,
887    call_id: &str,
888    tool_name: &str,
889    input: &Value,
890    result: &ToolResult,
891    duration_ms: u128,
892) {
893    if tool_name != "bash" {
894        return;
895    }
896    let Some(command) = input.get("command").and_then(Value::as_str) else {
897        return;
898    };
899    let command = microclaw_core::redact::redact_secrets(command);
900    let command = bounded_text(&command, 2_000).0;
901    let output = microclaw_core::redact::redact_secrets(&result.content);
902    let (output, truncated) = bounded_text(&output, MAX_PROCESS_OUTPUT_BYTES);
903    let kind = if is_verification_command(&command) {
904        RuntimeProcessKind::Verification
905    } else {
906        RuntimeProcessKind::Command
907    };
908    let _ = tx.send(AgentEvent::ProcessOutput {
909        call_id: call_id.to_string(),
910        command,
911        output,
912        exit_code: result.status_code,
913        duration_ms: u64::try_from(duration_ms).unwrap_or(u64::MAX),
914        truncated,
915        kind,
916    });
917}
918
919fn is_verification_command(command: &str) -> bool {
920    let normalized = command.to_ascii_lowercase();
921    [
922        "cargo test",
923        "cargo check",
924        "cargo clippy",
925        "npm test",
926        "npm run test",
927        "npm run build",
928        "pnpm test",
929        "pnpm build",
930        "yarn test",
931        "yarn build",
932        "pytest",
933        "go test",
934        "swift test",
935        "dotnet test",
936        "mvn test",
937        "gradle test",
938    ]
939    .iter()
940    .any(|marker| normalized.contains(marker))
941}
942
943fn bounded_text(text: &str, limit: usize) -> (String, bool) {
944    if text.len() <= limit {
945        return (text.to_string(), false);
946    }
947    let half = limit / 2;
948    let head_end = microclaw_core::text::floor_char_boundary(text, half);
949    let mut tail_start = text.len() - half;
950    while tail_start < text.len() && !text.is_char_boundary(tail_start) {
951        tail_start += 1;
952    }
953    (
954        format!(
955            "{}\n\n[… {} bytes omitted …]\n\n{}",
956            &text[..head_end],
957            tail_start.saturating_sub(head_end),
958            &text[tail_start..]
959        ),
960        true,
961    )
962}
963
964/// Forward a `file_diff` metadata payload (attached by edit_file/write_file)
965/// as an `AgentEvent::FileDiff` so channels and the web UI can render what
966/// changed. Successful edits only — error results never carry the payload.
967fn emit_file_diff_event(
968    tx: &tokio::sync::mpsc::UnboundedSender<AgentEvent>,
969    result: &crate::internal::tool_runtime::runtime::ToolResult,
970) {
971    let Some(metadata) = result.metadata.as_ref() else {
972        return;
973    };
974    let single = metadata
975        .get(microclaw_core::diff::FILE_DIFF_METADATA_KEY)
976        .into_iter();
977    let several = metadata
978        .get(microclaw_core::diff::FILE_DIFFS_METADATA_KEY)
979        .and_then(|v| v.as_array())
980        .into_iter()
981        .flatten();
982    for fd in single.chain(several) {
983        let (Some(path), Some(diff)) = (
984            fd.get("path").and_then(|v| v.as_str()),
985            fd.get("diff").and_then(|v| v.as_str()),
986        ) else {
987            continue;
988        };
989        let _ = tx.send(AgentEvent::FileDiff {
990            path: path.to_string(),
991            diff: diff.to_string(),
992            added: fd.get("added").and_then(|v| v.as_u64()).unwrap_or(0) as usize,
993            removed: fd.get("removed").and_then(|v| v.as_u64()).unwrap_or(0) as usize,
994            truncated: fd
995                .get("truncated")
996                .and_then(|v| v.as_bool())
997                .unwrap_or(false),
998        });
999    }
1000}
1001
1002/// Ask a registered in-run approval broker (see [`crate::approval_broker`])
1003/// to decide on a gated tool call, waiting for the answer. Returns `None`
1004/// when no broker is registered for this chat, so the caller falls back to
1005/// ending the turn with a text approval prompt. A broker that goes away
1006/// while waiting counts as a denial.
1007async fn await_in_run_approval(
1008    state: &AppState,
1009    event_tx: Option<&UnboundedSender<AgentEvent>>,
1010    caller_channel: &str,
1011    chat_id: i64,
1012    tool_name: &str,
1013    preview: Option<&str>,
1014) -> Option<ApprovalDecision> {
1015    let tx = event_tx?;
1016    if !crate::approval_broker::is_registered(caller_channel, chat_id) {
1017        return None;
1018    }
1019    let approval_id = uuid::Uuid::new_v4().to_string();
1020    let receiver = crate::approval_broker::open(caller_channel, chat_id, &approval_id)?;
1021    let advisory = crate::agent_engine::review_high_risk_action(state, tool_name, preview).await;
1022    let _ = tx.send(AgentEvent::ApprovalRequired {
1023        approval_id: approval_id.clone(),
1024        tool: tool_name.to_string(),
1025        preview: preview.map(str::to_string),
1026        options: crate::agent_engine::approval_options(tool_name),
1027        advisory,
1028    });
1029    // Close the slot if this future is dropped (run cancelled) before an
1030    // answer arrives.
1031    struct CloseOnDrop<'a>(&'a str, i64, &'a str);
1032    impl Drop for CloseOnDrop<'_> {
1033        fn drop(&mut self) {
1034            crate::approval_broker::close(self.0, self.1, self.2);
1035        }
1036    }
1037    let _close = CloseOnDrop(caller_channel, chat_id, &approval_id);
1038    let decision = receiver.await.unwrap_or(ApprovalDecision::Deny);
1039    let _ = crate::internal::storage::db::call_blocking(state.db.clone(), {
1040        let tool = tool_name.to_string();
1041        let outcome = match decision {
1042            ApprovalDecision::ApproveOnce => "approved_once",
1043            ApprovalDecision::ApproveAlways => "approved_always",
1044            ApprovalDecision::Deny => "denied",
1045            ApprovalDecision::Defer => "deferred",
1046        };
1047        move |db| {
1048            db.log_audit_event(
1049                "approval",
1050                &format!("chat:{chat_id}"),
1051                "in_run_decision",
1052                Some(&tool),
1053                outcome,
1054                Some("answered through the in-run approval broker"),
1055            )
1056        }
1057    })
1058    .await;
1059    Some(decision)
1060}
1061
1062/// Record a standing per-chat allowance for `tool_name` (the same grant an
1063/// "always allow" text reply records).
1064async fn record_standing_grant(state: &AppState, chat_id: i64, tool_name: &str) {
1065    let grant_key = format!("approved_tool:{chat_id}:{tool_name}");
1066    let tool = tool_name.to_string();
1067    let _ = crate::internal::storage::db::call_blocking(state.db.clone(), move |db| {
1068        db.set_runtime_meta(&grant_key, &chrono::Utc::now().to_rfc3339())?;
1069        db.log_audit_event(
1070            "approval",
1071            &format!("chat:{chat_id}"),
1072            "standing_grant",
1073            Some(&tool),
1074            "recorded",
1075            Some("operator chose 'always allow' on an in-run approval"),
1076        )?;
1077        Ok::<_, microclaw_core::error::MicroClawError>(())
1078    })
1079    .await;
1080}
1081
1082/// Build a short, operator-readable preview of the action awaiting approval.
1083/// For bash this is the actual command; for any other future high-risk tool
1084/// we fall back to a truncated JSON dump so the operator can still see what
1085/// is queued instead of just the tool name.
1086fn approval_preview(name: &str, input: &Value) -> Option<String> {
1087    if name == "bash" {
1088        if let Some(cmd) = input
1089            .get("command")
1090            .or_else(|| input.get("cmd"))
1091            .and_then(|v| v.as_str())
1092        {
1093            let trimmed = cmd.trim();
1094            if trimmed.is_empty() {
1095                return None;
1096            }
1097            return Some(truncate_for_log(trimmed, 400));
1098        }
1099    }
1100    let dump = input.to_string();
1101    if dump == "{}" || dump == "null" {
1102        None
1103    } else {
1104        Some(truncate_for_log(&dump, 400))
1105    }
1106}
1107
1108fn with_high_risk_approval_marker(input: &Value) -> Value {
1109    let mut approved_input = input.clone();
1110    if let Some(obj) = approved_input.as_object_mut() {
1111        obj.insert(
1112            "__microclaw_high_risk_approved".to_string(),
1113            Value::Bool(true),
1114        );
1115        return approved_input;
1116    }
1117    serde_json::json!({
1118        "__microclaw_high_risk_approved": true,
1119        "__microclaw_original_input": input,
1120    })
1121}
1122
1123fn format_failed_action_for_user(tool_name: &str, input: &Value, result_content: &str) -> String {
1124    let error_summary = truncate_for_log(result_content, 140);
1125    if tool_name == "bash" {
1126        if let Some(command) = input
1127            .get("command")
1128            .or_else(|| input.get("cmd"))
1129            .and_then(|v| v.as_str())
1130        {
1131            let command_summary = truncate_for_log(command, 140);
1132            return format!("bash `{command_summary}` failed: {error_summary}");
1133        }
1134    }
1135    let input_summary = truncate_for_log(&input.to_string(), 100);
1136    format!("{tool_name} input `{input_summary}` failed: {error_summary}")
1137}
1138
1139fn truncate_for_log(text: &str, max_chars: usize) -> String {
1140    let count = text.chars().count();
1141    if count <= max_chars {
1142        text.to_string()
1143    } else {
1144        let clipped = text.chars().take(max_chars).collect::<String>();
1145        format!("{clipped}...")
1146    }
1147}
1148
1149#[expect(clippy::too_many_arguments)]
1150fn emit_tool_span(
1151    exp: &Arc<OtlpTraceExporter>,
1152    trace_id: &[u8],
1153    span_id: Vec<u8>,
1154    parent_span_id: &[u8],
1155    name: &str,
1156    input: &Value,
1157    result: &ToolResult,
1158    start_time: u64,
1159    span_name: &str,
1160) {
1161    let mut attrs = vec![kv("tool.name", name), kv("input", &input.to_string())];
1162    if result.is_error {
1163        attrs.push(kv(
1164            "error.type",
1165            result.error_type.as_deref().unwrap_or("unknown"),
1166        ));
1167        attrs.push(kv("output", &result.content));
1168    } else {
1169        attrs.push(kv("output", &truncate_for_log(&result.content, 1000)));
1170    }
1171    exp.send_span(SpanData {
1172        trace_id: trace_id.to_vec(),
1173        span_id,
1174        parent_span_id: parent_span_id.to_vec(),
1175        name: span_name.to_string(),
1176        start_time_unix_nano: start_time,
1177        end_time_unix_nano: now_unix_nano(),
1178        attributes: attrs,
1179        status: if result.is_error {
1180            Some(Status {
1181                message: result.content.clone(),
1182                code: 2,
1183            })
1184        } else {
1185            Some(Status {
1186                message: "".to_string(),
1187                code: 1,
1188            })
1189        },
1190        kind: 1,
1191    });
1192}
1193
1194#[cfg(test)]
1195mod tests {
1196    use super::*;
1197
1198    #[test]
1199    fn test_partition_single_readonly() {
1200        let calls = vec![PendingToolCall {
1201            id: "1".into(),
1202            name: "read_file".into(),
1203            input: Value::Null,
1204        }];
1205        let waves = partition_into_waves(&calls, &HashMap::new());
1206        assert_eq!(waves, vec![vec![0]]);
1207    }
1208
1209    #[test]
1210    fn test_partition_all_readonly_parallel() {
1211        let calls = vec![
1212            PendingToolCall {
1213                id: "1".into(),
1214                name: "read_file".into(),
1215                input: Value::Null,
1216            },
1217            PendingToolCall {
1218                id: "2".into(),
1219                name: "glob".into(),
1220                input: Value::Null,
1221            },
1222            PendingToolCall {
1223                id: "3".into(),
1224                name: "web_search".into(),
1225                input: Value::Null,
1226            },
1227        ];
1228        let waves = partition_into_waves(&calls, &HashMap::new());
1229        assert_eq!(waves, vec![vec![0, 1, 2]]);
1230    }
1231
1232    #[test]
1233    fn test_partition_mixed_readonly_sideeffect() {
1234        let calls = vec![
1235            PendingToolCall {
1236                id: "1".into(),
1237                name: "read_file".into(),
1238                input: Value::Null,
1239            },
1240            PendingToolCall {
1241                id: "2".into(),
1242                name: "write_file".into(),
1243                input: Value::Null,
1244            },
1245            PendingToolCall {
1246                id: "3".into(),
1247                name: "glob".into(),
1248                input: Value::Null,
1249            },
1250        ];
1251        let waves = partition_into_waves(&calls, &HashMap::new());
1252        // Wave 1: readonly [0, 2], Wave 2: sideeffect [1]
1253        assert_eq!(waves, vec![vec![0, 2], vec![1]]);
1254    }
1255
1256    #[test]
1257    fn test_partition_exclusive_alone() {
1258        let calls = vec![
1259            PendingToolCall {
1260                id: "1".into(),
1261                name: "read_file".into(),
1262                input: Value::Null,
1263            },
1264            PendingToolCall {
1265                id: "2".into(),
1266                name: "bash".into(),
1267                input: Value::Null,
1268            },
1269            PendingToolCall {
1270                id: "3".into(),
1271                name: "glob".into(),
1272                input: Value::Null,
1273            },
1274        ];
1275        let waves = partition_into_waves(&calls, &HashMap::new());
1276        // Wave 1: readonly [0, 2], Wave 2: exclusive [1]
1277        assert_eq!(waves, vec![vec![0, 2], vec![1]]);
1278    }
1279
1280    #[test]
1281    fn test_partition_config_override() {
1282        let mut overrides = HashMap::new();
1283        overrides.insert("mcp_custom_search".to_string(), "read_only".to_string());
1284
1285        let calls = vec![
1286            PendingToolCall {
1287                id: "1".into(),
1288                name: "read_file".into(),
1289                input: Value::Null,
1290            },
1291            PendingToolCall {
1292                id: "2".into(),
1293                name: "mcp_custom_search".into(),
1294                input: Value::Null,
1295            },
1296        ];
1297        let waves = partition_into_waves(&calls, &overrides);
1298        // Both should be in single parallel wave (mcp_custom_search overridden to ReadOnly)
1299        assert_eq!(waves, vec![vec![0, 1]]);
1300    }
1301
1302    #[test]
1303    fn test_partition_empty() {
1304        let calls: Vec<PendingToolCall> = vec![];
1305        let waves = partition_into_waves(&calls, &HashMap::new());
1306        assert!(waves.is_empty());
1307    }
1308
1309    #[test]
1310    fn successful_todo_write_emits_structured_plan() {
1311        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
1312        emit_plan_update_event(
1313            &tx,
1314            "todo_write",
1315            &serde_json::json!({
1316                "chat_id": 42,
1317                "todos": [
1318                    {"task": "Inspect", "status": "completed"},
1319                    {"task": "Implement", "status": "in_progress"},
1320                    {"task": "Verify", "status": "pending"}
1321                ]
1322            }),
1323            false,
1324        );
1325
1326        assert_eq!(
1327            rx.try_recv().unwrap(),
1328            AgentEvent::PlanUpdated {
1329                steps: vec![
1330                    RuntimePlanStep {
1331                        title: "Inspect".into(),
1332                        status: RuntimePlanStepStatus::Completed,
1333                    },
1334                    RuntimePlanStep {
1335                        title: "Implement".into(),
1336                        status: RuntimePlanStepStatus::InProgress,
1337                    },
1338                    RuntimePlanStep {
1339                        title: "Verify".into(),
1340                        status: RuntimePlanStepStatus::Pending,
1341                    },
1342                ],
1343            }
1344        );
1345    }
1346
1347    #[test]
1348    fn failed_todo_write_does_not_publish_a_plan() {
1349        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
1350        emit_plan_update_event(
1351            &tx,
1352            "todo_write",
1353            &serde_json::json!({"todos": [{"task": "Nope", "status": "pending"}]}),
1354            true,
1355        );
1356        assert!(rx.try_recv().is_err());
1357    }
1358
1359    #[test]
1360    fn bash_result_emits_bounded_redacted_verification_evidence() {
1361        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
1362        let secret = "sk-proj-abcdefghijklmnopqrstuvwxyz0123456789";
1363        let output = format!(
1364            "{secret}\n{}\nTAIL",
1365            "x".repeat(MAX_PROCESS_OUTPUT_BYTES + 100)
1366        );
1367        let result = ToolResult::success(output).with_status_code(0);
1368
1369        emit_process_output_event(
1370            &tx,
1371            "call-check",
1372            "bash",
1373            &serde_json::json!({"command": "cargo test -p microclaw-core"}),
1374            &result,
1375            321,
1376        );
1377
1378        let event = rx.try_recv().unwrap();
1379        let AgentEvent::ProcessOutput {
1380            call_id,
1381            output,
1382            exit_code,
1383            duration_ms,
1384            truncated,
1385            kind,
1386            ..
1387        } = event
1388        else {
1389            panic!("expected process output event");
1390        };
1391        assert_eq!(call_id, "call-check");
1392        assert_eq!(exit_code, Some(0));
1393        assert_eq!(duration_ms, 321);
1394        assert!(truncated);
1395        assert_eq!(kind, RuntimeProcessKind::Verification);
1396        assert!(!output.contains(secret));
1397        assert!(output.to_ascii_lowercase().contains("redacted"));
1398        assert!(output.ends_with("TAIL"));
1399    }
1400
1401    #[test]
1402    fn non_process_tool_does_not_emit_process_output() {
1403        let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
1404        emit_process_output_event(
1405            &tx,
1406            "call-read",
1407            "read_file",
1408            &serde_json::json!({"path": "README.md"}),
1409            &ToolResult::success("contents".into()),
1410            1,
1411        );
1412        assert!(rx.try_recv().is_err());
1413    }
1414}