Skip to main content

microclaw_engine/tools/
background_process.rs

1//! Background shell processes for long-running commands.
2//!
3//! `bash` with `run_in_background: true` starts a host process (dev server,
4//! file watcher, long test suite) and returns immediately with an id. The
5//! `bash_output` tool returns output produced since the previous read plus the
6//! exit status, and `bash_kill` stops the process. Processes are scoped to the
7//! session that started them. The registry belongs to a `ToolRegistry`, so
8//! every process still running is killed when that registry is dropped.
9
10use std::collections::HashMap;
11use std::path::Path;
12use std::sync::{Arc, Mutex};
13
14use async_trait::async_trait;
15use serde_json::json;
16use tokio::io::AsyncReadExt;
17
18use microclaw_core::llm_types::ToolDefinition;
19use microclaw_core::text::floor_char_boundary;
20
21use super::{schema_object, Tool, ToolResult};
22
23/// Retained output per process; older bytes are dropped first.
24const MAX_BUFFERED_BYTES: usize = 512 * 1024;
25/// Output returned by a single `bash_output` call.
26const MAX_RETURNED_BYTES: usize = 30_000;
27/// Concurrent background processes per session.
28const MAX_PROCESSES_PER_SESSION: usize = 8;
29
30#[derive(Default)]
31struct OutputBuffer {
32    text: String,
33    /// Absolute offset of `text[0]` in the full output stream.
34    base: usize,
35    /// Absolute offset the agent has read up to.
36    cursor: usize,
37}
38
39impl OutputBuffer {
40    fn push(&mut self, chunk: &str) {
41        self.text.push_str(chunk);
42        if self.text.len() > MAX_BUFFERED_BYTES {
43            let excess = self.text.len() - MAX_BUFFERED_BYTES;
44            let cut = floor_char_boundary(&self.text, excess);
45            self.text.drain(..cut);
46            self.base += cut;
47        }
48    }
49
50    /// Return unread output (bounded) and whether earlier output was dropped.
51    fn take_unread(&mut self) -> (String, bool) {
52        let mut dropped = false;
53        if self.cursor < self.base {
54            self.cursor = self.base;
55            dropped = true;
56        }
57        let start = self.cursor - self.base;
58        let mut end = self.text.len();
59        if end - start > MAX_RETURNED_BYTES {
60            end = floor_char_boundary(&self.text, start + MAX_RETURNED_BYTES);
61        }
62        let out = self.text[start..end].to_string();
63        self.cursor = self.base + end;
64        (out, dropped)
65    }
66
67    fn has_unread(&self) -> bool {
68        self.base + self.text.len() > self.cursor
69    }
70}
71
72struct BackgroundProcess {
73    session: String,
74    command: String,
75    started_at: chrono::DateTime<chrono::Utc>,
76    output: Arc<Mutex<OutputBuffer>>,
77    exit_code: Arc<Mutex<Option<i32>>>,
78    kill_tx: Mutex<Option<tokio::sync::oneshot::Sender<()>>>,
79    /// The process's stdin, for `bash_input` (`None` once closed).
80    stdin: tokio::sync::Mutex<Option<tokio::process::ChildStdin>>,
81}
82
83/// Background processes started through one tool registry.
84#[derive(Default)]
85pub struct ProcessRegistry {
86    processes: Mutex<HashMap<String, Arc<BackgroundProcess>>>,
87}
88
89impl Drop for ProcessRegistry {
90    fn drop(&mut self) {
91        let processes = self.processes.get_mut().unwrap_or_else(|e| e.into_inner());
92        for process in processes.values() {
93            if let Some(tx) = lock(&process.kill_tx).take() {
94                let _ = tx.send(());
95            }
96        }
97    }
98}
99
100fn lock<T>(mutex: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
101    mutex.lock().unwrap_or_else(|e| e.into_inner())
102}
103
104fn next_id() -> String {
105    let short = uuid::Uuid::new_v4().simple().to_string();
106    format!("bg_{}", &short[..8])
107}
108
109/// Session key used to scope background processes to the chat that owns them.
110pub fn session_key(input: &serde_json::Value) -> String {
111    super::auth_context_from_input(input)
112        .map(|auth| format!("{}-{}", auth.caller_channel, auth.caller_chat_id))
113        .unwrap_or_else(|| "shared".to_string())
114}
115
116impl ProcessRegistry {
117    /// Spawn `command` in `working_dir` on the host and register it.
118    pub fn spawn(
119        &self,
120        session: &str,
121        command: &str,
122        working_dir: &Path,
123        envs: &HashMap<String, String>,
124    ) -> Result<String, String> {
125        let spec = crate::internal::tool_runtime::command_runner::shell_command(command);
126        self.spawn_spec(session, command, &spec, working_dir, envs)
127    }
128
129    /// Spawn an already-built command line (e.g. wrapped by the local OS
130    /// sandbox) and register it.
131    pub fn spawn_spec(
132        &self,
133        session: &str,
134        command: &str,
135        spec: &crate::internal::tool_runtime::command_runner::CommandSpec,
136        working_dir: &Path,
137        envs: &HashMap<String, String>,
138    ) -> Result<String, String> {
139        let running = lock(&self.processes)
140            .values()
141            .filter(|p| p.session == session && lock(&p.exit_code).is_none())
142            .count();
143        if running >= MAX_PROCESSES_PER_SESSION {
144            return Err(format!(
145            "Too many background processes are running ({running}). Stop one with bash_kill first."
146        ));
147        }
148        let mut cmd =
149            crate::internal::tool_runtime::command_runner::build_command(spec, Some(working_dir));
150        for (k, v) in envs {
151            cmd.env(k, v);
152        }
153        cmd.stdout(std::process::Stdio::piped())
154            .stderr(std::process::Stdio::piped())
155            .stdin(std::process::Stdio::piped())
156            .kill_on_drop(true);
157        let mut child = cmd
158            .spawn()
159            .map_err(|e| format!("Failed to start background command: {e}"))?;
160        let stdin = child.stdin.take();
161
162        let id = next_id();
163        let output = Arc::new(Mutex::new(OutputBuffer::default()));
164        let exit_code = Arc::new(Mutex::new(None));
165        let (kill_tx, mut kill_rx) = tokio::sync::oneshot::channel::<()>();
166
167        let readers: Vec<_> = [
168            child
169                .stdout
170                .take()
171                .map(|s| Box::new(s) as Box<dyn tokio::io::AsyncRead + Unpin + Send>),
172            child
173                .stderr
174                .take()
175                .map(|s| Box::new(s) as Box<dyn tokio::io::AsyncRead + Unpin + Send>),
176        ]
177        .into_iter()
178        .flatten()
179        .map(|mut stream| {
180            let output = output.clone();
181            tokio::spawn(async move {
182                let mut buf = vec![0u8; 8192];
183                loop {
184                    match stream.read(&mut buf).await {
185                        Ok(0) | Err(_) => break,
186                        Ok(n) => lock(&output).push(&String::from_utf8_lossy(&buf[..n])),
187                    }
188                }
189            })
190        })
191        .collect();
192
193        {
194            let exit_code = exit_code.clone();
195            tokio::spawn(async move {
196                let status = tokio::select! {
197                    status = child.wait() => status.ok().and_then(|s| s.code()).unwrap_or(-1),
198                    _ = &mut kill_rx => {
199                        let _ = child.kill().await;
200                        -9
201                    }
202                };
203                for reader in readers {
204                    let _ = reader.await;
205                }
206                *lock(&exit_code) = Some(status);
207            });
208        }
209
210        lock(&self.processes).insert(
211            id.clone(),
212            Arc::new(BackgroundProcess {
213                session: session.to_string(),
214                command: command.to_string(),
215                started_at: chrono::Utc::now(),
216                output,
217                exit_code,
218                kill_tx: Mutex::new(Some(kill_tx)),
219                stdin: tokio::sync::Mutex::new(stdin),
220            }),
221        );
222        Ok(id)
223    }
224
225    fn lookup(&self, session: &str, id: &str) -> Result<Arc<BackgroundProcess>, String> {
226        let registry = lock(&self.processes);
227        match registry.get(id) {
228            Some(process) if process.session == session => Ok(process.clone()),
229            _ => {
230                let known: Vec<_> = registry
231                    .iter()
232                    .filter(|(_, p)| p.session == session)
233                    .map(|(id, p)| format!("{id} ({})", p.command))
234                    .collect();
235                let hint = if known.is_empty() {
236                    "No background processes exist in this session.".to_string()
237                } else {
238                    format!("Known processes: {}", known.join(", "))
239                };
240                Err(format!("Unknown background process '{id}'. {hint}"))
241            }
242        }
243    }
244}
245
246fn status_line(process: &BackgroundProcess) -> String {
247    let elapsed = (chrono::Utc::now() - process.started_at)
248        .num_seconds()
249        .max(0);
250    match *lock(&process.exit_code) {
251        Some(code) => format!("status: exited with code {code} (after {elapsed}s)"),
252        None => format!("status: running ({elapsed}s)"),
253    }
254}
255
256/// Write to a background process's stdin: answer prompts, drive a REPL or an
257/// interactive CLI started with `run_in_background`.
258pub struct BashInputTool(pub Arc<ProcessRegistry>);
259
260/// How long `bash_input` waits for the process to answer.
261const INPUT_SETTLE_MS: u64 = 400;
262
263#[async_trait]
264impl Tool for BashInputTool {
265    fn name(&self) -> &str {
266        "bash_input"
267    }
268
269    fn definition(&self) -> ToolDefinition {
270        ToolDefinition {
271            name: "bash_input".into(),
272            description: "Send text to the stdin of a background process started with bash run_in_background (a REPL, a debugger, a command waiting for an answer), then return the output it produced. Include a trailing newline to submit a line. Set close to true to send end-of-input. The process has no terminal, so start REPLs in a line-oriented, unbuffered mode (e.g. `python3 -u -i`, `node -i`).".into(),
273            input_schema: schema_object(
274                json!({
275                    "id": {
276                        "type": "string",
277                        "description": "Background process id returned by bash (e.g. bg_1a2b3c4d)"
278                    },
279                    "input": {
280                        "type": "string",
281                        "description": "Text to write; usually ends with \\n"
282                    },
283                    "close": {
284                        "type": "boolean",
285                        "description": "Close stdin after writing (end of input)"
286                    },
287                    "wait_ms": {
288                        "type": "integer",
289                        "description": "How long to wait for output before returning (default 400, max 10000)"
290                    }
291                }),
292                &["id"],
293            ),
294        }
295    }
296
297    async fn execute(&self, input: serde_json::Value) -> ToolResult {
298        let session = session_key(&input);
299        let Some(id) = input.get("id").and_then(|v| v.as_str()) else {
300            return ToolResult::error("Missing 'id' parameter".into());
301        };
302        let process = match self.0.lookup(&session, id) {
303            Ok(process) => process,
304            Err(msg) => return ToolResult::error(msg),
305        };
306        let text = input.get("input").and_then(|v| v.as_str()).unwrap_or("");
307        let close = input
308            .get("close")
309            .and_then(|v| v.as_bool())
310            .unwrap_or(false);
311        {
312            let mut stdin = process.stdin.lock().await;
313            let Some(pipe) = stdin.as_mut() else {
314                return ToolResult::error(format!("{id}: stdin is already closed"));
315            };
316            if !text.is_empty() {
317                use tokio::io::AsyncWriteExt;
318                if let Err(e) = pipe.write_all(text.as_bytes()).await {
319                    return ToolResult::error(format!(
320                        "{id}: could not write to stdin ({e}); {}",
321                        status_line(&process)
322                    ));
323                }
324                let _ = pipe.flush().await;
325            }
326            if close {
327                *stdin = None;
328            }
329        }
330        let wait = input
331            .get("wait_ms")
332            .and_then(|v| v.as_u64())
333            .unwrap_or(INPUT_SETTLE_MS)
334            .min(10_000);
335        tokio::time::sleep(std::time::Duration::from_millis(wait)).await;
336        let (output, dropped) = lock(&process.output).take_unread();
337        let mut text = format!("{}\n", status_line(&process));
338        if dropped {
339            text.push_str("[earlier output was dropped]\n");
340        }
341        if output.is_empty() {
342            text.push_str("(no new output yet; read it later with bash_output)");
343        } else {
344            text.push_str(&output);
345        }
346        ToolResult::success(text)
347    }
348}
349
350pub struct BashOutputTool(pub Arc<ProcessRegistry>);
351
352#[async_trait]
353impl Tool for BashOutputTool {
354    fn name(&self) -> &str {
355        "bash_output"
356    }
357
358    fn definition(&self) -> ToolDefinition {
359        ToolDefinition {
360            name: "bash_output".into(),
361            description: "Read new output from a background process started with bash run_in_background. Returns output produced since the previous read and whether the process is still running. Omit id to list this session's background processes.".into(),
362            input_schema: schema_object(
363                json!({
364                    "id": {
365                        "type": "string",
366                        "description": "Background process id returned by bash (e.g. bg_1a2b3c4d)"
367                    },
368                    "filter": {
369                        "type": "string",
370                        "description": "Optional regex; only matching output lines are returned"
371                    }
372                }),
373                &[],
374            ),
375        }
376    }
377
378    async fn execute(&self, input: serde_json::Value) -> ToolResult {
379        let session = session_key(&input);
380        let Some(id) = input.get("id").and_then(|v| v.as_str()) else {
381            let registry = lock(&self.0.processes);
382            let mut rows: Vec<_> = registry
383                .iter()
384                .filter(|(_, p)| p.session == session)
385                .map(|(id, p)| {
386                    (
387                        p.started_at,
388                        format!("{id}  {}  {}", status_line(p), p.command),
389                    )
390                })
391                .collect();
392            rows.sort();
393            return if rows.is_empty() {
394                ToolResult::success("No background processes in this session.".into())
395            } else {
396                ToolResult::success(
397                    rows.into_iter()
398                        .map(|(_, row)| row)
399                        .collect::<Vec<_>>()
400                        .join("\n"),
401                )
402            };
403        };
404        let process = match self.0.lookup(&session, id) {
405            Ok(p) => p,
406            Err(msg) => return ToolResult::error(msg),
407        };
408        let filter = match input.get("filter").and_then(|v| v.as_str()) {
409            Some(raw) => match regex::Regex::new(raw) {
410                Ok(re) => Some(re),
411                Err(e) => return ToolResult::error(format!("Invalid filter regex: {e}")),
412            },
413            None => None,
414        };
415        let (mut text, dropped) = lock(&process.output).take_unread();
416        if let Some(re) = filter {
417            text = text
418                .lines()
419                .filter(|line| re.is_match(line))
420                .collect::<Vec<_>>()
421                .join("\n");
422        }
423        let more = lock(&process.output).has_unread();
424        let mut out = format!("{id}  {}\n", status_line(&process));
425        if dropped {
426            out.push_str("(older output was dropped; only the most recent output is retained)\n");
427        }
428        if text.is_empty() {
429            out.push_str("(no new output)");
430        } else {
431            out.push_str(&text);
432        }
433        if more {
434            out.push_str("\n... (more output available; call bash_output again)");
435        }
436        ToolResult::success(out)
437    }
438}
439
440pub struct BashKillTool(pub Arc<ProcessRegistry>);
441
442#[async_trait]
443impl Tool for BashKillTool {
444    fn name(&self) -> &str {
445        "bash_kill"
446    }
447
448    fn definition(&self) -> ToolDefinition {
449        ToolDefinition {
450            name: "bash_kill".into(),
451            description: "Stop a background process started with bash run_in_background.".into(),
452            input_schema: schema_object(
453                json!({
454                    "id": {
455                        "type": "string",
456                        "description": "Background process id returned by bash"
457                    }
458                }),
459                &["id"],
460            ),
461        }
462    }
463
464    async fn execute(&self, input: serde_json::Value) -> ToolResult {
465        let session = session_key(&input);
466        let Some(id) = input.get("id").and_then(|v| v.as_str()) else {
467            return ToolResult::error("Missing 'id' parameter".into());
468        };
469        let process = match self.0.lookup(&session, id) {
470            Ok(p) => p,
471            Err(msg) => return ToolResult::error(msg),
472        };
473        if lock(&process.exit_code).is_some() {
474            return ToolResult::success(format!(
475                "{id} already finished; {}",
476                status_line(&process)
477            ));
478        }
479        if let Some(tx) = lock(&process.kill_tx).take() {
480            let _ = tx.send(());
481        }
482        for _ in 0..50 {
483            if lock(&process.exit_code).is_some() {
484                break;
485            }
486            tokio::time::sleep(std::time::Duration::from_millis(20)).await;
487        }
488        ToolResult::success(format!("Stopped {id}; {}", status_line(&process)))
489    }
490}
491
492#[cfg(test)]
493mod tests {
494    use super::*;
495
496    #[test]
497    fn output_buffer_tracks_cursor_and_drops_oldest() {
498        let mut buf = OutputBuffer::default();
499        buf.push("hello ");
500        buf.push("world");
501        assert_eq!(buf.take_unread(), ("hello world".to_string(), false));
502        assert!(!buf.has_unread());
503        buf.push(&"x".repeat(MAX_BUFFERED_BYTES + 10));
504        let (text, dropped) = buf.take_unread();
505        assert!(dropped);
506        assert_eq!(text.len(), MAX_RETURNED_BYTES);
507        assert!(buf.has_unread());
508    }
509
510    #[cfg(unix)]
511    #[tokio::test]
512    async fn bash_input_drives_an_interactive_process() {
513        let dir = tempfile::tempdir().unwrap();
514        let auth = json!({"caller_channel": "test", "caller_chat_id": 7, "control_chat_ids": []});
515        let session = session_key(&json!({"__microclaw_auth": auth.clone()}));
516        let registry = Arc::new(ProcessRegistry::default());
517        let input_tool = BashInputTool(registry.clone());
518        // A line-oriented "REPL": echoes each line upper-cased until EOF.
519        let id = registry
520            .spawn(
521                &session,
522                "while read line; do echo \"got:$line\" | tr a-z A-Z; done; echo bye",
523                dir.path(),
524                &HashMap::new(),
525            )
526            .unwrap();
527        let first = input_tool
528            .execute(json!({"id": id, "input": "hello\n", "wait_ms": 1000, "__microclaw_auth": auth.clone()}))
529            .await;
530        assert!(!first.is_error, "{}", first.content);
531        assert!(first.content.contains("GOT:HELLO"), "{}", first.content);
532        let last = input_tool
533            .execute(json!({"id": id, "input": "again\n", "close": true, "wait_ms": 1000, "__microclaw_auth": auth.clone()}))
534            .await;
535        assert!(last.content.contains("GOT:AGAIN"), "{}", last.content);
536        assert!(last.content.contains("bye"), "{}", last.content);
537        let closed = input_tool
538            .execute(json!({"id": id, "input": "x\n", "__microclaw_auth": auth}))
539            .await;
540        assert!(closed.is_error);
541    }
542
543    #[cfg(unix)]
544    #[tokio::test]
545    async fn background_process_streams_output_and_can_be_killed() {
546        let dir = tempfile::tempdir().unwrap();
547        let input = |extra: serde_json::Value| {
548            let mut value = json!({
549                "__microclaw_auth": {
550                    "caller_channel": "test",
551                    "caller_chat_id": 1,
552                    "control_chat_ids": []
553                }
554            });
555            if let (Some(map), Some(extra)) = (value.as_object_mut(), extra.as_object()) {
556                map.extend(extra.clone());
557            }
558            value
559        };
560        let session = session_key(&input(json!({})));
561        let registry = Arc::new(ProcessRegistry::default());
562        let output_tool = BashOutputTool(registry.clone());
563        let kill_tool = BashKillTool(registry.clone());
564        let id = registry
565            .spawn(
566                &session,
567                "echo started; sleep 30",
568                dir.path(),
569                &HashMap::new(),
570            )
571            .unwrap();
572        let mut seen = String::new();
573        for _ in 0..100 {
574            let result = output_tool.execute(input(json!({"id": id}))).await;
575            seen.push_str(&result.content);
576            if seen.contains("started") {
577                break;
578            }
579            tokio::time::sleep(std::time::Duration::from_millis(20)).await;
580        }
581        assert!(seen.contains("started"), "{seen}");
582        assert!(seen.contains("running"));
583
584        let listed = output_tool.execute(input(json!({}))).await;
585        assert!(listed.content.contains(&id));
586
587        let killed = kill_tool.execute(input(json!({"id": id}))).await;
588        assert!(killed.content.contains("exited"), "{}", killed.content);
589
590        let other_session = output_tool
591            .execute(json!({"id": id, "__microclaw_auth": {"caller_channel": "other", "caller_chat_id": 2, "control_chat_ids": []}}))
592            .await;
593        assert!(other_session.is_error);
594    }
595
596    #[cfg(unix)]
597    #[tokio::test]
598    async fn dropping_the_registry_kills_running_processes() {
599        let dir = tempfile::tempdir().unwrap();
600        let registry = Arc::new(ProcessRegistry::default());
601        let id = registry
602            .spawn("s", "sleep 30", dir.path(), &HashMap::new())
603            .unwrap();
604        let process = registry.lookup("s", &id).unwrap();
605        drop(registry);
606        for _ in 0..100 {
607            if lock(&process.exit_code).is_some() {
608                break;
609            }
610            tokio::time::sleep(std::time::Duration::from_millis(20)).await;
611        }
612        assert!(
613            lock(&process.exit_code).is_some(),
614            "process should be killed"
615        );
616    }
617}