Skip to main content

microclaw_engine/tools/
background_process.rs

1//! Background shell processes for long-running commands.
2//!
3//! `bash` with `run_in_background: true` starts a host process (dev server,
4//! file watcher, long test suite) and returns immediately with an id. The
5//! `bash_output` tool returns output produced since the previous read plus the
6//! exit status, and `bash_kill` stops the process. Processes are scoped to the
7//! session that started them. The registry belongs to a `ToolRegistry`, so
8//! every process still running is killed when that registry is dropped.
9
10use std::collections::HashMap;
11use std::path::Path;
12use std::sync::{Arc, Mutex};
13
14use async_trait::async_trait;
15use serde_json::json;
16use tokio::io::AsyncReadExt;
17
18use microclaw_core::llm_types::ToolDefinition;
19use microclaw_core::text::floor_char_boundary;
20
21use super::{schema_object, Tool, ToolResult};
22
23/// Retained output per process; older bytes are dropped first.
24const MAX_BUFFERED_BYTES: usize = 512 * 1024;
25/// Output returned by a single `bash_output` call.
26const MAX_RETURNED_BYTES: usize = 30_000;
27/// Concurrent background processes per session.
28const MAX_PROCESSES_PER_SESSION: usize = 8;
29
30#[derive(Default)]
31struct OutputBuffer {
32    text: String,
33    /// Absolute offset of `text[0]` in the full output stream.
34    base: usize,
35    /// Absolute offset the agent has read up to.
36    cursor: usize,
37}
38
39impl OutputBuffer {
40    fn push(&mut self, chunk: &str) {
41        self.text.push_str(chunk);
42        if self.text.len() > MAX_BUFFERED_BYTES {
43            let excess = self.text.len() - MAX_BUFFERED_BYTES;
44            let cut = floor_char_boundary(&self.text, excess);
45            self.text.drain(..cut);
46            self.base += cut;
47        }
48    }
49
50    /// Return unread output (bounded) and whether earlier output was dropped.
51    fn take_unread(&mut self) -> (String, bool) {
52        let mut dropped = false;
53        if self.cursor < self.base {
54            self.cursor = self.base;
55            dropped = true;
56        }
57        let start = self.cursor - self.base;
58        let mut end = self.text.len();
59        if end - start > MAX_RETURNED_BYTES {
60            end = floor_char_boundary(&self.text, start + MAX_RETURNED_BYTES);
61        }
62        let out = self.text[start..end].to_string();
63        self.cursor = self.base + end;
64        (out, dropped)
65    }
66
67    fn has_unread(&self) -> bool {
68        self.base + self.text.len() > self.cursor
69    }
70}
71
72struct BackgroundProcess {
73    session: String,
74    command: String,
75    started_at: chrono::DateTime<chrono::Utc>,
76    output: Arc<Mutex<OutputBuffer>>,
77    exit_code: Arc<Mutex<Option<i32>>>,
78    kill_tx: Mutex<Option<tokio::sync::oneshot::Sender<()>>>,
79    /// The process's stdin, for `bash_input` (`None` once closed).
80    stdin: tokio::sync::Mutex<Option<tokio::process::ChildStdin>>,
81}
82
83/// Background processes started through one tool registry.
84#[derive(Default)]
85pub struct ProcessRegistry {
86    processes: Mutex<HashMap<String, Arc<BackgroundProcess>>>,
87}
88
89impl Drop for ProcessRegistry {
90    fn drop(&mut self) {
91        let processes = self.processes.get_mut().unwrap_or_else(|e| e.into_inner());
92        for process in processes.values() {
93            if let Some(tx) = lock(&process.kill_tx).take() {
94                let _ = tx.send(());
95            }
96        }
97    }
98}
99
100/// SIGKILL the process group led by `pid` (see `process_group(0)`).
101fn kill_group(pid: Option<u32>) {
102    #[cfg(unix)]
103    if let Some(pid) = pid.and_then(|p| libc::pid_t::try_from(p).ok()) {
104        // SAFETY: plain syscall on a group id this registry created.
105        unsafe {
106            libc::killpg(pid, libc::SIGKILL);
107        }
108    }
109    #[cfg(not(unix))]
110    let _ = pid;
111}
112
113fn lock<T>(mutex: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
114    mutex.lock().unwrap_or_else(|e| e.into_inner())
115}
116
117fn next_id() -> String {
118    let short = uuid::Uuid::new_v4().simple().to_string();
119    format!("bg_{}", &short[..8])
120}
121
122/// Session key used to scope background processes to the chat that owns them.
123pub fn session_key(input: &serde_json::Value) -> String {
124    super::auth_context_from_input(input)
125        .map(|auth| format!("{}-{}", auth.caller_channel, auth.caller_chat_id))
126        .unwrap_or_else(|| "shared".to_string())
127}
128
129impl ProcessRegistry {
130    /// Spawn `command` in `working_dir` on the host and register it.
131    pub fn spawn(
132        &self,
133        session: &str,
134        command: &str,
135        working_dir: &Path,
136        envs: &HashMap<String, String>,
137    ) -> Result<String, String> {
138        let spec = crate::internal::tool_runtime::command_runner::shell_command(command);
139        self.spawn_spec(session, command, &spec, working_dir, envs)
140    }
141
142    /// Spawn an already-built command line (e.g. wrapped by the local OS
143    /// sandbox) and register it.
144    pub fn spawn_spec(
145        &self,
146        session: &str,
147        command: &str,
148        spec: &crate::internal::tool_runtime::command_runner::CommandSpec,
149        working_dir: &Path,
150        envs: &HashMap<String, String>,
151    ) -> Result<String, String> {
152        let running = lock(&self.processes)
153            .values()
154            .filter(|p| p.session == session && lock(&p.exit_code).is_none())
155            .count();
156        if running >= MAX_PROCESSES_PER_SESSION {
157            return Err(format!(
158            "Too many background processes are running ({running}). Stop one with bash_kill first."
159        ));
160        }
161        let mut cmd =
162            crate::internal::tool_runtime::command_runner::build_command(spec, Some(working_dir));
163        for (k, v) in envs {
164            cmd.env(k, v);
165        }
166        cmd.stdout(std::process::Stdio::piped())
167            .stderr(std::process::Stdio::piped())
168            .stdin(std::process::Stdio::piped())
169            .kill_on_drop(true);
170        // Its own process group, so bash_kill stops the whole command
171        // (`cd app && npm run dev` leaves node as a grandchild that holds the
172        // output pipes), not only the shell.
173        #[cfg(unix)]
174        cmd.process_group(0);
175        let mut child = cmd
176            .spawn()
177            .map_err(|e| format!("Failed to start background command: {e}"))?;
178        let stdin = child.stdin.take();
179        let pid = child.id();
180
181        let id = next_id();
182        let output = Arc::new(Mutex::new(OutputBuffer::default()));
183        let exit_code = Arc::new(Mutex::new(None));
184        let (kill_tx, mut kill_rx) = tokio::sync::oneshot::channel::<()>();
185
186        let readers: Vec<_> = [
187            child
188                .stdout
189                .take()
190                .map(|s| Box::new(s) as Box<dyn tokio::io::AsyncRead + Unpin + Send>),
191            child
192                .stderr
193                .take()
194                .map(|s| Box::new(s) as Box<dyn tokio::io::AsyncRead + Unpin + Send>),
195        ]
196        .into_iter()
197        .flatten()
198        .map(|mut stream| {
199            let output = output.clone();
200            tokio::spawn(async move {
201                let mut buf = vec![0u8; 8192];
202                loop {
203                    match stream.read(&mut buf).await {
204                        Ok(0) | Err(_) => break,
205                        Ok(n) => lock(&output).push(&String::from_utf8_lossy(&buf[..n])),
206                    }
207                }
208            })
209        })
210        .collect();
211
212        {
213            let exit_code = exit_code.clone();
214            tokio::spawn(async move {
215                let status = tokio::select! {
216                    status = child.wait() => status.ok().and_then(|s| s.code()).unwrap_or(-1),
217                    _ = &mut kill_rx => {
218                        kill_group(pid);
219                        let _ = child.kill().await;
220                        -9
221                    }
222                };
223                // A process that left the group can keep the pipes open;
224                // after a kill, do not wait for it.
225                for reader in readers {
226                    if status == -9 {
227                        let _ =
228                            tokio::time::timeout(std::time::Duration::from_secs(2), reader).await;
229                    } else {
230                        let _ = reader.await;
231                    }
232                }
233                *lock(&exit_code) = Some(status);
234            });
235        }
236
237        lock(&self.processes).insert(
238            id.clone(),
239            Arc::new(BackgroundProcess {
240                session: session.to_string(),
241                command: command.to_string(),
242                started_at: chrono::Utc::now(),
243                output,
244                exit_code,
245                kill_tx: Mutex::new(Some(kill_tx)),
246                stdin: tokio::sync::Mutex::new(stdin),
247            }),
248        );
249        Ok(id)
250    }
251
252    fn lookup(&self, session: &str, id: &str) -> Result<Arc<BackgroundProcess>, String> {
253        let registry = lock(&self.processes);
254        match registry.get(id) {
255            Some(process) if process.session == session => Ok(process.clone()),
256            _ => {
257                let known: Vec<_> = registry
258                    .iter()
259                    .filter(|(_, p)| p.session == session)
260                    .map(|(id, p)| format!("{id} ({})", p.command))
261                    .collect();
262                let hint = if known.is_empty() {
263                    "No background processes exist in this session.".to_string()
264                } else {
265                    format!("Known processes: {}", known.join(", "))
266                };
267                Err(format!("Unknown background process '{id}'. {hint}"))
268            }
269        }
270    }
271}
272
273fn status_line(process: &BackgroundProcess) -> String {
274    let elapsed = (chrono::Utc::now() - process.started_at)
275        .num_seconds()
276        .max(0);
277    match *lock(&process.exit_code) {
278        Some(code) => format!("status: exited with code {code} (after {elapsed}s)"),
279        None => format!("status: running ({elapsed}s)"),
280    }
281}
282
283/// Write to a background process's stdin: answer prompts, drive a REPL or an
284/// interactive CLI started with `run_in_background`.
285pub struct BashInputTool(pub Arc<ProcessRegistry>);
286
287/// How long `bash_input` waits for the process to answer.
288const INPUT_SETTLE_MS: u64 = 400;
289
290#[async_trait]
291impl Tool for BashInputTool {
292    fn name(&self) -> &str {
293        "bash_input"
294    }
295
296    fn definition(&self) -> ToolDefinition {
297        ToolDefinition {
298            name: "bash_input".into(),
299            description: "Send text to the stdin of a background process started with bash run_in_background (a REPL, a debugger, a command waiting for an answer), then return the output it produced. Include a trailing newline to submit a line. Set close to true to send end-of-input. The process has no terminal, so start REPLs in a line-oriented, unbuffered mode (e.g. `python3 -u -i`, `node -i`).".into(),
300            input_schema: schema_object(
301                json!({
302                    "id": {
303                        "type": "string",
304                        "description": "Background process id returned by bash (e.g. bg_1a2b3c4d)"
305                    },
306                    "input": {
307                        "type": "string",
308                        "description": "Text to write; usually ends with \\n"
309                    },
310                    "close": {
311                        "type": "boolean",
312                        "description": "Close stdin after writing (end of input)"
313                    },
314                    "wait_ms": {
315                        "type": "integer",
316                        "description": "How long to wait for output before returning (default 400, max 10000)"
317                    }
318                }),
319                &["id"],
320            ),
321        }
322    }
323
324    async fn execute(&self, input: serde_json::Value) -> ToolResult {
325        let session = session_key(&input);
326        let Some(id) = input.get("id").and_then(|v| v.as_str()) else {
327            return ToolResult::error("Missing 'id' parameter".into());
328        };
329        let process = match self.0.lookup(&session, id) {
330            Ok(process) => process,
331            Err(msg) => return ToolResult::error(msg),
332        };
333        let text = input.get("input").and_then(|v| v.as_str()).unwrap_or("");
334        let close = input
335            .get("close")
336            .and_then(|v| v.as_bool())
337            .unwrap_or(false);
338        {
339            let mut stdin = process.stdin.lock().await;
340            let Some(pipe) = stdin.as_mut() else {
341                return ToolResult::error(format!("{id}: stdin is already closed"));
342            };
343            if !text.is_empty() {
344                use tokio::io::AsyncWriteExt;
345                if let Err(e) = pipe.write_all(text.as_bytes()).await {
346                    return ToolResult::error(format!(
347                        "{id}: could not write to stdin ({e}); {}",
348                        status_line(&process)
349                    ));
350                }
351                let _ = pipe.flush().await;
352            }
353            if close {
354                *stdin = None;
355            }
356        }
357        let wait = input
358            .get("wait_ms")
359            .and_then(|v| v.as_u64())
360            .unwrap_or(INPUT_SETTLE_MS)
361            .min(10_000);
362        tokio::time::sleep(std::time::Duration::from_millis(wait)).await;
363        let (output, dropped) = lock(&process.output).take_unread();
364        let mut text = format!("{}\n", status_line(&process));
365        if dropped {
366            text.push_str("[earlier output was dropped]\n");
367        }
368        if output.is_empty() {
369            text.push_str("(no new output yet; read it later with bash_output)");
370        } else {
371            text.push_str(&output);
372        }
373        ToolResult::success(text)
374    }
375}
376
377pub struct BashOutputTool(pub Arc<ProcessRegistry>);
378
379#[async_trait]
380impl Tool for BashOutputTool {
381    fn name(&self) -> &str {
382        "bash_output"
383    }
384
385    fn definition(&self) -> ToolDefinition {
386        ToolDefinition {
387            name: "bash_output".into(),
388            description: "Read new output from a background process started with bash run_in_background. Returns output produced since the previous read and whether the process is still running. Omit id to list this session's background processes.".into(),
389            input_schema: schema_object(
390                json!({
391                    "id": {
392                        "type": "string",
393                        "description": "Background process id returned by bash (e.g. bg_1a2b3c4d)"
394                    },
395                    "filter": {
396                        "type": "string",
397                        "description": "Optional regex; only matching output lines are returned"
398                    }
399                }),
400                &[],
401            ),
402        }
403    }
404
405    async fn execute(&self, input: serde_json::Value) -> ToolResult {
406        let session = session_key(&input);
407        let Some(id) = input.get("id").and_then(|v| v.as_str()) else {
408            let registry = lock(&self.0.processes);
409            let mut rows: Vec<_> = registry
410                .iter()
411                .filter(|(_, p)| p.session == session)
412                .map(|(id, p)| {
413                    (
414                        p.started_at,
415                        format!("{id}  {}  {}", status_line(p), p.command),
416                    )
417                })
418                .collect();
419            rows.sort();
420            return if rows.is_empty() {
421                ToolResult::success("No background processes in this session.".into())
422            } else {
423                ToolResult::success(
424                    rows.into_iter()
425                        .map(|(_, row)| row)
426                        .collect::<Vec<_>>()
427                        .join("\n"),
428                )
429            };
430        };
431        let process = match self.0.lookup(&session, id) {
432            Ok(p) => p,
433            Err(msg) => return ToolResult::error(msg),
434        };
435        let filter = match input.get("filter").and_then(|v| v.as_str()) {
436            Some(raw) => match regex::Regex::new(raw) {
437                Ok(re) => Some(re),
438                Err(e) => return ToolResult::error(format!("Invalid filter regex: {e}")),
439            },
440            None => None,
441        };
442        let (mut text, dropped) = lock(&process.output).take_unread();
443        if let Some(re) = filter {
444            text = text
445                .lines()
446                .filter(|line| re.is_match(line))
447                .collect::<Vec<_>>()
448                .join("\n");
449        }
450        let more = lock(&process.output).has_unread();
451        let mut out = format!("{id}  {}\n", status_line(&process));
452        if dropped {
453            out.push_str("(older output was dropped; only the most recent output is retained)\n");
454        }
455        if text.is_empty() {
456            out.push_str("(no new output)");
457        } else {
458            out.push_str(&text);
459        }
460        if more {
461            out.push_str("\n... (more output available; call bash_output again)");
462        }
463        ToolResult::success(out)
464    }
465}
466
467pub struct BashKillTool(pub Arc<ProcessRegistry>);
468
469#[async_trait]
470impl Tool for BashKillTool {
471    fn name(&self) -> &str {
472        "bash_kill"
473    }
474
475    fn definition(&self) -> ToolDefinition {
476        ToolDefinition {
477            name: "bash_kill".into(),
478            description: "Stop a background process started with bash run_in_background.".into(),
479            input_schema: schema_object(
480                json!({
481                    "id": {
482                        "type": "string",
483                        "description": "Background process id returned by bash"
484                    }
485                }),
486                &["id"],
487            ),
488        }
489    }
490
491    async fn execute(&self, input: serde_json::Value) -> ToolResult {
492        let session = session_key(&input);
493        let Some(id) = input.get("id").and_then(|v| v.as_str()) else {
494            return ToolResult::error("Missing 'id' parameter".into());
495        };
496        let process = match self.0.lookup(&session, id) {
497            Ok(p) => p,
498            Err(msg) => return ToolResult::error(msg),
499        };
500        if lock(&process.exit_code).is_some() {
501            return ToolResult::success(format!(
502                "{id} already finished; {}",
503                status_line(&process)
504            ));
505        }
506        if let Some(tx) = lock(&process.kill_tx).take() {
507            let _ = tx.send(());
508        }
509        for _ in 0..50 {
510            if lock(&process.exit_code).is_some() {
511                break;
512            }
513            tokio::time::sleep(std::time::Duration::from_millis(20)).await;
514        }
515        ToolResult::success(format!("Stopped {id}; {}", status_line(&process)))
516    }
517}
518
519#[cfg(test)]
520mod tests {
521    use super::*;
522
523    #[test]
524    fn output_buffer_tracks_cursor_and_drops_oldest() {
525        let mut buf = OutputBuffer::default();
526        buf.push("hello ");
527        buf.push("world");
528        assert_eq!(buf.take_unread(), ("hello world".to_string(), false));
529        assert!(!buf.has_unread());
530        buf.push(&"x".repeat(MAX_BUFFERED_BYTES + 10));
531        let (text, dropped) = buf.take_unread();
532        assert!(dropped);
533        assert_eq!(text.len(), MAX_RETURNED_BYTES);
534        assert!(buf.has_unread());
535    }
536
537    #[cfg(unix)]
538    #[tokio::test]
539    async fn bash_input_drives_an_interactive_process() {
540        let dir = tempfile::tempdir().unwrap();
541        let auth = json!({"caller_channel": "test", "caller_chat_id": 7, "control_chat_ids": []});
542        let session = session_key(&json!({"__microclaw_auth": auth.clone()}));
543        let registry = Arc::new(ProcessRegistry::default());
544        let input_tool = BashInputTool(registry.clone());
545        // A line-oriented "REPL": echoes each line upper-cased until EOF.
546        let id = registry
547            .spawn(
548                &session,
549                "while read line; do echo \"got:$line\" | tr a-z A-Z; done; echo bye",
550                dir.path(),
551                &HashMap::new(),
552            )
553            .unwrap();
554        let first = input_tool
555            .execute(json!({"id": id, "input": "hello\n", "wait_ms": 1000, "__microclaw_auth": auth.clone()}))
556            .await;
557        assert!(!first.is_error, "{}", first.content);
558        assert!(first.content.contains("GOT:HELLO"), "{}", first.content);
559        let last = input_tool
560            .execute(json!({"id": id, "input": "again\n", "close": true, "wait_ms": 1000, "__microclaw_auth": auth.clone()}))
561            .await;
562        assert!(last.content.contains("GOT:AGAIN"), "{}", last.content);
563        assert!(last.content.contains("bye"), "{}", last.content);
564        let closed = input_tool
565            .execute(json!({"id": id, "input": "x\n", "__microclaw_auth": auth}))
566            .await;
567        assert!(closed.is_error);
568    }
569
570    #[cfg(unix)]
571    #[tokio::test]
572    async fn background_process_streams_output_and_can_be_killed() {
573        let dir = tempfile::tempdir().unwrap();
574        let input = |extra: serde_json::Value| {
575            let mut value = json!({
576                "__microclaw_auth": {
577                    "caller_channel": "test",
578                    "caller_chat_id": 1,
579                    "control_chat_ids": []
580                }
581            });
582            if let (Some(map), Some(extra)) = (value.as_object_mut(), extra.as_object()) {
583                map.extend(extra.clone());
584            }
585            value
586        };
587        let session = session_key(&input(json!({})));
588        let registry = Arc::new(ProcessRegistry::default());
589        let output_tool = BashOutputTool(registry.clone());
590        let kill_tool = BashKillTool(registry.clone());
591        let id = registry
592            .spawn(
593                &session,
594                "echo started; sleep 30",
595                dir.path(),
596                &HashMap::new(),
597            )
598            .unwrap();
599        let mut seen = String::new();
600        for _ in 0..100 {
601            let result = output_tool.execute(input(json!({"id": id}))).await;
602            seen.push_str(&result.content);
603            if seen.contains("started") {
604                break;
605            }
606            tokio::time::sleep(std::time::Duration::from_millis(20)).await;
607        }
608        assert!(seen.contains("started"), "{seen}");
609        assert!(seen.contains("running"));
610
611        let listed = output_tool.execute(input(json!({}))).await;
612        assert!(listed.content.contains(&id));
613
614        let killed = kill_tool.execute(input(json!({"id": id}))).await;
615        assert!(killed.content.contains("exited"), "{}", killed.content);
616
617        let other_session = output_tool
618            .execute(json!({"id": id, "__microclaw_auth": {"caller_channel": "other", "caller_chat_id": 2, "control_chat_ids": []}}))
619            .await;
620        assert!(other_session.is_error);
621    }
622
623    #[cfg(unix)]
624    #[tokio::test]
625    async fn dropping_the_registry_kills_running_processes() {
626        let dir = tempfile::tempdir().unwrap();
627        let registry = Arc::new(ProcessRegistry::default());
628        let id = registry
629            .spawn("s", "sleep 30", dir.path(), &HashMap::new())
630            .unwrap();
631        let process = registry.lookup("s", &id).unwrap();
632        drop(registry);
633        for _ in 0..100 {
634            if lock(&process.exit_code).is_some() {
635                break;
636            }
637            tokio::time::sleep(std::time::Duration::from_millis(20)).await;
638        }
639        assert!(
640            lock(&process.exit_code).is_some(),
641            "process should be killed"
642        );
643    }
644}