1use std::borrow::Cow;
4use std::collections::VecDeque;
5use std::ffi::{OsStr, OsString};
6use std::fs::File;
7use std::io::Write as _;
8use std::path::{Path, PathBuf};
9use std::sync::Arc;
10use std::time::{Duration, Instant};
11
12use rig_core::completion::ToolAnnotations;
13use rig_core::redact;
14use rig_core::tool::{
15 ExitStatus, KeptEnd, OutputCut, Tool, ToolContext, ToolExecutionError, args_schema,
16};
17use schemars::JsonSchema;
18use serde::Deserialize;
19
20use crate::process::{Ended, KillOnDrop, Launcher, OutputLine, Process, Spawn};
21use crate::workspace::{Local, Workspace};
22use crate::{CUT_MARK, Live, MAX_LINE_CHARS, OutputLimit, Spill, blocking, clip};
23
24const OUTPUT_LIMIT: u64 = 10 * 1024 * 1024;
28
29#[derive(Clone, Debug)]
39pub struct Shell {
40 pub workspace: Arc<dyn Workspace>,
42 pub launcher: Launcher,
44 pub live: Option<Live>,
47 pub spill: Option<Spill>,
49 pub cut: Option<OutputLimit>,
53 pub confine: bool,
57}
58
59impl Default for Shell {
60 fn default() -> Self {
63 Self {
64 workspace: Arc::new(Local),
65 launcher: Launcher::default(),
66 live: None,
67 spill: None,
68 cut: Some(OutputLimit::default()),
69 confine: false,
70 }
71 }
72}
73
74#[derive(Clone, Debug, PartialEq, Eq, Deserialize, JsonSchema)]
76#[serde(deny_unknown_fields)]
77pub struct ShellArgs {
78 pub command: String,
80 pub timeout_secs: Option<u64>,
82 pub cwd: Option<String>,
85}
86
87impl ShellArgs {
88 pub fn new(command: impl Into<String>) -> Self {
90 Self {
91 command: command.into(),
92 timeout_secs: None,
93 cwd: None,
94 }
95 }
96
97 pub fn timeout_secs(mut self, secs: u64) -> Self {
99 self.timeout_secs = Some(secs);
100 self
101 }
102
103 pub fn in_dir(mut self, dir: impl Into<String>) -> Self {
105 self.cwd = Some(dir.into());
106 self
107 }
108}
109
110#[derive(Clone, Debug, PartialEq, Eq)]
112pub struct ShellRun {
113 pub output: String,
117 pub status: ExitStatus,
119}
120
121impl Shell {
122 pub async fn run(&self, args: ShellArgs) -> Result<ShellRun, ToolExecutionError> {
127 self.execute(args).await.map(|(ran, _)| ran)
128 }
129
130 async fn execute(
132 &self,
133 args: ShellArgs,
134 ) -> Result<(ShellRun, Option<OutputCut>), ToolExecutionError> {
135 let timeout = args
136 .timeout_secs
137 .map(|secs| Duration::from_secs(secs.max(1)));
138 let process = self.start(&args)?;
139 let _stop = KillOnDrop(process.group());
140 let shell = self.clone();
141 blocking(move || shell.finish(process, timeout)).await
142 }
143}
144
145impl Tool for Shell {
146 const NAME: &'static str = "shell";
147 type Args = ShellArgs;
148 type Output = String;
149 type Error = ToolExecutionError;
150
151 fn description(&self) -> String {
152 let kept = match self.cut {
153 Some(cut) => format!(
154 "; past {} lines or {} KB, its first and last lines and the lines with errors \
155 or failures from between",
156 cut.lines,
157 cut.bytes / 1024
158 ),
159 None => String::new(),
160 };
161 format!(
162 "Run a shell command with `sh -c` in the current directory, or in cwd, and wait \
163 until it ends, however long it takes; set timeout_secs only for a command that may \
164 hang. Returns the exit code and stdout and stderr interleaved line by line{kept}. Each call starts a fresh `sh` with an \
165 empty stdin. Do not run what never ends, such as a server, in the foreground, and \
166 do not assume a git repository."
167 )
168 }
169
170 fn parameters(&self) -> serde_json::Value {
171 args_schema::<ShellArgs>()
172 }
173
174 fn annotations(&self) -> ToolAnnotations {
175 ToolAnnotations::default().with_title("Shell")
176 }
177
178 async fn call(
179 &self,
180 context: &mut ToolContext,
181 args: ShellArgs,
182 ) -> Result<String, ToolExecutionError> {
183 let (ran, cut) = self.execute(args).await?;
184 context.insert_result(ran.status)?;
185 if let Some(cut) = cut {
186 context.insert_result(cut)?;
187 }
188 Ok(ran.output)
189 }
190}
191
192impl Shell {
193 fn start(&self, args: &ShellArgs) -> Result<Process, ToolExecutionError> {
197 let base = self.launcher.cwd.as_deref();
198 let dir = match &args.cwd {
199 Some(dir) => {
200 if self.confine {
201 self.workspace.place(base, true, dir)?;
202 }
203 Some(
204 self.workspace
205 .join(base.unwrap_or(Path::new("")), Path::new(dir)),
206 )
207 }
208 None => base.map(Path::to_path_buf),
209 };
210 let how = Spawn {
211 stdin: None,
212 merge_stderr: false,
213 };
214 let line = [OsString::from("-c"), OsString::from(&args.command)];
215 self.workspace
216 .spawn(&self.launcher, dir.as_deref(), OsStr::new("sh"), &line, how)
217 .map_err(|error| ToolExecutionError::other(format!("could not run sh: {error}")))
218 }
219
220 fn finish(
223 &self,
224 mut process: Process,
225 timeout: Option<Duration>,
226 ) -> Result<(ShellRun, Option<OutputCut>), ToolExecutionError> {
227 let mut kept = Kept {
228 spill: self.spill.clone(),
229 cut: self.cut,
230 live: self.live.clone(),
231 ..Kept::default()
232 };
233 let deadline = timeout.and_then(|timeout| Instant::now().checked_add(timeout));
234 let end = process
235 .run(deadline, |line| {
236 kept.push(&line);
237 kept.written <= OUTPUT_LIMIT
238 })
239 .map_err(|error| {
240 ToolExecutionError::other(format!("waiting for the command failed: {error}"))
241 })?;
242 let silent = kept.written == 0;
243 let (mut output, cut) = kept.text();
244 let status = match end {
245 Ended::Exited(status) => match status.code() {
246 Some(code) => {
247 output.push_str(&format!("{EXIT_PREFIX}{code}]"));
248 ExitStatus::Exited(code)
249 }
250 None => {
251 output.push_str(&killed(status, process.remote()));
252 ExitStatus::Signaled
253 }
254 },
255 Ended::TimedOut => {
256 let secs = timeout.unwrap_or_default().as_secs();
257 output.push_str(&format!(
258 "{TIMEOUT_PREFIX}{secs}s and stopped{}",
259 there(&process)
260 ));
261 if silent {
262 output.push_str(SILENT);
263 }
264 output.push(']');
265 ExitStatus::TimedOut
266 }
267 Ended::Stopped => {
268 output.push_str(&format!(
269 "{LIMIT_PREFIX}{} MB{}]",
270 OUTPUT_LIMIT / 1024 / 1024,
271 there(&process)
272 ));
273 ExitStatus::Killed
274 }
275 };
276 Ok((ShellRun { output, status }, cut))
277 }
278}
279
280const SILENT: &str = ". It printed nothing. If it should have, it may hold its output back when \
282 that is not a terminal (`python3 -u` or `stdbuf -oL` make it print as it \
283 goes); if it needs longer, give a larger `timeout_secs`";
284
285fn there(process: &Process) -> String {
289 match process.remote() {
290 Some(host) => format!(", on {host} too"),
291 None => String::new(),
292 }
293}
294
295const EXIT_PREFIX: &str = "[exit code ";
297const KILLED_PREFIX: &str = "[killed by ";
299const TIMEOUT_PREFIX: &str = "[timed out after ";
301const LIMIT_PREFIX: &str = "[stopped after printing more than ";
303
304const STATUS_STARTS: [&str; 4] = [EXIT_PREFIX, KILLED_PREFIX, TIMEOUT_PREFIX, LIMIT_PREFIX];
307
308fn killed(status: std::process::ExitStatus, remote: Option<&str>) -> String {
313 #[cfg(unix)]
314 if let Some(signal) = std::os::unix::process::ExitStatusExt::signal(&status) {
315 let (name, why) = sent(signal);
316 if let Some(host) = remote {
317 return format!(
318 "{KILLED_PREFIX}{name} (signal {signal}), which this tool did not send: {why}. It \
319 was the connection to {host}, and the command there was stopped with it]"
320 );
321 }
322 return format!(
323 "{KILLED_PREFIX}{name} (signal {signal}), which this tool did not send: {why}. It was \
324 the process this call started here, `sh` or the last program it ran; when that \
325 program was a client of another machine, such as a remote shell, the command \
326 there may still be running or may have finished, so check there before running \
327 it again]"
328 );
329 }
330 #[cfg(not(unix))]
331 let _ = (status, remote);
332 format!("{KILLED_PREFIX}a signal]")
333}
334
335#[cfg(unix)]
337pub(crate) fn sent(signal: i32) -> (&'static str, &'static str) {
338 match signal {
339 libc::SIGHUP => ("SIGHUP", "its terminal or session went away"),
340 libc::SIGINT => ("SIGINT", "something on this machine interrupted it"),
341 libc::SIGQUIT => ("SIGQUIT", "something on this machine asked it to quit"),
342 libc::SIGILL => ("SIGILL", "it crashed"),
343 libc::SIGTRAP => ("SIGTRAP", "it crashed, or a debugger stopped it"),
344 libc::SIGABRT => ("SIGABRT", "it aborted itself, as on a failed assertion"),
345 libc::SIGBUS => ("SIGBUS", "it crashed"),
346 libc::SIGFPE => ("SIGFPE", "it crashed on an arithmetic error"),
347 libc::SIGKILL => (
348 "SIGKILL",
349 "something on this machine killed it, such as the out-of-memory killer or a `kill`",
350 ),
351 libc::SIGSEGV => ("SIGSEGV", "it crashed"),
352 libc::SIGPIPE => ("SIGPIPE", "it wrote to a pipe that nothing read any more"),
353 libc::SIGTERM => ("SIGTERM", "something on this machine asked it to stop"),
354 _ => ("a signal", "something on this machine sent it"),
355 }
356}
357
358pub fn split_status(text: &str) -> (&str, Option<&str>) {
371 let start = text.rfind('\n').map_or(0, |newline| newline + 1);
372 match text.split_at_checked(start) {
373 Some((printed, last))
374 if last.ends_with(']') && STATUS_STARTS.iter().any(|at| last.starts_with(at)) =>
375 {
376 (printed, Some(last))
377 }
378 _ => (text, None),
379 }
380}
381
382const MAX_HELD: usize = 1024 * 1024;
385const CONTEXT: usize = 3;
388const MAX_PICKED: usize = 300;
390
391fn matters(line: &str) -> bool {
395 let start = line.trim_start();
396 start.starts_with("error")
397 || start.starts_with("Error")
398 || [
399 "FAILED",
400 "FAIL:",
401 "failed",
402 "panicked",
403 "Traceback",
404 "Error:",
405 ]
406 .iter()
407 .any(|word| line.contains(word))
408}
409
410fn plain(line: &str) -> Cow<'_, str> {
414 if !line.contains('\x1b') {
415 return Cow::Borrowed(line);
416 }
417 let mut text = String::with_capacity(line.len());
418 let mut chars = line.chars().peekable();
419 while let Some(c) = chars.next() {
420 if c != '\x1b' {
421 text.push(c);
422 continue;
423 }
424 match chars.next() {
425 Some('[') => {
427 let _ = chars.by_ref().find(|c| ('\x40'..='\x7e').contains(c));
428 }
429 Some(']' | 'P' | 'X' | '^' | '_') => {
431 while let Some(c) = chars.next() {
432 if c == '\x07' || (c == '\x1b' && chars.next_if_eq(&'\\').is_some()) {
433 break;
434 }
435 }
436 }
437 Some(' '..='/') => {
439 let _ = chars.by_ref().find(|c| !(' '..='/').contains(c));
440 }
441 _ => {}
442 }
443 }
444 Cow::Owned(text)
445}
446
447struct Line {
450 at: usize,
452 text: String,
454 matters: bool,
456 long: bool,
458}
459
460impl Line {
461 fn shorten(&mut self, chars: usize) {
463 let body = self.text.trim_end_matches(['\n', '\r']);
464 let kept = clip(body, chars);
465 if kept.len() < body.len() {
466 self.text = format!("{kept}{CUT_MARK}\n");
467 self.long = true;
468 }
469 }
470}
471
472#[derive(Default)]
480struct Kept {
481 written: u64,
483 cut: Option<OutputLimit>,
485 held: [String; 2],
488 lines: usize,
490 head: Vec<Line>,
492 head_bytes: usize,
494 picked: Vec<Line>,
496 picked_bytes: usize,
498 later: VecDeque<Line>,
501 later_bytes: usize,
503 missed: bool,
505 context: usize,
507 tail: VecDeque<Line>,
509 tail_bytes: usize,
511 cutting: bool,
513 spill: Option<Spill>,
515 spilled: Option<(PathBuf, File)>,
517 live: Option<Live>,
519 in_key: bool,
522}
523
524impl Kept {
525 fn push(&mut self, piece: &OutputLine) {
529 self.written += piece.text().len() as u64;
530 let held = match piece {
531 OutputLine::Out(_) => 0,
532 OutputLine::Err(_) => 1,
533 };
534 let Some(line) = self.held.get_mut(held) else {
535 return;
536 };
537 line.push_str(piece.text());
538 if line.ends_with('\n') || line.len() >= MAX_HELD {
539 let line = std::mem::take(line);
540 self.line(line);
541 }
542 }
543
544 fn flush(&mut self) {
547 for held in 0..self.held.len() {
548 if let Some(line) = self.held.get_mut(held).map(std::mem::take)
549 && !line.is_empty()
550 {
551 self.line(line);
552 }
553 }
554 }
555
556 fn line(&mut self, line: String) {
561 let line = plain(&line);
562 let mut line = self.redact(&line).into_owned();
563 if line.is_empty() {
564 return;
565 }
566 if !line.ends_with('\n') {
567 line.push('\n');
568 }
569 if let Some(live) = &self.live {
570 live.line(&line);
571 }
572 if let Some((_, file)) = &mut self.spilled
573 && file.write_all(line.as_bytes()).is_err()
574 {
575 self.spilled = None;
576 }
577 let mut line = Line {
578 at: self.lines,
579 matters: self.cut.is_some() && matters(&line),
580 text: line,
581 long: false,
582 };
583 if self.cutting {
584 line.shorten(MAX_LINE_CHARS);
585 }
586 self.tail_bytes += line.text.len();
587 self.tail.push_back(line);
588 self.lines += 1;
589 let Some(limit) = self.cut else {
590 return;
591 };
592 if !self.cutting {
593 if self.tail.len() <= limit.lines && self.tail_bytes <= limit.bytes {
594 return;
595 }
596 self.cutting = true;
597 self.spill();
598 for line in &mut self.tail {
599 line.shorten(MAX_LINE_CHARS);
600 }
601 self.tail_bytes = self.tail.iter().map(|line| line.text.len()).sum();
602 while self.head.len() < limit.lines / 2
603 && let Some(line) = self.tail.pop_front()
604 {
605 if self.head_bytes + line.text.len() > limit.bytes / 2 {
606 self.tail.push_front(line);
607 break;
608 }
609 self.head_bytes += line.text.len();
610 self.tail_bytes -= line.text.len();
611 self.head.push(line);
612 }
613 }
614 while self.tail.len()
616 > limit
617 .lines
618 .saturating_sub(self.head.len() + self.picked.len() + self.later.len())
619 || self.tail_bytes
620 > limit
621 .bytes
622 .saturating_sub(self.head_bytes + self.picked_bytes + self.later_bytes)
623 {
624 let Some(mut line) = self.tail.pop_front() else {
625 break;
626 };
627 self.tail_bytes -= line.text.len();
628 if line.matters {
629 self.context = CONTEXT + 1;
630 }
631 if self.context > 0 {
632 self.context -= 1;
633 line.shorten(MAX_PICKED);
634 self.pick(limit, line);
635 }
636 }
637 }
638
639 fn pick(&mut self, limit: OutputLimit, line: Line) {
643 let (lines, bytes) = (limit.lines / 8, limit.bytes / 8);
644 if self.later.is_empty()
645 && self.picked.len() < lines
646 && self.picked_bytes + line.text.len() <= bytes
647 {
648 self.picked_bytes += line.text.len();
649 self.picked.push(line);
650 return;
651 }
652 self.later_bytes += line.text.len();
653 self.later.push_back(line);
654 while self.later.len() > lines || self.later_bytes > bytes {
655 self.missed = true;
656 while let Some(line) = self.later.pop_front() {
657 self.later_bytes -= line.text.len();
658 if self.later.front().is_none_or(|line| line.matters) {
659 break;
660 }
661 }
662 }
663 }
664
665 fn redact<'a>(&mut self, piece: &'a str) -> Cow<'a, str> {
669 if self.in_key {
670 self.in_key = !piece.contains("-----END ");
671 return Cow::Borrowed("");
672 }
673 let mut redacted = redact::text(piece);
674 if let Cow::Owned(line) = &mut redacted {
675 self.in_key = redact::find(piece).any(|secret| {
676 secret.kind == redact::Kind::PrivateKey
677 && piece
678 .get(secret.range)
679 .is_some_and(|key| !key.contains("-----END "))
680 });
681 if piece.ends_with('\n') && !line.ends_with('\n') {
683 line.push('\n');
684 }
685 }
686 redacted
687 }
688
689 fn spill(&mut self) {
692 let Some(spill) = self.spill.take() else {
693 return;
694 };
695 self.spilled = spill.create("shell").ok().filter(|(_, file)| {
696 let mut file = file;
697 self.tail
698 .iter()
699 .try_for_each(|line| file.write_all(line.text.as_bytes()))
700 .is_ok()
701 });
702 }
703
704 fn text(mut self) -> (String, Option<OutputCut>) {
709 self.flush();
710 let kept_lines = self.head.len() + self.picked.len() + self.later.len() + self.tail.len();
711 let errors = match self.missed {
712 true => "the first and last lines with errors or failures",
713 false => "lines with errors or failures",
714 };
715 let cut = self.lines - kept_lines;
716 let kept: Vec<&Line> = (self.head.iter().chain(&self.picked))
717 .chain(&self.later)
718 .chain(&self.tail)
719 .collect();
720 let long: Vec<usize> = kept
721 .iter()
722 .filter(|line| line.long)
723 .map(|line| line.at)
724 .collect();
725 let mut text = String::new();
726 let mut fact = None;
727 if cut > 0 || !long.is_empty() {
728 let mut what = Vec::new();
729 if cut > 0 {
730 what.push(format!(
731 "{cut} of {} lines cut, but for {errors}",
732 self.lines
733 ));
734 }
735 if !long.is_empty() {
736 what.push(format!(
737 "{} long {} cut short",
738 long.len(),
739 if long.len() == 1 { "line" } else { "lines" }
740 ));
741 }
742 let what = what.join(", and ");
743 match &self.spilled {
744 Some((path, _)) => {
745 let mut next = 0;
747 let (first, gap) = kept
748 .iter()
749 .find_map(|line| {
750 let gap = (line.at > next).then(|| (next + 1, line.at - next));
751 next = line.at + 1;
752 gap
753 })
754 .unwrap_or((next + 1, cut));
755 let read = match long.first() {
756 Some(at) if cut == 0 => format!(
757 "with offset {} and limit 1 for the first long line, whose note says \
758 how to read on in it",
759 at + 1
760 ),
761 _ => format!(
762 "with offset {first} and limit {} for the first lines cut",
763 gap.min(crate::MAX_LINES)
764 ),
765 };
766 text.push_str(&format!(
767 "[{what}; all of the output is in {}: `read` it there {read}, rather \
768 than running the command again]\n",
769 path.display(),
770 ));
771 }
772 None => text.push_str(&format!("[{what}]\n")),
773 }
774 fact = Some(OutputCut {
775 kept: KeptEnd::Ends,
776 kept_lines,
777 total_lines: self.lines,
778 spill: self
779 .spilled
780 .as_ref()
781 .map(|(path, _)| path.to_string_lossy().into_owned()),
782 });
783 }
784 let mut next = 0;
785 for line in kept {
786 if line.at > next {
787 text.push_str(&format!(
788 "[… {} lines cut (lines {}-{}) …]\n",
789 line.at - next,
790 next + 1,
791 line.at
792 ));
793 }
794 text.push_str(&line.text);
795 next = line.at + 1;
796 }
797 (text, fact)
798 }
799}
800
801#[cfg(test)]
802mod tests;