use std::borrow::Cow;
use std::collections::VecDeque;
use std::ffi::{OsStr, OsString};
use std::fs::File;
use std::io::Write as _;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::{Duration, Instant};
use rig_core::completion::ToolAnnotations;
use rig_core::redact;
use rig_core::tool::{
ExitStatus, KeptEnd, OutputCut, Tool, ToolContext, ToolExecutionError, args_schema,
};
use schemars::JsonSchema;
use serde::Deserialize;
use crate::process::{Ended, KillOnDrop, Launcher, OutputLine, Process, Spawn};
use crate::workspace::{Local, Workspace};
use crate::{CUT_MARK, Live, MAX_LINE_CHARS, OutputLimit, Spill, blocking, clip};
const OUTPUT_LIMIT: u64 = 10 * 1024 * 1024;
#[derive(Clone, Debug)]
pub struct Shell {
pub workspace: Arc<dyn Workspace>,
pub launcher: Launcher,
pub live: Option<Live>,
pub spill: Option<Spill>,
pub cut: Option<OutputLimit>,
pub confine: bool,
}
impl Default for Shell {
fn default() -> Self {
Self {
workspace: Arc::new(Local),
launcher: Launcher::default(),
live: None,
spill: None,
cut: Some(OutputLimit::default()),
confine: false,
}
}
}
#[derive(Clone, Debug, PartialEq, Eq, Deserialize, JsonSchema)]
#[serde(deny_unknown_fields)]
pub struct ShellArgs {
pub command: String,
pub timeout_secs: Option<u64>,
pub cwd: Option<String>,
}
impl ShellArgs {
pub fn new(command: impl Into<String>) -> Self {
Self {
command: command.into(),
timeout_secs: None,
cwd: None,
}
}
pub fn timeout_secs(mut self, secs: u64) -> Self {
self.timeout_secs = Some(secs);
self
}
pub fn in_dir(mut self, dir: impl Into<String>) -> Self {
self.cwd = Some(dir.into());
self
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ShellRun {
pub output: String,
pub status: ExitStatus,
}
impl Shell {
pub async fn run(&self, args: ShellArgs) -> Result<ShellRun, ToolExecutionError> {
self.execute(args).await.map(|(ran, _)| ran)
}
async fn execute(
&self,
args: ShellArgs,
) -> Result<(ShellRun, Option<OutputCut>), ToolExecutionError> {
let timeout = args
.timeout_secs
.map(|secs| Duration::from_secs(secs.max(1)));
let process = self.start(&args)?;
let _stop = KillOnDrop(process.group());
let shell = self.clone();
blocking(move || shell.finish(process, timeout)).await
}
}
impl Tool for Shell {
const NAME: &'static str = "shell";
type Args = ShellArgs;
type Output = String;
type Error = ToolExecutionError;
fn description(&self) -> String {
let kept = match self.cut {
Some(cut) => format!(
"; past {} lines or {} KB, its first and last lines and the lines with errors \
or failures from between",
cut.lines,
cut.bytes / 1024
),
None => String::new(),
};
format!(
"Run a shell command with `sh -c` in the current directory, or in cwd, and wait \
until it ends, however long it takes; set timeout_secs only for a command that may \
hang. Returns the exit code and stdout and stderr interleaved line by line{kept}. Each call starts a fresh `sh` with an \
empty stdin. Do not run what never ends, such as a server, in the foreground, and \
do not assume a git repository."
)
}
fn parameters(&self) -> serde_json::Value {
args_schema::<ShellArgs>()
}
fn annotations(&self) -> ToolAnnotations {
ToolAnnotations::default().with_title("Shell")
}
async fn call(
&self,
context: &mut ToolContext,
args: ShellArgs,
) -> Result<String, ToolExecutionError> {
let (ran, cut) = self.execute(args).await?;
context.insert_result(ran.status)?;
if let Some(cut) = cut {
context.insert_result(cut)?;
}
Ok(ran.output)
}
}
impl Shell {
fn start(&self, args: &ShellArgs) -> Result<Process, ToolExecutionError> {
let base = self.launcher.cwd.as_deref();
let dir = match &args.cwd {
Some(dir) => {
if self.confine {
self.workspace.place(base, true, dir)?;
}
Some(
self.workspace
.join(base.unwrap_or(Path::new("")), Path::new(dir)),
)
}
None => base.map(Path::to_path_buf),
};
let how = Spawn {
stdin: None,
merge_stderr: false,
};
let line = [OsString::from("-c"), OsString::from(&args.command)];
self.workspace
.spawn(&self.launcher, dir.as_deref(), OsStr::new("sh"), &line, how)
.map_err(|error| ToolExecutionError::other(format!("could not run sh: {error}")))
}
fn finish(
&self,
mut process: Process,
timeout: Option<Duration>,
) -> Result<(ShellRun, Option<OutputCut>), ToolExecutionError> {
let mut kept = Kept {
spill: self.spill.clone(),
cut: self.cut,
live: self.live.clone(),
..Kept::default()
};
let deadline = timeout.and_then(|timeout| Instant::now().checked_add(timeout));
let end = process
.run(deadline, |line| {
kept.push(&line);
kept.written <= OUTPUT_LIMIT
})
.map_err(|error| {
ToolExecutionError::other(format!("waiting for the command failed: {error}"))
})?;
let silent = kept.written == 0;
let (mut output, cut) = kept.text();
let status = match end {
Ended::Exited(status) => match status.code() {
Some(code) => {
output.push_str(&format!("{EXIT_PREFIX}{code}]"));
ExitStatus::Exited(code)
}
None => {
output.push_str(&killed(status, process.remote()));
ExitStatus::Signaled
}
},
Ended::TimedOut => {
let secs = timeout.unwrap_or_default().as_secs();
output.push_str(&format!(
"{TIMEOUT_PREFIX}{secs}s and stopped{}",
there(&process)
));
if silent {
output.push_str(SILENT);
}
output.push(']');
ExitStatus::TimedOut
}
Ended::Stopped => {
output.push_str(&format!(
"{LIMIT_PREFIX}{} MB{}]",
OUTPUT_LIMIT / 1024 / 1024,
there(&process)
));
ExitStatus::Killed
}
};
Ok((ShellRun { output, status }, cut))
}
}
const SILENT: &str = ". It printed nothing. If it should have, it may hold its output back when \
that is not a terminal (`python3 -u` or `stdbuf -oL` make it print as it \
goes); if it needs longer, give a larger `timeout_secs`";
fn there(process: &Process) -> String {
match process.remote() {
Some(host) => format!(", on {host} too"),
None => String::new(),
}
}
const EXIT_PREFIX: &str = "[exit code ";
const KILLED_PREFIX: &str = "[killed by ";
const TIMEOUT_PREFIX: &str = "[timed out after ";
const LIMIT_PREFIX: &str = "[stopped after printing more than ";
const STATUS_STARTS: [&str; 4] = [EXIT_PREFIX, KILLED_PREFIX, TIMEOUT_PREFIX, LIMIT_PREFIX];
fn killed(status: std::process::ExitStatus, remote: Option<&str>) -> String {
#[cfg(unix)]
if let Some(signal) = std::os::unix::process::ExitStatusExt::signal(&status) {
let (name, why) = sent(signal);
if let Some(host) = remote {
return format!(
"{KILLED_PREFIX}{name} (signal {signal}), which this tool did not send: {why}. It \
was the connection to {host}, and the command there was stopped with it]"
);
}
return format!(
"{KILLED_PREFIX}{name} (signal {signal}), which this tool did not send: {why}. It was \
the process this call started here, `sh` or the last program it ran; when that \
program was a client of another machine, such as a remote shell, the command \
there may still be running or may have finished, so check there before running \
it again]"
);
}
#[cfg(not(unix))]
let _ = (status, remote);
format!("{KILLED_PREFIX}a signal]")
}
#[cfg(unix)]
pub(crate) fn sent(signal: i32) -> (&'static str, &'static str) {
match signal {
libc::SIGHUP => ("SIGHUP", "its terminal or session went away"),
libc::SIGINT => ("SIGINT", "something on this machine interrupted it"),
libc::SIGQUIT => ("SIGQUIT", "something on this machine asked it to quit"),
libc::SIGILL => ("SIGILL", "it crashed"),
libc::SIGTRAP => ("SIGTRAP", "it crashed, or a debugger stopped it"),
libc::SIGABRT => ("SIGABRT", "it aborted itself, as on a failed assertion"),
libc::SIGBUS => ("SIGBUS", "it crashed"),
libc::SIGFPE => ("SIGFPE", "it crashed on an arithmetic error"),
libc::SIGKILL => (
"SIGKILL",
"something on this machine killed it, such as the out-of-memory killer or a `kill`",
),
libc::SIGSEGV => ("SIGSEGV", "it crashed"),
libc::SIGPIPE => ("SIGPIPE", "it wrote to a pipe that nothing read any more"),
libc::SIGTERM => ("SIGTERM", "something on this machine asked it to stop"),
_ => ("a signal", "something on this machine sent it"),
}
}
pub fn split_status(text: &str) -> (&str, Option<&str>) {
let start = text.rfind('\n').map_or(0, |newline| newline + 1);
match text.split_at_checked(start) {
Some((printed, last))
if last.ends_with(']') && STATUS_STARTS.iter().any(|at| last.starts_with(at)) =>
{
(printed, Some(last))
}
_ => (text, None),
}
}
const MAX_HELD: usize = 1024 * 1024;
const CONTEXT: usize = 3;
const MAX_PICKED: usize = 300;
fn matters(line: &str) -> bool {
let start = line.trim_start();
start.starts_with("error")
|| start.starts_with("Error")
|| [
"FAILED",
"FAIL:",
"failed",
"panicked",
"Traceback",
"Error:",
]
.iter()
.any(|word| line.contains(word))
}
fn plain(line: &str) -> Cow<'_, str> {
if !line.contains('\x1b') {
return Cow::Borrowed(line);
}
let mut text = String::with_capacity(line.len());
let mut chars = line.chars().peekable();
while let Some(c) = chars.next() {
if c != '\x1b' {
text.push(c);
continue;
}
match chars.next() {
Some('[') => {
let _ = chars.by_ref().find(|c| ('\x40'..='\x7e').contains(c));
}
Some(']' | 'P' | 'X' | '^' | '_') => {
while let Some(c) = chars.next() {
if c == '\x07' || (c == '\x1b' && chars.next_if_eq(&'\\').is_some()) {
break;
}
}
}
Some(' '..='/') => {
let _ = chars.by_ref().find(|c| !(' '..='/').contains(c));
}
_ => {}
}
}
Cow::Owned(text)
}
struct Line {
at: usize,
text: String,
matters: bool,
long: bool,
}
impl Line {
fn shorten(&mut self, chars: usize) {
let body = self.text.trim_end_matches(['\n', '\r']);
let kept = clip(body, chars);
if kept.len() < body.len() {
self.text = format!("{kept}{CUT_MARK}\n");
self.long = true;
}
}
}
#[derive(Default)]
struct Kept {
written: u64,
cut: Option<OutputLimit>,
held: [String; 2],
lines: usize,
head: Vec<Line>,
head_bytes: usize,
picked: Vec<Line>,
picked_bytes: usize,
later: VecDeque<Line>,
later_bytes: usize,
missed: bool,
context: usize,
tail: VecDeque<Line>,
tail_bytes: usize,
cutting: bool,
spill: Option<Spill>,
spilled: Option<(PathBuf, File)>,
live: Option<Live>,
in_key: bool,
}
impl Kept {
fn push(&mut self, piece: &OutputLine) {
self.written += piece.text().len() as u64;
let held = match piece {
OutputLine::Out(_) => 0,
OutputLine::Err(_) => 1,
};
let Some(line) = self.held.get_mut(held) else {
return;
};
line.push_str(piece.text());
if line.ends_with('\n') || line.len() >= MAX_HELD {
let line = std::mem::take(line);
self.line(line);
}
}
fn flush(&mut self) {
for held in 0..self.held.len() {
if let Some(line) = self.held.get_mut(held).map(std::mem::take)
&& !line.is_empty()
{
self.line(line);
}
}
}
fn line(&mut self, line: String) {
let line = plain(&line);
let mut line = self.redact(&line).into_owned();
if line.is_empty() {
return;
}
if !line.ends_with('\n') {
line.push('\n');
}
if let Some(live) = &self.live {
live.line(&line);
}
if let Some((_, file)) = &mut self.spilled
&& file.write_all(line.as_bytes()).is_err()
{
self.spilled = None;
}
let mut line = Line {
at: self.lines,
matters: self.cut.is_some() && matters(&line),
text: line,
long: false,
};
if self.cutting {
line.shorten(MAX_LINE_CHARS);
}
self.tail_bytes += line.text.len();
self.tail.push_back(line);
self.lines += 1;
let Some(limit) = self.cut else {
return;
};
if !self.cutting {
if self.tail.len() <= limit.lines && self.tail_bytes <= limit.bytes {
return;
}
self.cutting = true;
self.spill();
for line in &mut self.tail {
line.shorten(MAX_LINE_CHARS);
}
self.tail_bytes = self.tail.iter().map(|line| line.text.len()).sum();
while self.head.len() < limit.lines / 2
&& let Some(line) = self.tail.pop_front()
{
if self.head_bytes + line.text.len() > limit.bytes / 2 {
self.tail.push_front(line);
break;
}
self.head_bytes += line.text.len();
self.tail_bytes -= line.text.len();
self.head.push(line);
}
}
while self.tail.len()
> limit
.lines
.saturating_sub(self.head.len() + self.picked.len() + self.later.len())
|| self.tail_bytes
> limit
.bytes
.saturating_sub(self.head_bytes + self.picked_bytes + self.later_bytes)
{
let Some(mut line) = self.tail.pop_front() else {
break;
};
self.tail_bytes -= line.text.len();
if line.matters {
self.context = CONTEXT + 1;
}
if self.context > 0 {
self.context -= 1;
line.shorten(MAX_PICKED);
self.pick(limit, line);
}
}
}
fn pick(&mut self, limit: OutputLimit, line: Line) {
let (lines, bytes) = (limit.lines / 8, limit.bytes / 8);
if self.later.is_empty()
&& self.picked.len() < lines
&& self.picked_bytes + line.text.len() <= bytes
{
self.picked_bytes += line.text.len();
self.picked.push(line);
return;
}
self.later_bytes += line.text.len();
self.later.push_back(line);
while self.later.len() > lines || self.later_bytes > bytes {
self.missed = true;
while let Some(line) = self.later.pop_front() {
self.later_bytes -= line.text.len();
if self.later.front().is_none_or(|line| line.matters) {
break;
}
}
}
}
fn redact<'a>(&mut self, piece: &'a str) -> Cow<'a, str> {
if self.in_key {
self.in_key = !piece.contains("-----END ");
return Cow::Borrowed("");
}
let mut redacted = redact::text(piece);
if let Cow::Owned(line) = &mut redacted {
self.in_key = redact::find(piece).any(|secret| {
secret.kind == redact::Kind::PrivateKey
&& piece
.get(secret.range)
.is_some_and(|key| !key.contains("-----END "))
});
if piece.ends_with('\n') && !line.ends_with('\n') {
line.push('\n');
}
}
redacted
}
fn spill(&mut self) {
let Some(spill) = self.spill.take() else {
return;
};
self.spilled = spill.create("shell").ok().filter(|(_, file)| {
let mut file = file;
self.tail
.iter()
.try_for_each(|line| file.write_all(line.text.as_bytes()))
.is_ok()
});
}
fn text(mut self) -> (String, Option<OutputCut>) {
self.flush();
let kept_lines = self.head.len() + self.picked.len() + self.later.len() + self.tail.len();
let errors = match self.missed {
true => "the first and last lines with errors or failures",
false => "lines with errors or failures",
};
let cut = self.lines - kept_lines;
let kept: Vec<&Line> = (self.head.iter().chain(&self.picked))
.chain(&self.later)
.chain(&self.tail)
.collect();
let long: Vec<usize> = kept
.iter()
.filter(|line| line.long)
.map(|line| line.at)
.collect();
let mut text = String::new();
let mut fact = None;
if cut > 0 || !long.is_empty() {
let mut what = Vec::new();
if cut > 0 {
what.push(format!(
"{cut} of {} lines cut, but for {errors}",
self.lines
));
}
if !long.is_empty() {
what.push(format!(
"{} long {} cut short",
long.len(),
if long.len() == 1 { "line" } else { "lines" }
));
}
let what = what.join(", and ");
match &self.spilled {
Some((path, _)) => {
let mut next = 0;
let (first, gap) = kept
.iter()
.find_map(|line| {
let gap = (line.at > next).then(|| (next + 1, line.at - next));
next = line.at + 1;
gap
})
.unwrap_or((next + 1, cut));
let read = match long.first() {
Some(at) if cut == 0 => format!(
"with offset {} and limit 1 for the first long line, whose note says \
how to read on in it",
at + 1
),
_ => format!(
"with offset {first} and limit {} for the first lines cut",
gap.min(crate::MAX_LINES)
),
};
text.push_str(&format!(
"[{what}; all of the output is in {}: `read` it there {read}, rather \
than running the command again]\n",
path.display(),
));
}
None => text.push_str(&format!("[{what}]\n")),
}
fact = Some(OutputCut {
kept: KeptEnd::Ends,
kept_lines,
total_lines: self.lines,
spill: self
.spilled
.as_ref()
.map(|(path, _)| path.to_string_lossy().into_owned()),
});
}
let mut next = 0;
for line in kept {
if line.at > next {
text.push_str(&format!(
"[… {} lines cut (lines {}-{}) …]\n",
line.at - next,
next + 1,
line.at
));
}
text.push_str(&line.text);
next = line.at + 1;
}
(text, fact)
}
}
#[cfg(test)]
mod tests;