mod alias;
mod archive;
pub(crate) mod arg_parser;
mod assert;
mod awk;
mod base64;
mod bc;
mod caller;
mod cat;
mod checksum;
mod clear;
mod column;
mod comm;
mod compgen;
mod csv;
mod curl;
mod cuttr;
mod date;
mod diff;
mod dirstack;
mod disk;
mod dotenv;
mod echo;
mod environ;
mod envsubst;
mod expand;
mod export;
mod expr;
mod fc;
mod fileops;
mod flow;
mod fold;
mod generated;
mod glob_cmd;
mod grep;
mod headtail;
mod help;
mod hextools;
mod http;
mod iconv;
mod inspect;
mod introspect;
mod join;
#[cfg(feature = "jq")]
mod jq;
mod json;
mod log;
mod ls;
mod mapfile;
mod mkfifo;
mod navigation;
mod nl;
mod numfmt;
mod parallel;
mod paste;
mod patch;
mod path;
mod pipeline;
mod printf;
mod read;
mod retry;
mod rg;
pub(crate) mod search_common;
mod sed;
mod semver;
mod seq;
mod shuf;
mod sleep;
mod sortuniq;
mod source;
mod split;
mod strings;
mod system;
mod template;
mod test;
mod textrev;
pub(crate) mod timeout;
mod tomlq;
mod trap;
mod tree;
mod truncate;
mod vars;
mod verify;
mod wait;
mod wc;
mod yaml;
mod yes;
mod zip_cmd;
pub(crate) const MAX_FORMAT_WIDTH: usize = 10_000;
pub(crate) mod git;
pub(crate) mod ssh;
#[cfg(feature = "python")]
mod python;
#[cfg(feature = "typescript")]
mod typescript;
#[cfg(feature = "sqlite")]
mod sqlite;
pub use alias::{Alias, Unalias};
pub use archive::{Gunzip, Gzip, Tar};
pub use assert::Assert;
pub use awk::Awk;
pub use base64::Base64;
pub use bc::Bc;
pub use caller::Caller;
pub use cat::Cat;
pub use checksum::{Md5sum, Sha1sum, Sha256sum};
pub use clear::Clear;
pub use column::Column;
pub use comm::Comm;
pub use compgen::Compgen;
pub use csv::Csv;
pub use curl::{Curl, Wget};
pub use cuttr::{Cut, Tr};
pub use date::Date;
pub use diff::Diff;
pub use dirstack::{Dirs, Popd, Pushd};
pub use disk::{Df, Du};
pub use dotenv::Dotenv;
pub use echo::Echo;
pub use environ::{Env, History, Printenv};
pub use envsubst::Envsubst;
pub use expand::{Expand, Unexpand};
pub use export::Export;
pub use expr::Expr;
pub use fc::Fc;
pub use fileops::{Chmod, Chown, Cp, Kill, Ln, Mkdir, Mktemp, Mv, Rm, Touch};
pub use flow::{Break, Colon, Continue, Exit, False, Return, True};
pub use fold::Fold;
pub use glob_cmd::GlobCmd;
pub use grep::Grep;
pub use headtail::{Head, Tail};
pub use help::Help;
pub use hextools::{Hexdump, Od, Xxd};
pub use http::Http;
pub use iconv::Iconv;
pub use inspect::{File, Less, Stat};
pub use introspect::{Hash, Type, Which};
pub use join::Join;
#[cfg(feature = "jq")]
pub use jq::Jq;
pub use json::Json;
pub use log::Log;
pub(crate) use ls::glob_match;
pub use ls::{Find, Ls, Rmdir};
pub use mapfile::Mapfile;
pub use mkfifo::Mkfifo;
pub use navigation::{Cd, Pwd};
pub use nl::Nl;
pub use numfmt::Numfmt;
pub use parallel::Parallel;
pub use paste::Paste;
pub use patch::Patch;
pub use path::{Basename, Dirname, Readlink, Realpath};
pub use pipeline::{Tee, Watch, Xargs};
pub use printf::Printf;
pub use read::Read;
pub use retry::Retry;
pub use rg::Rg;
pub use sed::Sed;
pub use semver::Semver;
pub use seq::Seq;
pub use shuf::Shuf;
pub use sleep::Sleep;
pub use sortuniq::{Sort, Uniq};
pub use source::Source;
pub use split::Split;
pub use strings::Strings;
pub use system::{DEFAULT_HOSTNAME, DEFAULT_USERNAME, Hostname, Id, Uname, Whoami};
pub use template::Template;
pub use test::{Bracket, Test};
pub use textrev::{Rev, Tac};
pub use timeout::Timeout;
pub use tomlq::Tomlq;
pub use trap::Trap;
pub use tree::Tree;
pub use truncate::Truncate;
pub use vars::{Eval, Local, Readonly, Set, Shift, Shopt, Times, Unset};
pub use verify::Verify;
pub use wait::Wait;
pub use wc::Wc;
pub use yaml::Yaml;
pub use yes::Yes;
pub use zip_cmd::{Unzip, Zip};
#[cfg(feature = "git")]
pub use git::Git;
#[cfg(feature = "ssh")]
pub use ssh::{Scp, Sftp, Ssh};
#[cfg(feature = "python")]
pub(crate) use python::PythonInprocessOptIn;
#[cfg(feature = "python")]
pub use python::{Python, PythonExternalFnHandler, PythonExternalFns, PythonLimits};
#[cfg(feature = "typescript")]
pub use typescript::{
TypeScriptConfig, TypeScriptExtension, TypeScriptExternalFnHandler, TypeScriptExternalFns,
TypeScriptLimits,
};
#[cfg(feature = "sqlite")]
pub(crate) use sqlite::SqliteInprocessOptIn;
#[cfg(feature = "sqlite")]
pub use sqlite::{Sqlite, SqliteBackend, SqliteLimits};
use async_trait::async_trait;
use clap::{CommandFactory, FromArgMatches};
use std::any::{Any, TypeId};
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use crate::error::Result;
use crate::fs::FileSystem;
use crate::interpreter::ExecResult;
pub(crate) async fn read_text_file(
fs: &dyn FileSystem,
path: &Path,
cmd_name: &str,
) -> std::result::Result<String, ExecResult> {
let content = fs
.read_file(path)
.await
.map_err(|e| ExecResult::err(format!("{cmd_name}: {}: {e}\n", path.display()), 1))?;
if path == Path::new("/dev/urandom") || path == Path::new("/dev/random") {
return Ok(content.iter().map(|&b| b as char).collect());
}
Ok(String::from_utf8_lossy(&content).into_owned())
}
pub(crate) fn check_help_version(
args: &[String],
help_text: &str,
version: Option<&str>,
) -> Option<ExecResult> {
for arg in args {
match arg.as_str() {
"--help" => return Some(ExecResult::ok(help_text.to_string())),
"--version" => {
if let Some(ver) = version {
return Some(ExecResult::ok(format!("{ver}\n")));
}
}
s if !s.starts_with('-') => break,
_ => {}
}
}
None
}
pub(crate) use crate::interpreter::ShellRef;
pub use crate::interpreter::BuiltinSideEffect;
pub trait Extension: Send + Sync {
fn builtins(&self) -> Vec<(String, Box<dyn Builtin>)>;
}
#[derive(Default)]
pub struct ExecutionExtensions {
values: HashMap<TypeId, Box<dyn Any + Send + Sync>>,
}
impl ExecutionExtensions {
pub fn new() -> Self {
Self::default()
}
pub fn insert<T>(&mut self, value: T) -> Option<T>
where
T: Send + Sync + 'static,
{
self.values
.insert(TypeId::of::<T>(), Box::new(value))
.and_then(|prev| prev.downcast::<T>().ok().map(|prev| *prev))
}
pub fn with<T>(mut self, value: T) -> Self
where
T: Send + Sync + 'static,
{
let _ = self.insert(value);
self
}
pub fn get<T>(&self) -> Option<&T>
where
T: Send + Sync + 'static,
{
self.values
.get(&TypeId::of::<T>())
.and_then(|value| value.downcast_ref::<T>())
}
pub fn is_empty(&self) -> bool {
self.values.is_empty()
}
}
#[derive(Debug, Clone)]
pub struct SubCommand {
pub name: String,
pub args: Vec<String>,
pub stdin: Option<String>,
}
#[derive(Debug)]
pub enum ExecutionPlan {
Timeout {
duration: std::time::Duration,
preserve_status: bool,
command: SubCommand,
},
Batch {
commands: Vec<SubCommand>,
},
BatchWithStatus {
commands: Vec<SubCommand>,
stderr_prefix: String,
force_error_exit: bool,
},
}
pub fn resolve_path(cwd: &Path, path_str: &str) -> PathBuf {
let path = Path::new(path_str);
let joined = if path.is_absolute() {
path.to_path_buf()
} else {
cwd.join(path)
};
normalize_path(&joined)
}
use crate::fs::normalize_path;
pub struct Context<'a> {
pub args: &'a [String],
pub env: &'a HashMap<String, String>,
#[allow(dead_code)] pub variables: &'a mut HashMap<String, String>,
pub cwd: &'a mut PathBuf,
pub fs: Arc<dyn FileSystem>,
pub stdin: Option<&'a str>,
#[cfg(feature = "http_client")]
pub http_client: Option<&'a crate::network::HttpClient>,
#[cfg(feature = "git")]
pub git_client: Option<&'a crate::builtins::git::GitClient>,
#[cfg(feature = "ssh")]
pub ssh_client: Option<&'a crate::builtins::ssh::SshClient>,
pub(crate) shell: Option<ShellRef<'a>>,
}
impl<'a> Context<'a> {
pub fn execution_extension<T>(&self) -> Option<&T>
where
T: Send + Sync + 'static,
{
self.shell
.as_ref()
.and_then(|shell| shell.execution_extensions.get::<T>())
}
#[cfg(test)]
pub fn new_for_test(
args: &'a [String],
env: &'a std::collections::HashMap<String, String>,
variables: &'a mut std::collections::HashMap<String, String>,
cwd: &'a mut std::path::PathBuf,
fs: std::sync::Arc<dyn crate::fs::FileSystem>,
stdin: Option<&'a str>,
) -> Self {
Self {
args,
env,
variables,
cwd,
fs,
stdin,
#[cfg(feature = "http_client")]
http_client: None,
#[cfg(feature = "git")]
git_client: None,
#[cfg(feature = "ssh")]
ssh_client: None,
shell: None,
}
}
}
#[async_trait]
pub trait Builtin: Send + Sync {
async fn execute(&self, ctx: Context<'_>) -> Result<ExecResult>;
async fn execution_plan(&self, _ctx: &Context<'_>) -> Result<Option<ExecutionPlan>> {
Ok(None)
}
fn llm_hint(&self) -> Option<&'static str> {
None
}
}
#[async_trait]
pub trait ClapBuiltin: Send + Sync {
type Args: clap::Parser + Send + 'static;
async fn execute_clap(&self, args: Self::Args, ctx: &mut BashkitContext<'_>) -> Result<()>;
fn llm_hint(&self) -> Option<&'static str> {
None
}
}
pub struct BashkitContext<'a> {
inner: Context<'a>,
pub stdout: String,
pub stderr: String,
pub exit_code: i32,
}
impl<'a> BashkitContext<'a> {
fn new(inner: Context<'a>) -> Self {
Self {
inner,
stdout: String::new(),
stderr: String::new(),
exit_code: 0,
}
}
fn into_exec_result(self) -> ExecResult {
ExecResult {
stdout: self.stdout,
stderr: self.stderr,
exit_code: self.exit_code,
..Default::default()
}
}
pub fn raw_args(&self) -> &[String] {
self.inner.args
}
pub fn env(&self) -> &HashMap<String, String> {
self.inner.env
}
pub fn variables(&mut self) -> &mut HashMap<String, String> {
self.inner.variables
}
pub fn cwd(&self) -> &Path {
self.inner.cwd
}
pub fn cwd_mut(&mut self) -> &mut PathBuf {
self.inner.cwd
}
pub fn fs(&self) -> Arc<dyn FileSystem> {
Arc::clone(&self.inner.fs)
}
pub fn stdin(&self) -> Option<&str> {
self.inner.stdin
}
pub fn execution_extension<T>(&self) -> Option<&T>
where
T: Send + Sync + 'static,
{
self.inner.execution_extension::<T>()
}
pub fn write_stdout(&mut self, output: impl AsRef<str>) {
self.stdout.push_str(output.as_ref());
}
pub fn write_stderr(&mut self, output: impl AsRef<str>) {
self.stderr.push_str(output.as_ref());
}
pub fn set_exit_code(&mut self, exit_code: i32) {
self.exit_code = exit_code;
}
pub fn fail(&mut self, stderr: impl AsRef<str>, exit_code: i32) {
self.write_stderr(stderr);
self.set_exit_code(exit_code);
}
}
#[async_trait]
impl<T> Builtin for T
where
T: ClapBuiltin,
{
async fn execute(&self, ctx: Context<'_>) -> Result<ExecResult> {
let mut command = <T::Args as CommandFactory>::command().color(clap::ColorChoice::Never);
let command_name = command.get_name().to_string();
let argv = std::iter::once(command_name).chain(ctx.args.iter().cloned());
let mut matches = match command.try_get_matches_from_mut(argv) {
Ok(matches) => matches,
Err(error) => return Ok(clap_error_to_exec_result(error)),
};
let args = match <T::Args as FromArgMatches>::from_arg_matches_mut(&mut matches) {
Ok(args) => args,
Err(error) => return Ok(clap_error_to_exec_result(error)),
};
let mut ctx = BashkitContext::new(ctx);
self.execute_clap(args, &mut ctx).await?;
Ok(ctx.into_exec_result())
}
fn llm_hint(&self) -> Option<&'static str> {
ClapBuiltin::llm_hint(self)
}
}
fn clap_error_to_exec_result(error: clap::Error) -> ExecResult {
let text = error.to_string();
if matches!(
error.kind(),
clap::error::ErrorKind::DisplayHelp | clap::error::ErrorKind::DisplayVersion
) {
return ExecResult::ok(text);
}
ExecResult::err(cap_diagnostic(text, 1024), error.exit_code())
}
fn cap_diagnostic(mut text: String, max_bytes: usize) -> String {
if text.len() <= max_bytes {
return text;
}
let cut = text
.char_indices()
.map(|(idx, _)| idx)
.take_while(|idx| *idx <= max_bytes)
.last()
.unwrap_or(0);
text.truncate(cut);
text
}
#[async_trait]
impl Builtin for std::sync::Arc<dyn Builtin> {
async fn execute(&self, ctx: Context<'_>) -> Result<ExecResult> {
(**self).execute(ctx).await
}
async fn execution_plan(&self, ctx: &Context<'_>) -> Result<Option<ExecutionPlan>> {
(**self).execution_plan(ctx).await
}
fn llm_hint(&self) -> Option<&'static str> {
(**self).llm_hint()
}
}
#[cfg(test)]
pub(crate) use crate::testing as debug_leak_check;
#[cfg(test)]
mod tests {
use super::*;
use crate::fs::{FileSystem, InMemoryFs};
#[test]
fn test_resolve_path_absolute() {
let cwd = PathBuf::from("/home/user");
let result = resolve_path(&cwd, "/tmp/file.txt");
assert_eq!(result, PathBuf::from("/tmp/file.txt"));
}
#[test]
fn test_resolve_path_relative() {
let cwd = PathBuf::from("/home/user");
let result = resolve_path(&cwd, "downloads/file.txt");
assert_eq!(result, PathBuf::from("/home/user/downloads/file.txt"));
}
#[test]
fn test_resolve_path_dot_from_root() {
let cwd = PathBuf::from("/");
let result = resolve_path(&cwd, ".");
assert_eq!(result, PathBuf::from("/"));
}
#[test]
fn test_resolve_path_dot_from_normal_dir() {
let cwd = PathBuf::from("/home/user");
let result = resolve_path(&cwd, ".");
assert_eq!(result, PathBuf::from("/home/user"));
}
#[test]
fn test_resolve_path_dotdot() {
let cwd = PathBuf::from("/home/user");
let result = resolve_path(&cwd, "..");
assert_eq!(result, PathBuf::from("/home"));
}
#[test]
fn test_resolve_path_dotdot_from_root() {
let cwd = PathBuf::from("/");
let result = resolve_path(&cwd, "..");
assert_eq!(result, PathBuf::from("/"));
}
#[test]
fn test_resolve_path_complex() {
let cwd = PathBuf::from("/home/user");
let result = resolve_path(&cwd, "./downloads/../documents/./file.txt");
assert_eq!(result, PathBuf::from("/home/user/documents/file.txt"));
}
#[tokio::test]
async fn read_text_file_returns_lossy_utf8() {
let fs = InMemoryFs::new();
fs.write_file(Path::new("/tmp/data.bin"), b"hi\xffthere")
.await
.unwrap();
let text = read_text_file(&fs, Path::new("/tmp/data.bin"), "cat")
.await
.unwrap();
assert_eq!(text, "hi\u{fffd}there");
}
#[tokio::test]
async fn read_text_file_formats_missing_file_errors() {
let fs = InMemoryFs::new();
let err = read_text_file(&fs, Path::new("/tmp/missing.txt"), "cat")
.await
.unwrap_err();
assert_eq!(err.exit_code, 1);
assert!(err.stderr.contains("cat: /tmp/missing.txt:"));
}
#[test]
fn check_help_version_returns_help() {
let args = vec!["--help".to_string()];
let r = check_help_version(&args, "usage text\n", Some("v1.0"));
assert!(r.is_some());
assert_eq!(r.unwrap().stdout, "usage text\n");
}
#[test]
fn check_help_version_returns_version() {
let args = vec!["--version".to_string()];
let r = check_help_version(&args, "usage\n", Some("tool 1.0"));
assert!(r.is_some());
assert_eq!(r.unwrap().stdout, "tool 1.0\n");
}
#[test]
fn check_help_version_no_version_configured() {
let args = vec!["--version".to_string()];
let r = check_help_version(&args, "usage\n", None);
assert!(r.is_none());
}
#[test]
fn check_help_version_stops_at_non_flag() {
let args = vec!["file.txt".to_string(), "--help".to_string()];
let r = check_help_version(&args, "usage\n", None);
assert!(r.is_none());
}
#[test]
fn check_help_version_no_match() {
let args = vec!["-c".to_string(), "filter".to_string()];
let r = check_help_version(&args, "usage\n", Some("v1"));
assert!(r.is_none());
}
#[test]
fn no_debug_fmt_in_builtin_source() {
let pat = regex::Regex::new(r"\{[A-Za-z0-9_]*:#?\?\}").unwrap();
let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src/builtins");
let mut violations = Vec::new();
fn walk(dir: &std::path::Path, violations: &mut Vec<String>, pat: ®ex::Regex) {
for entry in std::fs::read_dir(dir).expect("read builtins dir") {
let entry = entry.unwrap();
let path = entry.path();
if path.is_dir() {
walk(&path, violations, pat);
continue;
}
if path.extension().is_none_or(|e| e != "rs") {
continue;
}
let src = std::fs::read_to_string(&path).expect("read source");
for (i, line) in src.lines().enumerate() {
if line.contains("// debug-ok:") {
continue;
}
if line.trim_start().starts_with("#[derive(") {
continue;
}
if pat.is_match(line) {
let rel = path
.strip_prefix(std::path::Path::new(env!("CARGO_MANIFEST_DIR")))
.unwrap_or(&path);
violations.push(format!(
"{}:{}: {}",
rel.display(),
i + 1,
line.trim_end()
));
}
}
}
}
walk(&dir, &mut violations, &pat);
assert!(
violations.is_empty(),
"Rust Debug formatting found in builtin source. This leaks \
internal struct shapes into stderr where LLM agents see them. \
Use Display ({{}}) or a domain-specific formatter. Add \
`// debug-ok: <reason>` to the line for legitimate test \
asserts.\n\nViolations:\n{}",
violations.join("\n")
);
}
#[test]
fn generated_args_headers_match_pinned_uutils_revision() {
let pin = generated::UUTILS_REVISION;
let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src/builtins/generated");
let mut mismatches = Vec::new();
for entry in std::fs::read_dir(&dir).expect("read generated dir") {
let entry = entry.unwrap();
let path = entry.path();
if path.extension().and_then(|s| s.to_str()) != Some("rs") {
continue;
}
let name = path.file_name().and_then(|s| s.to_str()).unwrap_or("");
if !name.ends_with("_args.rs") {
continue;
}
let body = std::fs::read_to_string(&path).expect("read generated file");
let header_rev = body
.lines()
.find_map(|l| {
l.strip_prefix("// Source: uutils/coreutils@")
.and_then(|rest| rest.split_whitespace().next())
})
.unwrap_or("");
if header_rev != pin {
mismatches.push(format!(
"{}: header references uutils@{header_rev}, pin is uutils@{pin}",
path.display()
));
}
}
assert!(
mismatches.is_empty(),
"Generated argument files drift from `generated::UUTILS_REVISION` \
(`{pin}`). Regenerate every util at the pinned rev or bump the \
pin to match. The drift workflow does both atomically; manual \
bumps must too.\n\n{}",
mismatches.join("\n")
);
}
#[tokio::test]
async fn every_builtin_handles_bogus_flag_cleanly() {
const TOOLS: &[&str] = &[
"cat",
"ls",
"wc",
"head",
"tail",
"sort",
"uniq",
"cut",
"tr",
"grep",
"sed",
"awk",
"find",
"tree",
"diff",
"comm",
"paste",
"column",
"join",
"split",
"fold",
"expand",
"unexpand",
"nl",
"tac",
"truncate",
"shuf",
"rev",
"strings",
"od",
"xxd",
"hexdump",
"base64",
"md5sum",
"sha1sum",
"sha256sum",
"tar",
"gzip",
"gunzip",
"zip",
"unzip",
"seq",
"expr",
"bc",
"numfmt",
"test",
"printf",
"echo",
"env",
"printenv",
"stat",
"file",
"basename",
"dirname",
"realpath",
"csv",
"json",
"yaml",
"tomlq",
"jq",
"semver",
"envsubst",
"template",
"patch",
];
for tool in TOOLS {
let r =
super::debug_leak_check::run(&format!("{tool} --xyzzy-not-a-real-flag </dev/null"))
.await;
super::debug_leak_check::assert_no_leak(&r, &format!("{tool}_bogus_flag"), &[]);
}
}
}