use std::collections::{BTreeMap, HashMap, HashSet};
use std::env;
use std::fs::{self, File, OpenOptions};
use std::io::{Read, Seek, SeekFrom, Write};
use std::os::unix::fs::{MetadataExt, OpenOptionsExt, PermissionsExt};
use std::os::unix::process::CommandExt;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::mpsc::{Receiver, SyncSender, TrySendError, sync_channel};
use std::sync::{Arc, Mutex};
use std::thread;
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use serde::{Deserialize, Serialize};
const CACHE_VERSION: u32 = 3;
const CACHE_MAX_BYTES: u64 = 2 * 1024 * 1024;
const OUTPUT_MAX_BYTES: u64 = 64 * 1024;
const MAX_OPTION_COUNT: usize = 256;
const MAX_OPTION_SPELLING_BYTES: usize = 128;
const QUEUE_CAPACITY: usize = 64;
const WORKER_COUNT: usize = 2;
const SUCCESS_TTL: Duration = Duration::from_secs(30 * 24 * 60 * 60);
const MISS_TTL: Duration = Duration::from_secs(24 * 60 * 60);
const MAN_TIMEOUT: Duration = Duration::from_millis(1_500);
#[cfg(target_os = "macos")]
const HELP_TIMEOUT: Duration = Duration::from_millis(1_000);
static TEMP_SEQUENCE: AtomicU64 = AtomicU64::new(0);
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CommandEntry {
pub name: String,
pub description: String,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CommandMatch {
pub name: String,
pub description: String,
pub description_pending: bool,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OptionMatch {
pub spelling: String,
pub description: String,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct OptionMatches {
pub entries: Vec<OptionMatch>,
pub pending: bool,
}
#[derive(Debug)]
pub struct CommandCatalog {
entries: Vec<CommandEntry>,
jobs: HashMap<String, DescriptionJob>,
state: Arc<Mutex<EnrichmentState>>,
queue: Option<SyncSender<DescriptionJob>>,
}
#[derive(Debug, Default)]
struct EnrichmentState {
descriptions: HashMap<String, String>,
options: HashMap<String, Vec<OptionMatch>>,
settled: HashSet<String>,
pending: HashSet<String>,
}
#[derive(Debug, Clone)]
struct DescriptionJob {
name: String,
path: PathBuf,
fingerprint: ExecutableFingerprint,
authored_description: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
struct ExecutableFingerprint {
path: PathBuf,
size: u64,
device: u64,
inode: u64,
mode: u32,
modified_secs: i64,
modified_nanos: i64,
changed_secs: i64,
changed_nanos: i64,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
struct CachedDescription {
fingerprint: ExecutableFingerprint,
checked_at_secs: u64,
description: Option<String>,
options: Vec<OptionMatch>,
}
#[derive(Debug, Default)]
struct Enrichment {
description: Option<String>,
options: Vec<OptionMatch>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
struct DescriptionCache {
version: u32,
entries: BTreeMap<String, CachedDescription>,
}
impl Default for DescriptionCache {
fn default() -> Self {
Self {
version: CACHE_VERSION,
entries: BTreeMap::new(),
}
}
}
impl Default for CommandCatalog {
fn default() -> Self {
Self {
entries: Vec::new(),
jobs: HashMap::new(),
state: Arc::new(Mutex::new(EnrichmentState::default())),
queue: None,
}
}
}
impl CommandCatalog {
pub fn discover(cache_file: PathBuf) -> Self {
let cache = load_cache(&cache_file);
let now = now_secs();
let mut seen = HashSet::new();
let mut entries = Vec::new();
let mut jobs = HashMap::new();
let mut state = EnrichmentState::default();
for (name, description) in SHELL_BUILTINS {
seen.insert((*name).to_owned());
entries.push(CommandEntry {
name: (*name).to_owned(),
description: (*description).to_owned(),
});
}
if let Some(path) = env::var_os("PATH") {
for directory in env::split_paths(&path) {
let Ok(children) = fs::read_dir(&directory) else {
continue;
};
for child in children.flatten() {
let Some(name) = child.file_name().to_str().map(str::to_owned) else {
continue;
};
if !valid_name(&name) || jobs.contains_key(&name) {
continue;
}
let path = child.path();
let Ok(metadata) = fs::metadata(&path) else {
continue;
};
if !metadata.is_file() || metadata.permissions().mode() & 0o111 == 0 {
continue;
}
let authored = known_description(&name);
let authored_description = !seen.insert(name.clone()) || authored.is_some();
if !authored_description || authored.is_some() {
entries.push(CommandEntry {
description: authored
.map(str::to_owned)
.unwrap_or_else(|| fallback_description(&path)),
name: name.clone(),
});
}
let fingerprint = fingerprint(&path, &metadata);
let job = DescriptionJob {
name: name.clone(),
path,
fingerprint,
authored_description,
};
if let Some(cached) = cache.entries.get(&name)
&& cached.fingerprint == job.fingerprint
&& cache_is_fresh(cached, now)
{
state.settled.insert(name.clone());
if !authored_description && let Some(description) = &cached.description {
state.descriptions.insert(name.clone(), description.clone());
}
if !cached.options.is_empty() {
state.options.insert(name.clone(), cached.options.clone());
}
}
jobs.insert(name, job);
}
}
}
entries.sort_unstable_by(|left, right| left.name.cmp(&right.name));
let state = Arc::new(Mutex::new(state));
let queue = start_workers(Arc::clone(&state), cache_file, cache);
Self {
entries,
jobs,
state,
queue,
}
}
pub fn matching(&self, prefix: &str, limit: usize) -> Vec<CommandMatch> {
let mut state = self.state.lock().expect("command state lock poisoned");
self.entries
.iter()
.filter(|entry| entry.name.starts_with(prefix))
.take(limit)
.map(|entry| {
let job = self.jobs.get(&entry.name);
let description = if job.is_some_and(|job| job.authored_description) {
entry.description.clone()
} else {
state
.descriptions
.get(&entry.name)
.cloned()
.unwrap_or_else(|| entry.description.clone())
};
let metadata_pending = enqueue_if_unsettled(&mut state, job, self.queue.as_ref());
let description_pending =
metadata_pending && !job.is_some_and(|job| job.authored_description);
CommandMatch {
name: entry.name.clone(),
description,
description_pending,
}
})
.collect()
}
pub fn matching_options(&self, command: &str, prefix: &str, limit: usize) -> OptionMatches {
let Some(job) = self.jobs.get(command) else {
return OptionMatches {
entries: Vec::new(),
pending: false,
};
};
let mut state = self.state.lock().expect("command state lock poisoned");
let pending = enqueue_if_unsettled(&mut state, Some(job), self.queue.as_ref());
let entries = state
.options
.get(command)
.into_iter()
.flatten()
.filter(|option| option.spelling.starts_with(prefix))
.take(limit)
.cloned()
.collect();
OptionMatches { entries, pending }
}
pub fn inventory(&self) -> Vec<CommandMatch> {
let state = self.state.lock().expect("command state lock poisoned");
self.entries
.iter()
.map(|entry| CommandMatch {
name: entry.name.clone(),
description: if self
.jobs
.get(&entry.name)
.is_some_and(|job| job.authored_description)
{
entry.description.clone()
} else {
state
.descriptions
.get(&entry.name)
.cloned()
.unwrap_or_else(|| entry.description.clone())
},
description_pending: false,
})
.collect()
}
#[cfg(test)]
pub fn from_entries(entries: impl IntoIterator<Item = CommandEntry>) -> Self {
let mut entries: Vec<_> = entries.into_iter().collect();
entries.sort_unstable_by(|left, right| left.name.cmp(&right.name));
Self {
entries,
..Self::default()
}
}
#[cfg(test)]
pub fn from_options(command: &str, options: Vec<OptionMatch>) -> Self {
let entry = CommandEntry {
name: command.to_owned(),
description: "Test command".to_owned(),
};
let job = DescriptionJob {
name: command.to_owned(),
path: PathBuf::from(command),
fingerprint: ExecutableFingerprint {
path: PathBuf::from(command),
size: 0,
device: 0,
inode: 0,
mode: 0,
modified_secs: 0,
modified_nanos: 0,
changed_secs: 0,
changed_nanos: 0,
},
authored_description: true,
};
let mut state = EnrichmentState::default();
state.options.insert(command.to_owned(), options);
state.settled.insert(command.to_owned());
Self {
entries: vec![entry],
jobs: HashMap::from([(command.to_owned(), job)]),
state: Arc::new(Mutex::new(state)),
queue: None,
}
}
}
fn enqueue_if_unsettled(
state: &mut EnrichmentState,
job: Option<&DescriptionJob>,
queue: Option<&SyncSender<DescriptionJob>>,
) -> bool {
let (Some(job), Some(queue)) = (job, queue) else {
return false;
};
if state.settled.contains(&job.name) {
return false;
}
if state.pending.insert(job.name.clone()) {
match queue.try_send(job.clone()) {
Ok(()) => {}
Err(TrySendError::Full(_)) => {
state.pending.remove(&job.name);
}
Err(TrySendError::Disconnected(_)) => {
state.pending.remove(&job.name);
state.settled.insert(job.name.clone());
return false;
}
}
}
true
}
fn start_workers(
state: Arc<Mutex<EnrichmentState>>,
cache_file: PathBuf,
cache: DescriptionCache,
) -> Option<SyncSender<DescriptionJob>> {
let (sender, receiver) = sync_channel(QUEUE_CAPACITY);
let receiver = Arc::new(Mutex::new(receiver));
let cache = Arc::new(Mutex::new(cache));
let mut started = 0;
for index in 0..WORKER_COUNT {
let state = Arc::clone(&state);
let receiver = Arc::clone(&receiver);
let cache = Arc::clone(&cache);
let cache_file = cache_file.clone();
let worker = thread::Builder::new()
.name(format!("aster-description-{index}"))
.spawn(move || description_worker(&receiver, &state, &cache_file, &cache));
if worker.is_ok() {
started += 1;
}
}
(started > 0).then_some(sender)
}
fn description_worker(
receiver: &Mutex<Receiver<DescriptionJob>>,
state: &Mutex<EnrichmentState>,
cache_file: &Path,
cache: &Mutex<DescriptionCache>,
) {
loop {
let job = {
let receiver = receiver.lock().expect("description queue lock poisoned");
receiver.recv()
};
let Ok(job) = job else {
return;
};
let unchanged_before = fingerprint_matches(&job);
let enrichment = if unchanged_before {
discover_enrichment(&job, cache_file.parent().unwrap_or(Path::new("/tmp")))
} else {
Enrichment::default()
};
let cacheable = unchanged_before && fingerprint_matches(&job);
{
let mut state = state.lock().expect("command state lock poisoned");
state.pending.remove(&job.name);
state.settled.insert(job.name.clone());
if cacheable {
if !job.authored_description
&& let Some(description) = &enrichment.description
{
state
.descriptions
.insert(job.name.clone(), description.clone());
}
if !enrichment.options.is_empty() {
state
.options
.insert(job.name.clone(), enrichment.options.clone());
}
}
}
if cacheable {
let mut cache = cache.lock().expect("description cache lock poisoned");
cache.entries.insert(
job.name,
CachedDescription {
fingerprint: job.fingerprint,
checked_at_secs: now_secs(),
description: enrichment.description,
options: enrichment.options,
},
);
let _ = save_cache(cache_file, &cache);
}
}
}
fn discover_enrichment(job: &DescriptionJob, output_dir: &Path) -> Enrichment {
let mut enrichment = man_enrichment(&job.name, output_dir).unwrap_or_default();
if (enrichment.description.is_none() || enrichment.options.is_empty())
&& let Some(help) = help_enrichment(job, output_dir)
{
if enrichment.description.is_none() {
enrichment.description = help.description;
}
if enrichment.options.is_empty() {
enrichment.options = help.options;
}
}
enrichment
}
fn man_enrichment(name: &str, output_dir: &Path) -> Option<Enrichment> {
let man = Path::new("/usr/bin/man");
if !man.is_file() {
return None;
}
let mut command = Command::new(man);
command
.arg("--")
.arg(name)
.env_clear()
.env("HOME", "/nonexistent")
.env("LC_ALL", "C")
.env("MANPAGER", "cat")
.env("PAGER", "cat");
let output = run_bounded(command, output_dir, MAN_TIMEOUT)?;
Some(Enrichment {
description: parse_man_description(name, &output),
options: parse_options(&output),
})
}
#[cfg(target_os = "macos")]
fn help_enrichment(job: &DescriptionJob, output_dir: &Path) -> Option<Enrichment> {
let sandbox = Path::new("/usr/bin/sandbox-exec");
if !sandbox.is_file() {
return None;
}
let mut command = Command::new(sandbox);
command
.arg("-p")
.arg(
"(version 1) (deny default) (allow process-exec) (allow file-read*) \
(allow sysctl-read) (allow mach-lookup)",
)
.arg(&job.path)
.arg("--help")
.env_clear()
.env("HOME", "/nonexistent")
.env("LC_ALL", "C")
.env("NO_COLOR", "1")
.env("PAGER", "cat")
.env("MANPAGER", "cat")
.env("TERM", "dumb");
let output = run_bounded(command, output_dir, HELP_TIMEOUT)?;
Some(Enrichment {
description: parse_help_description(&job.name, &output),
options: parse_options(&output),
})
}
#[cfg(not(target_os = "macos"))]
fn help_enrichment(_job: &DescriptionJob, _output_dir: &Path) -> Option<Enrichment> {
None
}
fn run_bounded(mut command: Command, output_dir: &Path, timeout: Duration) -> Option<String> {
let (path, mut output) = temporary_output(output_dir)?;
let stdout = output.try_clone().ok()?;
let stderr = output.try_clone().ok()?;
command
.current_dir("/")
.stdin(Stdio::null())
.stdout(Stdio::from(stdout))
.stderr(Stdio::from(stderr))
.process_group(0);
unsafe {
command.pre_exec(|| {
let limit = libc::rlimit {
rlim_cur: OUTPUT_MAX_BYTES,
rlim_max: OUTPUT_MAX_BYTES,
};
if libc::setrlimit(libc::RLIMIT_FSIZE, &limit) == -1 {
return Err(std::io::Error::last_os_error());
}
Ok(())
});
}
let mut child = match command.spawn() {
Ok(child) => child,
Err(_) => {
let _ = fs::remove_file(path);
return None;
}
};
let _ = fs::remove_file(&path);
let deadline = Instant::now() + timeout;
loop {
match child.try_wait() {
Ok(Some(_)) => break,
Ok(None) if Instant::now() < deadline => thread::sleep(Duration::from_millis(10)),
Ok(None) | Err(_) => {
unsafe {
libc::kill(-(child.id() as i32), libc::SIGKILL);
}
let _ = child.kill();
let _ = child.wait();
break;
}
}
}
unsafe {
libc::kill(-(child.id() as i32), libc::SIGKILL);
}
let _ = child.wait();
output.seek(SeekFrom::Start(0)).ok()?;
let mut bytes = Vec::new();
output.take(OUTPUT_MAX_BYTES).read_to_end(&mut bytes).ok()?;
String::from_utf8(bytes).ok()
}
fn temporary_output(directory: &Path) -> Option<(PathBuf, File)> {
for _ in 0..10 {
let sequence = TEMP_SEQUENCE.fetch_add(1, Ordering::Relaxed);
let path = directory.join(format!(
".description-output-{}-{sequence}",
std::process::id()
));
let file = OpenOptions::new()
.create_new(true)
.read(true)
.write(true)
.mode(0o600)
.open(&path);
match file {
Ok(file) => return Some((path, file)),
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue,
Err(_) => return None,
}
}
None
}
fn parse_man_description(name: &str, output: &str) -> Option<String> {
let mut name_section = false;
for line in clean_lines(output) {
if line == "NAME" {
name_section = true;
continue;
}
if name_section {
name_section = false;
if let Some(description) = line.strip_prefix(name)
&& description.chars().next().is_some_and(char::is_whitespace)
&& let Some(description) =
sanitize_description(description.trim_start().trim_start_matches('-'))
{
return Some(description);
}
}
let Some((names, description)) = line.split_once(" - ") else {
continue;
};
let exact = names.split(',').any(|candidate| {
candidate
.trim()
.strip_prefix(name)
.is_some_and(|rest| rest.is_empty() || rest.starts_with('('))
});
if exact && let Some(description) = sanitize_description(description) {
return Some(description);
}
}
None
}
#[cfg(any(target_os = "macos", test))]
fn parse_help_description(name: &str, output: &str) -> Option<String> {
for line in clean_lines(output) {
let lower = line.to_ascii_lowercase();
let structural = [
"usage",
"options",
"commands",
"arguments",
"available commands",
"flags",
"examples",
]
.iter()
.any(|heading| lower == *heading || lower.starts_with(&format!("{heading}:")));
let command_usage =
lower.starts_with(&format!("{name} [")) || lower.starts_with(&format!("{name} <"));
let diagnostic = [
"error:",
"warning:",
"failed",
"couldn't",
"unrecognized option",
"unknown option",
]
.iter()
.any(|text| lower.contains(text));
if structural || command_usage || diagnostic || line.starts_with(['-', '[']) {
continue;
}
if let Some(description) = sanitize_description(&line)
&& description.split_whitespace().count() >= 2
{
return Some(description);
}
}
None
}
fn parse_options(output: &str) -> Vec<OptionMatch> {
let mut entries: Vec<OptionMatch> = Vec::new();
let mut in_options = false;
let mut section_indent = 0;
let mut continuation_indexes = Vec::new();
let mut declaration_indent = 0;
for raw_line in output.lines() {
let Some(line) = clean_line(raw_line) else {
continuation_indexes.clear();
continue;
};
let text = line.trim();
let indent = line.len() - line.trim_start_matches(' ').len();
if is_options_heading(text) {
if in_options {
break;
}
in_options = true;
section_indent = indent;
continuation_indexes.clear();
continue;
}
if !in_options {
continue;
}
if indent <= section_indent && is_section_heading(text) {
break;
}
if !option_line_has_unsafe_data(raw_line)
&& let Some(declaration) = parse_option_declaration(text)
{
continuation_indexes.clear();
declaration_indent = indent;
for spelling in declaration.spellings {
if let Some(index) = entries
.iter()
.position(|option| option.spelling == spelling)
{
if entries[index].description.is_empty()
&& let Some(description) = &declaration.description
{
entries[index].description = description.clone();
}
continuation_indexes.push(index);
} else if entries.len() < MAX_OPTION_COUNT {
entries.push(OptionMatch {
spelling,
description: declaration.description.clone().unwrap_or_default(),
});
continuation_indexes.push(entries.len() - 1);
}
}
continue;
}
if !continuation_indexes.is_empty()
&& indent > declaration_indent
&& !text.starts_with('-')
&& !is_section_heading(text)
&& let Some(continuation) = sanitize_description(text)
{
for &index in &continuation_indexes {
let combined = if entries[index].description.is_empty() {
continuation.clone()
} else {
format!("{} {continuation}", entries[index].description)
};
if let Some(description) = sanitize_description(&combined) {
entries[index].description = description;
}
}
} else {
continuation_indexes.clear();
}
}
entries
}
struct ParsedOptionDeclaration {
spellings: Vec<String>,
description: Option<String>,
}
fn parse_option_declaration(line: &str) -> Option<ParsedOptionDeclaration> {
if !line.starts_with('-') {
return None;
}
let bytes = line.as_bytes();
let mut position = 0;
let mut spellings = Vec::new();
loop {
let (spelling, end) = option_spelling_at(line, position)?;
spellings.push(spelling.to_owned());
position = end;
if bytes.get(position) == Some(&b'=') {
position += 1;
let argument_start = position;
while bytes
.get(position)
.is_some_and(|byte| !byte.is_ascii_whitespace() && *byte != b',')
{
position += 1;
}
if position == argument_start {
return None;
}
} else if bytes.get(position) == Some(&b'[') {
let argument_end = bytes[position..].iter().position(|byte| *byte == b']')? + position;
let argument = &line[position + 1..argument_end];
if !argument.strip_prefix('=').is_some_and(is_option_argument) {
return None;
}
position = argument_end + 1;
}
if bytes.get(position) == Some(&b',') {
position += 1;
skip_ascii_spaces(bytes, &mut position);
if bytes.get(position) == Some(&b'-') {
continue;
}
return None;
}
if position == bytes.len() {
break;
}
if !bytes[position].is_ascii_whitespace() {
return None;
}
skip_ascii_spaces(bytes, &mut position);
if position == bytes.len() {
break;
}
if bytes[position] == b'/' {
position += 1;
skip_ascii_spaces(bytes, &mut position);
continue;
}
if bytes[position] == b'-' {
continue;
}
let token_end = bytes[position..]
.iter()
.position(u8::is_ascii_whitespace)
.map_or(bytes.len(), |offset| position + offset);
let argument = line[position..token_end].trim_end_matches(',');
if is_option_argument(argument) {
position = token_end;
skip_ascii_spaces(bytes, &mut position);
if line[..token_end].ends_with(',') {
continue;
}
}
let description = (position < bytes.len())
.then(|| sanitize_description(&line[position..]))
.flatten();
return Some(ParsedOptionDeclaration {
spellings,
description,
});
}
Some(ParsedOptionDeclaration {
spellings,
description: None,
})
}
fn option_spelling_at(line: &str, position: usize) -> Option<(&str, usize)> {
let bytes = line.as_bytes();
if bytes.get(position) != Some(&b'-') {
return None;
}
let mut end = position + 1;
if bytes.get(end) == Some(&b'-') {
end += 1;
let name_start = end;
while bytes
.get(end)
.is_some_and(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_'))
{
end += 1;
}
if end == name_start {
return None;
}
} else {
if !bytes.get(end).is_some_and(u8::is_ascii_alphanumeric) {
return None;
}
end += 1;
}
let spelling = &line[position..end];
let delimiter = bytes.get(end);
if !valid_option_spelling(spelling)
|| delimiter
.is_some_and(|byte| !byte.is_ascii_whitespace() && !matches!(byte, b',' | b'=' | b'['))
{
return None;
}
Some((spelling, end))
}
fn valid_option_spelling(spelling: &str) -> bool {
if !spelling.is_ascii()
|| spelling.len() > MAX_OPTION_SPELLING_BYTES
|| !spelling.starts_with('-')
{
return false;
}
let bytes = spelling.as_bytes();
if bytes.get(1) == Some(&b'-') {
bytes.len() >= 3
&& bytes[2].is_ascii_alphanumeric()
&& bytes.last().is_some_and(u8::is_ascii_alphanumeric)
&& bytes[2..]
.iter()
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_'))
} else {
bytes.len() == 2 && bytes[1].is_ascii_alphanumeric()
}
}
fn option_line_has_unsafe_data(line: &str) -> bool {
let characters: Vec<_> = line.chars().collect();
characters.iter().enumerate().any(|(index, &character)| {
if is_directional_format(character) {
return true;
}
if character == '\u{8}' {
return index == 0
|| index + 1 == characters.len()
|| (characters[index - 1] != characters[index + 1]
&& characters[index - 1] != '_');
}
character.is_control() && character != '\t'
})
}
fn is_option_argument(token: &str) -> bool {
let token = token
.strip_prefix('<')
.and_then(|token| token.strip_suffix('>'))
.or_else(|| {
token
.strip_prefix('[')
.and_then(|token| token.strip_suffix(']'))
})
.unwrap_or(token)
.trim_end_matches("...");
!token.is_empty()
&& token.is_ascii()
&& token
.bytes()
.all(|byte| byte.is_ascii_uppercase() || byte.is_ascii_digit() || byte == b'_')
}
fn skip_ascii_spaces(bytes: &[u8], position: &mut usize) {
while bytes.get(*position).is_some_and(u8::is_ascii_whitespace) {
*position += 1;
}
}
fn is_options_heading(line: &str) -> bool {
matches!(
line.trim_end_matches(':').to_ascii_lowercase().as_str(),
"option"
| "options"
| "flags"
| "global options"
| "general options"
| "optional arguments"
| "the following options are available"
)
}
fn is_section_heading(line: &str) -> bool {
let heading = line.trim_end_matches(':');
let lower = heading.to_ascii_lowercase();
if matches!(
lower.as_str(),
"usage"
| "arguments"
| "commands"
| "available commands"
| "examples"
| "description"
| "synopsis"
| "operands"
| "environment"
| "exit status"
| "files"
| "authors"
| "bugs"
| "see also"
) {
return true;
}
heading
.chars()
.any(|character| character.is_ascii_alphabetic())
&& heading
.chars()
.all(|character| !character.is_ascii_alphabetic() || character.is_ascii_uppercase())
}
fn clean_lines(output: &str) -> impl Iterator<Item = String> + '_ {
output
.lines()
.filter_map(clean_line)
.map(|line| line.trim().to_owned())
}
fn clean_line(line: &str) -> Option<String> {
let mut clean = String::with_capacity(line.len());
let mut escape = 0;
for character in line.chars() {
if escape == 1 {
escape = match character {
'[' => 2,
']' => 3,
_ => 0,
};
continue;
}
if escape == 2 {
if ('@'..='~').contains(&character) {
escape = 0;
}
continue;
}
if escape == 3 {
continue;
}
if character == '\u{1b}' {
escape = 1;
} else if character == '\u{8}' {
clean.pop();
} else if character == '\t' {
clean.push(' ');
} else if !character.is_control() && !is_directional_format(character) {
clean.push(character);
}
}
let clean = clean.trim_end().to_owned();
(!clean.trim().is_empty()).then_some(clean)
}
fn sanitize_description(description: &str) -> Option<String> {
let mut clean = description.split_whitespace().collect::<Vec<_>>().join(" ");
clean.retain(|character| !character.is_control() && !is_directional_format(character));
if clean.is_empty() || !clean.chars().any(char::is_alphabetic) {
return None;
}
if clean.chars().count() > 200 {
clean = clean.chars().take(199).collect();
clean.push('…');
}
Some(clean)
}
fn is_directional_format(character: char) -> bool {
matches!(character, '\u{202a}'..='\u{202e}' | '\u{2066}'..='\u{2069}')
}
fn load_cache(path: &Path) -> DescriptionCache {
let Ok(metadata) = fs::metadata(path) else {
return DescriptionCache::default();
};
if metadata.len() > CACHE_MAX_BYTES {
return DescriptionCache::default();
}
let Ok(bytes) = fs::read(path) else {
return DescriptionCache::default();
};
let Ok(mut cache) = serde_json::from_slice::<DescriptionCache>(&bytes) else {
return DescriptionCache::default();
};
if cache.version != CACHE_VERSION {
return DescriptionCache::default();
}
for cached in cache.entries.values_mut() {
let mut seen = HashSet::new();
cached.options.retain_mut(|option| {
if !valid_option_spelling(&option.spelling) || !seen.insert(option.spelling.clone()) {
return false;
}
option.description = sanitize_description(&option.description).unwrap_or_default();
true
});
cached.options.truncate(MAX_OPTION_COUNT);
}
cache
}
fn save_cache(path: &Path, cache: &DescriptionCache) -> std::io::Result<()> {
let mut bounded = cache.clone();
let bytes = loop {
let bytes = serde_json::to_vec(&bounded)?;
if bytes.len() as u64 <= CACHE_MAX_BYTES || bounded.entries.is_empty() {
break bytes;
}
let Some(oldest) = bounded
.entries
.iter()
.min_by_key(|(_, entry)| entry.checked_at_secs)
.map(|(name, _)| name.clone())
else {
break bytes;
};
bounded.entries.remove(&oldest);
};
let sequence = TEMP_SEQUENCE.fetch_add(1, Ordering::Relaxed);
let temporary = path.with_extension(format!("tmp-{}-{sequence}", std::process::id()));
let result = (|| {
let mut file = OpenOptions::new()
.create_new(true)
.write(true)
.mode(0o600)
.open(&temporary)?;
file.write_all(&bytes)?;
file.write_all(b"\n")?;
file.sync_all()?;
fs::rename(&temporary, path)
})();
if result.is_err() {
let _ = fs::remove_file(temporary);
}
result
}
fn cache_is_fresh(cached: &CachedDescription, now: u64) -> bool {
let ttl = if !cached.options.is_empty() {
SUCCESS_TTL
} else {
MISS_TTL
};
now >= cached.checked_at_secs && now - cached.checked_at_secs <= ttl.as_secs()
}
fn fingerprint(path: &Path, metadata: &fs::Metadata) -> ExecutableFingerprint {
ExecutableFingerprint {
path: path.to_path_buf(),
size: metadata.len(),
device: metadata.dev(),
inode: metadata.ino(),
mode: metadata.mode(),
modified_secs: metadata.mtime(),
modified_nanos: metadata.mtime_nsec(),
changed_secs: metadata.ctime(),
changed_nanos: metadata.ctime_nsec(),
}
}
fn fingerprint_matches(job: &DescriptionJob) -> bool {
fs::metadata(&job.path)
.map(|metadata| fingerprint(&job.path, &metadata) == job.fingerprint)
.unwrap_or(false)
}
fn now_secs() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
}
fn valid_name(name: &str) -> bool {
!name.is_empty()
&& !name.chars().any(char::is_control)
&& name
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'+'))
}
fn fallback_description(path: &Path) -> String {
let path = path.to_string_lossy();
if path.contains("/.cargo/bin/") {
"Executable installed by Cargo".to_owned()
} else if path.contains("/homebrew/") || path.contains("/Cellar/") {
"Homebrew command".to_owned()
} else if path.contains("/.local/bin/") || path.contains("/bin/") && path.contains("/Users/") {
"User-installed command".to_owned()
} else {
"System command".to_owned()
}
}
fn known_description(name: &str) -> Option<&'static str> {
Some(match name {
"ansible" => "Define and run automation tasks",
"appwrite" => "Manage Appwrite projects and services",
"arch" => "Print architecture type or run a universal binary",
"asr" => "Apple Software Restore; copy volumes and disk images",
"atlas" => "CLI tool to manage MongoDB Atlas",
"aws" => "Official command line interface for Amazon Web Services",
"aws-vault" => "Securely store and access AWS credentials",
"bash" => "GNU Bourne Again shell",
"brew" => "The missing package manager for macOS",
"cargo" => "Rust package manager and build tool",
"cmake" => "Configure, build, and test software projects",
"code" => "Open Visual Studio Code",
"curl" => "Transfer data from or to a server",
"docker" => "Build and run applications in containers",
"fd" => "Fast and user-friendly file finder",
"fzf" => "Command-line fuzzy finder",
"gh" => "GitHub command line interface",
"git" => "Distributed version control system",
"go" => "Build and manage Go source code",
"iris" => "Interactive shell assistant",
"jq" => "Process and transform JSON",
"kubectl" => "Control Kubernetes clusters",
"make" => "Maintain and build groups of programs",
"node" => "Run JavaScript with Node.js",
"npm" => "JavaScript package manager",
"nvim" => "Edit text with Neovim",
"pnpm" => "Fast, disk-efficient JavaScript package manager",
"python" | "python3" => "Run the Python interpreter",
"rg" => "Recursively search files with ripgrep",
"rustc" => "Compile Rust source code",
"ssh" => "OpenSSH remote login client",
"tmux" => "Terminal multiplexer",
"yarn" => "JavaScript package manager",
"zsh" => "Z shell command interpreter",
_ => return None,
})
}
const SHELL_BUILTINS: &[(&str, &str)] = &[
("alias", "Define or display shell aliases"),
("autoload", "Mark shell functions for automatic loading"),
("bg", "Resume jobs in the background"),
("cd", "Change the current working directory"),
("command", "Execute a command without shell function lookup"),
("export", "Set environment variables for child processes"),
("fg", "Bring jobs into the foreground"),
("jobs", "Display active shell jobs"),
("setopt", "Enable Zsh options"),
(
"source",
"Execute commands from a file in the current shell",
),
("typeset", "Declare shell variables and attributes"),
("unalias", "Remove shell alias definitions"),
("unset", "Remove shell variables or functions"),
("unsetopt", "Disable Zsh options"),
];
#[cfg(test)]
mod tests {
use super::*;
use tempfile::tempdir;
#[test]
fn matches_sorted_prefixes() {
let catalog = CommandCatalog::from_entries([
CommandEntry {
name: "atlas".to_owned(),
description: "MongoDB Atlas".to_owned(),
},
CommandEntry {
name: "arch".to_owned(),
description: "Architecture".to_owned(),
},
]);
let names: Vec<_> = catalog
.matching("a", 10)
.into_iter()
.map(|entry| entry.name)
.collect();
assert_eq!(names, ["arch", "atlas"]);
}
#[test]
fn rejects_shell_metacharacters_in_names() {
assert!(valid_name("aws-vault"));
assert!(!valid_name("bad command"));
assert!(!valid_name("bad;command"));
}
#[test]
fn parses_exact_man_description() {
let output = "assetutil(1) - process asset catalog.car files\n\
other(1) - unrelated\n";
assert_eq!(
parse_man_description("assetutil", output).as_deref(),
Some("process asset catalog.car files")
);
assert_eq!(parse_man_description("asset", output), None);
}
#[test]
fn parses_overstruck_man_name_section() {
let output = "N\u{8}NA\u{8}AM\u{8}ME\u{8}E\n a\u{8}as\u{8}s - assembler\n";
assert_eq!(
parse_man_description("as", output).as_deref(),
Some("assembler")
);
}
#[test]
fn parses_man_name_section_without_separator() {
let output = "NAME\n assetutil process asset catalog files\n\nSYNOPSIS\n";
assert_eq!(
parse_man_description("assetutil", output).as_deref(),
Some("process asset catalog files")
);
}
#[test]
fn parses_prose_from_help_output() {
let output =
"Usage: tool [OPTIONS]\n\nInspect a project without changing it.\n\nOptions:\n";
assert_eq!(
parse_help_description("tool", output).as_deref(),
Some("Inspect a project without changing it.")
);
}
#[test]
fn rejects_help_diagnostics() {
let output = "tool: error: couldn't create cache file\nUsage: tool [OPTIONS]\n";
assert_eq!(parse_help_description("tool", output), None);
}
#[test]
fn parses_rendered_man_option_declarations_and_continuations() {
let output = "NAME\n tool - inspect things\nThe following options are available:\n\
\x20 -a, --all\n\
\x20 Include hidden entries.\n\
\x20 -o, --output FILE Write to FILE.\n\
\x20 --color=WHEN Control colored output.\nARGUMENTS\n\
\x20 FILE Input file.\n";
assert_eq!(
parse_options(output),
[
OptionMatch {
spelling: "-a".to_owned(),
description: "Include hidden entries.".to_owned(),
},
OptionMatch {
spelling: "--all".to_owned(),
description: "Include hidden entries.".to_owned(),
},
OptionMatch {
spelling: "-o".to_owned(),
description: "Write to FILE.".to_owned(),
},
OptionMatch {
spelling: "--output".to_owned(),
description: "Write to FILE.".to_owned(),
},
OptionMatch {
spelling: "--color".to_owned(),
description: "Control colored output.".to_owned(),
},
]
);
}
#[test]
fn parses_common_help_option_sections() {
let clap = "Usage: tool [OPTIONS]\n\nOptions:\n -q, --quiet Suppress output\n\
\x20 --format <FORMAT> Select a format\n";
let cobra = "Flags:\n -h, --help help for tool\nCommands:\n child\n";
let click = "Options:\n --color / --no-color Toggle color.\n --help Show this message.\n";
let argparse = "optional arguments:\n -v, --verbose increase verbosity\n -o OUTPUT, --output OUTPUT destination\n --color[=WHEN] color mode\n";
assert_eq!(
parse_options(clap)
.into_iter()
.map(|option| option.spelling)
.collect::<Vec<_>>(),
["-q", "--quiet", "--format"]
);
assert_eq!(
parse_options(cobra),
[
OptionMatch {
spelling: "-h".to_owned(),
description: "help for tool".to_owned(),
},
OptionMatch {
spelling: "--help".to_owned(),
description: "help for tool".to_owned(),
},
]
);
assert_eq!(parse_options(click).len(), 3);
assert_eq!(
parse_options(argparse)
.into_iter()
.map(|option| option.spelling)
.collect::<Vec<_>>(),
["-v", "--verbose", "-o", "--output", "--color"]
);
}
#[test]
fn rejects_usage_prose_subcommands_and_malicious_option_spellings() {
let output = "Usage: tool --usage-only\n\
\x20Prose mentions --prose-only but is not an option.\n\
\x20Options:\n\
\x20 --safe Safe option.\n\
\x20 --bad;touch Not safe.\n\
\x20 --also$(evil) Not safe.\n\
\x20 --con\u{7}trol Control data.\n\
\x20 -abc Combined spelling.\n\
\x20 —lookalike Unicode dash.\n\
\x20Commands:\n\
\x20 child\n\
\x20Options:\n\
\x20 --child-only Child option.\n";
assert_eq!(
parse_options(output),
[OptionMatch {
spelling: "--safe".to_owned(),
description: "Safe option.".to_owned(),
}]
);
assert!(!option_line_has_unsafe_data("-\u{8}-, h\u{8}h"));
}
#[test]
fn strips_terminal_controls_from_descriptions() {
let output = "tool(1) - \u{1b}[31mred\u{1b}[0m\u{202e} text\n";
assert_eq!(
parse_man_description("tool", output).as_deref(),
Some("red text")
);
}
#[test]
fn description_cache_round_trips() {
let directory = tempdir().unwrap();
let path = directory.path().join("descriptions.json");
let mut cache = DescriptionCache::default();
cache.entries.insert(
"tool".to_owned(),
CachedDescription {
fingerprint: ExecutableFingerprint {
path: PathBuf::from("/usr/bin/tool"),
size: 42,
device: 1,
inode: 2,
mode: 0o100755,
modified_secs: 3,
modified_nanos: 4,
changed_secs: 5,
changed_nanos: 6,
},
checked_at_secs: 7,
description: Some("Inspect a tool".to_owned()),
options: vec![OptionMatch {
spelling: "--verbose".to_owned(),
description: "Show more detail".to_owned(),
}],
},
);
save_cache(&path, &cache).unwrap();
let loaded = load_cache(&path);
assert_eq!(
loaded.entries["tool"].description.as_deref(),
Some("Inspect a tool")
);
assert_eq!(loaded.entries["tool"].fingerprint.size, 42);
assert_eq!(
loaded.entries["tool"].options,
[OptionMatch {
spelling: "--verbose".to_owned(),
description: "Show more detail".to_owned(),
}]
);
}
#[test]
fn authored_descriptions_remain_preferred_during_enrichment() {
let job = test_job("cargo", true);
let mut state = EnrichmentState::default();
state
.descriptions
.insert("cargo".to_owned(), "Parsed description".to_owned());
let catalog = CommandCatalog {
entries: vec![CommandEntry {
name: "cargo".to_owned(),
description: "Rust package manager and build tool".to_owned(),
}],
jobs: HashMap::from([("cargo".to_owned(), job)]),
state: Arc::new(Mutex::new(state)),
queue: None,
};
assert_eq!(
catalog.matching("cargo", 1)[0].description,
"Rust package manager and build tool"
);
}
#[test]
fn matches_cached_options_by_prefix_for_an_exact_command() {
let mut state = EnrichmentState::default();
state.settled.insert("tool".to_owned());
state.options.insert(
"tool".to_owned(),
vec![
OptionMatch {
spelling: "--all".to_owned(),
description: "Include all".to_owned(),
},
OptionMatch {
spelling: "--color".to_owned(),
description: "Control color".to_owned(),
},
OptionMatch {
spelling: "-v".to_owned(),
description: "Verbose".to_owned(),
},
],
);
let catalog = CommandCatalog {
jobs: HashMap::from([("tool".to_owned(), test_job("tool", false))]),
state: Arc::new(Mutex::new(state)),
..CommandCatalog::default()
};
let matches = catalog.matching_options("tool", "--", 1);
assert_eq!(matches.entries[0].spelling, "--all");
assert!(!matches.pending);
assert!(
catalog
.matching_options("unknown", "-", 10)
.entries
.is_empty()
);
assert!(!catalog.matching_options("unknown", "-", 10).pending);
}
#[test]
fn unsettled_options_report_pending_without_blocking_on_a_full_queue() {
let (sender, _receiver) = sync_channel(1);
sender.try_send(test_job("queued", false)).unwrap();
let catalog = CommandCatalog {
jobs: HashMap::from([("tool".to_owned(), test_job("tool", false))]),
queue: Some(sender),
..CommandCatalog::default()
};
let started = Instant::now();
let matches = catalog.matching_options("tool", "--", 10);
assert!(matches.entries.is_empty());
assert!(matches.pending);
assert!(started.elapsed() < Duration::from_millis(100));
}
#[test]
fn description_process_has_a_hard_timeout() {
let directory = tempdir().unwrap();
let mut command = Command::new("/bin/sleep");
command.arg("2");
let started = Instant::now();
assert_eq!(
run_bounded(command, directory.path(), Duration::from_millis(30)).as_deref(),
Some("")
);
assert!(started.elapsed() < Duration::from_secs(1));
}
fn test_job(name: &str, authored_description: bool) -> DescriptionJob {
DescriptionJob {
name: name.to_owned(),
path: PathBuf::from(format!("/usr/bin/{name}")),
fingerprint: ExecutableFingerprint {
path: PathBuf::from(format!("/usr/bin/{name}")),
size: 1,
device: 1,
inode: 1,
mode: 0o100755,
modified_secs: 1,
modified_nanos: 1,
changed_secs: 1,
changed_nanos: 1,
},
authored_description,
}
}
}