#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Word {
pub text: String,
pub glob: bool,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Redirect {
Out { target: String, append: bool },
Err { target: String, append: bool },
ErrToOut,
In(String),
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct Stage {
pub assignments: Vec<(String, String)>,
pub words: Vec<Word>,
pub redirects: Vec<Redirect>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct Pipeline {
pub stages: Vec<Stage>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Joiner {
Always,
IfOk,
IfFailed,
}
#[derive(Debug, Clone, PartialEq, Eq)]
enum Token {
Word(Word),
Pipe,
And,
Or,
Semi,
Out(bool),
Err(bool),
ErrToOut,
In,
}
pub fn is_name(s: &str) -> bool {
let mut chars = s.chars();
matches!(chars.next(), Some(c) if c.is_ascii_alphabetic() || c == '_')
&& chars.all(|c| c.is_ascii_alphanumeric() || c == '_')
}
pub fn split_list(line: &str) -> Vec<(Joiner, String)> {
let chars: Vec<char> = line.chars().collect();
let mut parts = Vec::new();
let mut joiner = Joiner::Always;
let mut current = String::new();
let mut i = 0;
let mut quote: Option<char> = None;
let mut word_start = true;
while i < chars.len() {
let c = chars[i];
if let Some(q) = quote {
current.push(c);
if c == '\\' && q == '"' {
if let Some(&next) = chars.get(i + 1) {
current.push(next);
i += 1;
}
} else if c == q {
quote = None;
}
i += 1;
continue;
}
let next = chars.get(i + 1).copied();
let op = match (c, next) {
(';', _) => Some((Joiner::Always, 1)),
('&', Some('&')) => Some((Joiner::IfOk, 2)),
('|', Some('|')) => Some((Joiner::IfFailed, 2)),
_ => None,
};
if let Some((next_joiner, width)) = op {
parts.push((joiner, std::mem::take(&mut current)));
joiner = next_joiner;
i += width;
word_start = true;
continue;
}
match c {
'#' if word_start => break,
'\'' | '"' => quote = Some(c),
'\\' => {
current.push(c);
if let Some(next) = next {
current.push(next);
}
i += 2;
word_start = false;
continue;
}
_ => {}
}
word_start = c.is_whitespace() || c == '|';
current.push(c);
i += 1;
}
parts.push((joiner, current));
parts
.into_iter()
.filter(|(j, text)| !(text.trim().is_empty() && *j == Joiner::Always))
.collect()
}
pub fn parse(
line: &str,
lookup: &dyn Fn(&str) -> Option<String>,
) -> Result<Vec<(Joiner, Pipeline)>, String> {
let tokens = tokenize(line, lookup)?;
let mut list: Vec<(Joiner, Pipeline)> = Vec::new();
let mut joiner = Joiner::Always;
let mut pipeline = Pipeline::default();
let mut stage = Stage::default();
let mut iter = tokens.into_iter().peekable();
fn finish_stage(stage: &mut Stage, pipeline: &mut Pipeline, op: &str) -> Result<(), String> {
if stage.words.is_empty() && stage.assignments.is_empty() {
return Err(format!("syntax error near '{op}': a command is missing"));
}
pipeline.stages.push(std::mem::take(stage));
Ok(())
}
while let Some(token) = iter.next() {
match token {
Token::Word(word) => {
if stage.words.is_empty() && !word.glob {
if let Some((name, value)) = word.text.split_once('=') {
if is_name(name) {
stage.assignments.push((name.to_string(), value.to_string()));
continue;
}
}
}
stage.words.push(word);
}
Token::Out(append) | Token::Err(append) => {
let is_err = matches!(token, Token::Err(_));
let Some(Token::Word(target)) = iter.next() else {
return Err("syntax error: a file name is missing after '>'".to_string());
};
stage.redirects.push(if is_err {
Redirect::Err { target: target.text, append }
} else {
Redirect::Out { target: target.text, append }
});
}
Token::ErrToOut => stage.redirects.push(Redirect::ErrToOut),
Token::In => {
let Some(Token::Word(target)) = iter.next() else {
return Err("syntax error: a file name is missing after '<'".to_string());
};
stage.redirects.push(Redirect::In(target.text));
}
Token::Pipe => finish_stage(&mut stage, &mut pipeline, "|")?,
Token::And | Token::Or | Token::Semi => {
let op = match token {
Token::And => "&&",
Token::Or => "||",
_ => ";",
};
if stage.words.is_empty() && stage.assignments.is_empty() {
if token != Token::Semi || !pipeline.stages.is_empty() {
return Err(format!("syntax error near '{op}': a command is missing"));
}
} else {
finish_stage(&mut stage, &mut pipeline, op)?;
}
if !pipeline.stages.is_empty() {
list.push((joiner, std::mem::take(&mut pipeline)));
} else if token != Token::Semi {
return Err(format!("syntax error near '{op}': a command is missing"));
}
joiner = match token {
Token::And => Joiner::IfOk,
Token::Or => Joiner::IfFailed,
_ => Joiner::Always,
};
}
}
}
if !stage.words.is_empty() || !stage.assignments.is_empty() || !stage.redirects.is_empty() {
finish_stage(&mut stage, &mut pipeline, "end of line")?;
} else if !pipeline.stages.is_empty() {
return Err("syntax error: a command is missing after '|'".to_string());
}
if !pipeline.stages.is_empty() {
list.push((joiner, pipeline));
} else if joiner != Joiner::Always {
return Err("syntax error: a command is missing at the end of the line".to_string());
}
Ok(list)
}
pub const STATUS_MARK: &str = "\u{E000}?\u{E001}";
pub fn render(list: &[(Joiner, Pipeline)]) -> String {
let mut line = String::new();
for (i, (joiner, pipeline)) in list.iter().enumerate() {
if i > 0 {
line.push_str(match joiner {
Joiner::Always => "; ",
Joiner::IfOk => " && ",
Joiner::IfFailed => " || ",
});
}
let stages: Vec<String> = pipeline.stages.iter().map(render_stage).collect();
line.push_str(&stages.join(" | "));
}
line
}
fn render_stage(stage: &Stage) -> String {
let mut parts: Vec<String> = Vec::new();
for (name, value) in &stage.assignments {
parts.push(format!("{name}={}", render_text(value)));
}
for word in &stage.words {
parts.push(render_text(&word.text));
}
for redirect in &stage.redirects {
parts.push(match redirect {
Redirect::Out { target, append } => {
format!("{} {}", if *append { ">>" } else { ">" }, render_text(target))
}
Redirect::Err { target, append } => {
format!("{} {}", if *append { "2>>" } else { "2>" }, render_text(target))
}
Redirect::ErrToOut => "2>&1".to_string(),
Redirect::In(file) => format!("< {}", render_text(file)),
});
}
parts.join(" ")
}
fn render_text(text: &str) -> String {
let plain = |c: char| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.' | '/' | ',' | ':' | '+' | '@' | '%');
let mut out = String::new();
for (i, piece) in text.split(STATUS_MARK).enumerate() {
if i > 0 {
out.push_str("\"$?\"");
}
if piece.chars().all(plain) {
out.push_str(piece);
} else {
out.push_str(&single_quote(piece));
}
}
if out.is_empty() {
out.push_str("''");
}
out
}
fn single_quote(text: &str) -> String {
format!("'{}'", text.replace('\'', r"'\''"))
}
#[allow(unused_assignments)]
fn tokenize(line: &str, lookup: &dyn Fn(&str) -> Option<String>) -> Result<Vec<Token>, String> {
let chars: Vec<char> = line.chars().collect();
let mut tokens = Vec::new();
let mut i = 0;
let mut text = String::new();
let mut glob = false;
let mut started = false;
macro_rules! end_word {
() => {
if started {
tokens.push(Token::Word(Word { text: std::mem::take(&mut text), glob }));
glob = false;
started = false;
}
};
}
while i < chars.len() {
let c = chars[i];
match c {
' ' | '\t' => {
end_word!();
i += 1;
}
'#' if !started => break, '\'' => {
started = true;
i += 1;
let close = chars[i..].iter().position(|&ch| ch == '\'').ok_or("a ' quote is never closed")?;
text.extend(&chars[i..i + close]);
i += close + 1;
}
'"' => {
started = true;
i += 1;
loop {
let Some(&ch) = chars.get(i) else {
return Err("a \" quote is never closed".to_string());
};
match ch {
'"' => {
i += 1;
break;
}
'\\' if matches!(chars.get(i + 1), Some('"' | '\\' | '$' | '`')) => {
text.push(chars[i + 1]);
i += 2;
}
'$' => {
let (value, used) = expand_dollar(&chars[i..], lookup)?;
text.push_str(&value);
i += used;
}
'`' => return Err(unsupported_substitution()),
_ => {
text.push(ch);
i += 1;
}
}
}
}
'\\' => {
started = true;
if let Some(&next) = chars.get(i + 1) {
text.push(next);
}
i += 2;
}
'$' => {
let (value, used) = expand_dollar(&chars[i..], lookup)?;
i += used;
let assigning = text.split_once('=').is_some_and(|(n, _)| is_name(n)) && !glob;
if assigning {
text.push_str(&value);
started = true;
continue;
}
if value.starts_with(char::is_whitespace) {
end_word!();
}
let mut pieces = value.split_whitespace().peekable();
while let Some(piece) = pieces.next() {
text.push_str(piece);
started = true;
if pieces.peek().is_some() {
end_word!();
}
}
if value.ends_with(char::is_whitespace) {
end_word!();
}
}
'`' => return Err(unsupported_substitution()),
'~' if !started => {
let next = chars.get(i + 1);
if next.is_none() || matches!(next, Some('/' | ' ' | '\t' | ';' | '|' | '&')) {
text.push_str(&lookup("HOME").unwrap_or_else(|| "~".to_string()));
} else {
text.push('~');
}
started = true;
i += 1;
}
'|' => {
end_word!();
if chars.get(i + 1) == Some(&'|') {
tokens.push(Token::Or);
i += 2;
} else {
tokens.push(Token::Pipe);
i += 1;
}
}
'&' => {
end_word!();
if chars.get(i + 1) == Some(&'&') {
tokens.push(Token::And);
i += 2;
} else {
return Err("running in the background with & isn't available in the sandbox".to_string());
}
}
';' => {
end_word!();
tokens.push(Token::Semi);
i += 1;
}
'>' => {
let is_err = started && text == "2" && !glob && i > 0 && chars[i - 1] == '2';
if is_err {
text.clear();
started = false;
} else {
end_word!();
}
let append = chars.get(i + 1) == Some(&'>');
i += if append { 2 } else { 1 };
if is_err {
if !append && chars.get(i) == Some(&'&') && chars.get(i + 1) == Some(&'1') {
tokens.push(Token::ErrToOut);
i += 2;
} else {
tokens.push(Token::Err(append));
}
} else {
tokens.push(Token::Out(append));
}
}
'<' => {
end_word!();
tokens.push(Token::In);
i += 1;
}
'*' | '?' | '[' => {
started = true;
glob = true;
text.push(c);
i += 1;
}
_ => {
started = true;
text.push(c);
i += 1;
}
}
}
end_word!();
Ok(tokens)
}
fn unsupported_substitution() -> String {
"running a command inside another with $(...) or backticks isn't available in the sandbox".to_string()
}
fn expand_dollar(
chars: &[char],
lookup: &dyn Fn(&str) -> Option<String>,
) -> Result<(String, usize), String> {
match chars.get(1) {
Some('(') => Err(unsupported_substitution()),
Some('{') => {
let close = chars
.iter()
.position(|&c| c == '}')
.ok_or("a ${ is never closed with }")?;
let name: String = chars[2..close].iter().collect();
if !(is_name(&name) || name.chars().all(|c| c.is_ascii_digit()) && !name.is_empty()) {
return Err(format!("${{{name}}}: bad variable name"));
}
Ok((lookup(&name).unwrap_or_default(), close + 1))
}
Some(&c) if c.is_ascii_digit() || matches!(c, '#' | '@' | '?' | '*') => {
Ok((lookup(&c.to_string()).unwrap_or_default(), 2))
}
Some(&c) if c.is_ascii_alphabetic() || c == '_' => {
let len = chars[1..]
.iter()
.take_while(|c| c.is_ascii_alphanumeric() || **c == '_')
.count();
let name: String = chars[1..1 + len].iter().collect();
Ok((lookup(&name).unwrap_or_default(), 1 + len))
}
_ => Ok(("$".to_string(), 1)),
}
}
pub fn glob_match(pattern: &str, name: &str) -> bool {
if name.starts_with('.') && !pattern.starts_with('.') {
return false;
}
fnmatch(pattern, name)
}
pub fn fnmatch(pattern: &str, name: &str) -> bool {
let p: Vec<char> = pattern.chars().collect();
let n: Vec<char> = name.chars().collect();
match_from(&p, &n)
}
fn match_from(p: &[char], n: &[char]) -> bool {
match p.first() {
None => n.is_empty(),
Some('*') => (0..=n.len()).any(|skip| match_from(&p[1..], &n[skip..])),
Some('?') => !n.is_empty() && match_from(&p[1..], &n[1..]),
Some('[') => {
let Some(close) = p.iter().skip(1).position(|&c| c == ']').map(|x| x + 1) else {
return n.first() == Some(&'[') && match_from(&p[1..], &n[1..]);
};
let Some(&ch) = n.first() else { return false };
let mut set = &p[1..close];
let negate = matches!(set.first(), Some('!' | '^'));
if negate {
set = &set[1..];
}
let mut hit = false;
let mut k = 0;
while k < set.len() {
if k + 2 < set.len() && set[k + 1] == '-' {
hit |= set[k] <= ch && ch <= set[k + 2];
k += 3;
} else {
hit |= set[k] == ch;
k += 1;
}
}
hit != negate && match_from(&p[close + 1..], &n[1..])
}
Some(&c) => n.first() == Some(&c) && match_from(&p[1..], &n[1..]),
}
}
#[cfg(test)]
mod tests {
use super::*;
fn vars(name: &str) -> Option<String> {
match name {
"HOME" => Some("/lesson-home".into()),
"NAME" => Some("Ada".into()),
"SPACED" => Some("a b".into()),
"EMPTY" => Some(String::new()),
"1" => Some("first".into()),
"#" => Some("1".into()),
_ => None,
}
}
fn one(line: &str) -> Stage {
let list = parse(line, &vars).unwrap();
assert_eq!(list.len(), 1, "{line}");
assert_eq!(list[0].1.stages.len(), 1, "{line}");
list[0].1.stages[0].clone()
}
fn texts(stage: &Stage) -> Vec<&str> {
stage.words.iter().map(|w| w.text.as_str()).collect()
}
#[test]
fn quotes_and_variables() {
assert_eq!(texts(&one("echo $NAME")), ["echo", "Ada"]);
assert_eq!(texts(&one("echo \"hi $NAME\"")), ["echo", "hi Ada"]);
assert_eq!(texts(&one("echo '$NAME'")), ["echo", "$NAME"]);
assert_eq!(texts(&one("echo ${NAME}s")), ["echo", "Adas"]);
assert_eq!(texts(&one("echo $1 $#")), ["echo", "first", "1"]);
assert_eq!(texts(&one("echo $MISSING end")), ["echo", "end"]);
assert_eq!(texts(&one("echo \"$MISSING\" end")), ["echo", "", "end"]);
assert_eq!(texts(&one("echo $SPACED")), ["echo", "a", "b"]);
assert_eq!(texts(&one("echo \"$SPACED\"")), ["echo", "a b"]);
assert_eq!(texts(&one("echo x$SPACED")), ["echo", "xa", "b"]);
assert_eq!(texts(&one("echo $EMPTY")), ["echo"]);
let s = one("X=$SPACED");
assert_eq!(s.assignments, [("X".to_string(), "a b".to_string())]);
assert_eq!(texts(&one("echo \"\"")), ["echo", ""]);
assert_eq!(texts(&one("echo a\\ b")), ["echo", "a b"]);
assert_eq!(texts(&one("echo costs $5")), ["echo", "costs"]);
assert_eq!(texts(&one("echo 5$")), ["echo", "5$"]);
}
#[test]
fn home_tilde() {
assert_eq!(texts(&one("cd ~")), ["cd", "/lesson-home"]);
assert_eq!(texts(&one("ls ~/notes")), ["ls", "/lesson-home/notes"]);
assert_eq!(texts(&one("echo a~b '~'")), ["echo", "a~b", "~"]);
}
#[test]
fn globs_are_marked_only_when_unquoted() {
let s = one("find . -name '*.txt' *.log");
assert!(!s.words[3].glob);
assert!(s.words[4].glob);
}
#[test]
fn assignments() {
let s = one("GREETING=hello");
assert_eq!(s.assignments, [("GREETING".to_string(), "hello".to_string())]);
assert!(s.words.is_empty());
let s = one("LANG=C sort names.txt");
assert_eq!(s.assignments.len(), 1);
assert_eq!(texts(&s), ["sort", "names.txt"]);
assert_eq!(texts(&one("echo A=1")), ["echo", "A=1"]);
}
#[test]
fn redirections() {
let s = one("ls > out.txt");
assert_eq!(texts(&s), ["ls"]);
assert_eq!(s.redirects, [Redirect::Out { target: "out.txt".into(), append: false }]);
let s = one("echo hi>>log.txt");
assert_eq!(texts(&s), ["echo", "hi"]);
assert_eq!(s.redirects, [Redirect::Out { target: "log.txt".into(), append: true }]);
let s = one("cat missing 2>/dev/null");
assert_eq!(texts(&s), ["cat", "missing"]);
assert_eq!(s.redirects, [Redirect::Err { target: "/dev/null".into(), append: false }]);
let s = one("cat a b > all.txt 2>&1");
assert_eq!(s.redirects.len(), 2);
assert_eq!(s.redirects[1], Redirect::ErrToOut);
let s = one("wc -l < names.txt");
assert_eq!(texts(&s), ["wc", "-l"]);
assert_eq!(s.redirects, [Redirect::In("names.txt".into())]);
assert_eq!(texts(&one("echo 2 > f")), ["echo", "2"]);
assert_eq!(texts(&one("echo a2> f")), ["echo", "a2"]);
}
#[test]
fn pipes_and_lists() {
let list = parse("cat log | grep ERROR | wc -l", &vars).unwrap();
assert_eq!(list[0].1.stages.len(), 3);
let list = parse("mkdir x && cd x; ls || echo empty", &vars).unwrap();
let joiners: Vec<Joiner> = list.iter().map(|(j, _)| *j).collect();
assert_eq!(joiners, [Joiner::Always, Joiner::IfOk, Joiner::Always, Joiner::IfFailed]);
assert!(parse("ls;", &vars).is_ok());
assert!(parse("ls && ", &vars).is_err());
assert!(parse("| grep x", &vars).is_err());
assert!(parse("ls |", &vars).is_err());
assert!(parse("echo > ", &vars).is_err());
}
#[test]
fn split_list_respects_quotes() {
let parts = split_list("X=5; echo $X && echo 'a;b' || echo \"c&&d\" # done");
let texts: Vec<&str> = parts.iter().map(|(_, t)| t.trim()).collect();
assert_eq!(texts, ["X=5", "echo $X", "echo 'a;b'", "echo \"c&&d\""]);
let joiners: Vec<Joiner> = parts.iter().map(|(j, _)| *j).collect();
assert_eq!(joiners, [Joiner::Always, Joiner::Always, Joiner::IfOk, Joiner::IfFailed]);
assert_eq!(split_list("ls;").len(), 1);
assert_eq!(split_list("cat a | grep b").len(), 1);
let parts = split_list("ls &&");
assert_eq!(parts.len(), 2);
assert!(parse(&parts[1].1, &vars).unwrap().is_empty());
}
#[test]
fn unsupported_syntax_says_so() {
for line in ["echo $(date)", "echo `date`", "sleep 5 &", "echo 'open"] {
assert!(parse(line, &vars).is_err(), "{line}");
}
}
#[test]
fn comments() {
assert!(parse("# just a note", &vars).unwrap().is_empty());
assert_eq!(texts(&one("echo hi # note")), ["echo", "hi"]);
assert_eq!(texts(&one("echo a#b")), ["echo", "a#b"]);
}
#[test]
fn glob_matching() {
assert!(glob_match("*.txt", "notes.txt"));
assert!(!glob_match("*.txt", "notes.md"));
assert!(glob_match("log?.txt", "log1.txt"));
assert!(!glob_match("log?.txt", "log10.txt"));
assert!(glob_match("[abc]*", "beta"));
assert!(glob_match("[a-c]*", "cat"));
assert!(!glob_match("[!a-c]*", "cat"));
assert!(!glob_match("*", ".hidden"));
assert!(glob_match(".*", ".hidden"));
assert!(glob_match("*", ""));
assert!(fnmatch("*.txt", ".hidden.txt"));
}
fn rendered(line: &str) -> String {
let lookup = |name: &str| match name {
"?" => Some(STATUS_MARK.to_string()),
"TRICK" => Some("$(touch pwned) `x` $HOME".to_string()),
"Q" => Some("it's".to_string()),
n => vars(n),
};
render(&parse(line, &lookup).unwrap())
}
#[test]
fn render_quotes_everything_the_shell_would_expand() {
assert_eq!(rendered("echo $NAME"), "echo Ada");
assert_eq!(rendered("echo \"Hi, $NAME!\""), "echo 'Hi, Ada!'");
assert_eq!(rendered("echo $SPACED"), "echo a b");
assert_eq!(rendered("echo \"$SPACED\""), "echo 'a b'");
assert_eq!(rendered("echo \"[$EMPTY]\""), "echo '[]'");
assert_eq!(rendered("echo \"$EMPTY\""), "echo ''");
assert_eq!(rendered("echo \"$Q\""), r"echo 'it'\''s'");
assert_eq!(rendered("echo \"$TRICK\""), "echo '$(touch pwned) `x` $HOME'");
assert_eq!(rendered("echo 'echo $1' > greet.sh"), "echo 'echo $1' > greet.sh");
assert_eq!(rendered("cd ~"), "cd /lesson-home");
assert_eq!(rendered("X=$NAME env"), "X=Ada env");
assert_eq!(rendered("echo X=1"), "echo 'X=1'");
}
#[test]
fn render_keeps_globs_lists_and_redirects() {
assert_eq!(rendered("ls *.txt [!a]? .*"), "ls '*.txt' '[!a]?' '.*'");
assert_eq!(rendered("ls '*.txt'"), "ls '*.txt'");
assert_eq!(
rendered("grep x a 2>/dev/null | sort >> out && echo ok || echo no; cat < in 2>&1"),
"grep x a 2> /dev/null | sort >> out && echo ok || echo no; cat < in 2>&1"
);
assert_eq!(rendered("false; echo $?"), "false; echo \"$?\"");
assert_eq!(rendered("echo \"status $?\""), "echo 'status '\"$?\"");
}
#[test]
fn rendered_lines_mean_the_same_to_sh() {
let line = "echo \"$TRICK\" $SPACED \"$Q\" x=1 '$NAME'";
let out = std::process::Command::new("sh")
.arg("-c")
.arg(rendered(line))
.env_clear()
.current_dir(std::env::temp_dir())
.output()
.expect("sh runs");
assert_eq!(
String::from_utf8_lossy(&out.stdout),
"$(touch pwned) `x` $HOME a b it's x=1 $NAME\n"
);
let status = std::process::Command::new("sh")
.arg("-c")
.arg(rendered("false; echo $?"))
.output()
.expect("sh runs");
assert_eq!(String::from_utf8_lossy(&status.stdout), "1\n");
}
}