use crate::shell::ast::Arg;
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Symbol {
Pipe,
OrOr,
AndAnd,
Ampersand,
Semicolon,
Redirect(String),
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Token {
Word(Arg),
Symbol(Symbol),
Eof,
}
#[derive(Clone, Debug)]
pub struct Lexer {
data: Vec<char>,
pos: usize,
win32_escapes: bool,
}
impl Lexer {
pub fn new(input: &str, win32_escapes: bool) -> Self {
Self {
data: input.chars().collect(),
pos: 0,
win32_escapes,
}
}
pub fn next_token(&mut self) -> Result<Token, String> {
self.skip_ws();
if self.pos >= self.data.len() {
return Ok(Token::Eof);
}
let ch = self.data[self.pos];
if Self::is_meta(ch) {
return Ok(Token::Symbol(self.read_symbol()?));
}
self.lex_word()
}
fn is_meta(c: char) -> bool {
matches!(c, ';' | '|' | '&' | '>' | '<')
}
fn skip_ws(&mut self) {
while self.pos < self.data.len() {
if self.data[self.pos].is_whitespace() {
self.pos += 1;
} else {
break;
}
}
}
fn read_symbol(&mut self) -> Result<Symbol, String> {
let ch = self.bump();
match ch {
';' => Ok(Symbol::Semicolon),
'|' => {
if self.matches('|') {
Ok(Symbol::OrOr)
} else {
Ok(Symbol::Pipe)
}
}
'&' => {
if self.matches('&') {
Ok(Symbol::AndAnd)
} else if self.matches('>') {
Ok(Symbol::Redirect("&>".to_string()))
} else {
Ok(Symbol::Ampersand)
}
}
'>' => {
if self.matches('&') {
Ok(Symbol::Redirect(">&".into()))
} else if self.matches('>') {
Ok(Symbol::Redirect(">>".into()))
} else {
Ok(Symbol::Redirect(">".into()))
}
}
'<' => {
if self.matches('&') {
Ok(Symbol::Redirect("<&".into()))
} else if self.matches('<') {
Ok(Symbol::Redirect("<<".into()))
} else {
Ok(Symbol::Redirect("<".into()))
}
}
_ => Err(format!("unexpected token start '{ch}'")),
}
}
fn matches(&mut self, expected: char) -> bool {
if self.pos < self.data.len() && self.data[self.pos] == expected {
self.pos += 1;
true
} else {
false
}
}
fn bump(&mut self) -> char {
let ch = self.data[self.pos];
self.pos += 1;
ch
}
fn lex_word(&mut self) -> Result<Token, String> {
let mut out = String::new();
let mut active_metacharacters = Vec::new();
let mut consumed = false;
while self.pos < self.data.len() {
let ch = self.data[self.pos];
if ch.is_whitespace() || Self::is_meta(ch) {
break;
}
self.pos += 1;
consumed = true;
if ch == '\\' && !self.win32_escapes {
if self.pos >= self.data.len() {
return Err("escape at end of argument".to_string());
}
let next = self.bump();
out.push(next);
continue;
}
if ch == '\'' || ch == '"' {
let quoted = self.lex_quoted(ch)?;
out.push_str("ed);
continue;
}
if matches!(ch, '*' | '?' | '[' | ']') {
active_metacharacters.push(out.chars().count());
}
out.push(ch);
}
if !consumed {
return Err("empty token".to_string());
}
Ok(Token::Word(Arg::from_glob_parts(
out,
active_metacharacters,
)))
}
fn lex_quoted(&mut self, delim: char) -> Result<String, String> {
let mut out = String::new();
while self.pos < self.data.len() {
let ch = self.bump();
if ch == delim {
return Ok(out);
}
if ch == '\\' && delim == '"' {
if self.pos >= self.data.len() {
return Err("escape at end of quoted argument".to_string());
}
let next = self.bump();
let translated = match next {
'\\' | '"' | '$' => next,
_ => {
out.push('\\');
next
}
};
out.push(translated);
continue;
}
out.push(ch);
}
Err(format!("unterminated {delim} quote"))
}
}
#[cfg(test)]
mod tests {
use super::{Lexer, Symbol, Token};
use crate::shell::ast::Arg;
#[test]
fn quoted_tokens() {
let mut lex = Lexer::new("\"a b\" 'c d' e", false);
assert_eq!(
lex.next_token().unwrap(),
Token::Word(Arg::Text("a b".into()))
);
assert_eq!(
lex.next_token().unwrap(),
Token::Word(Arg::Text("c d".into()))
);
assert_eq!(
lex.next_token().unwrap(),
Token::Word(Arg::Text("e".into()))
);
assert_eq!(lex.next_token().unwrap(), Token::Eof);
}
#[test]
fn redirections_are_tokens() {
let mut lex = Lexer::new("echo a | grep b > out 2>&1", false);
assert!(matches!(lex.next_token().unwrap(), Token::Word(_)));
assert!(matches!(lex.next_token().unwrap(), Token::Word(_)));
assert!(matches!(
lex.next_token().unwrap(),
Token::Symbol(Symbol::Pipe)
));
assert!(matches!(lex.next_token().unwrap(), Token::Word(_)));
assert!(matches!(lex.next_token().unwrap(), Token::Word(_)));
assert!(matches!(
lex.next_token().unwrap(),
Token::Symbol(Symbol::Redirect(_))
));
assert!(matches!(lex.next_token().unwrap(), Token::Word(_)));
assert!(matches!(lex.next_token().unwrap(), Token::Word(_)));
assert!(matches!(
lex.next_token().unwrap(),
Token::Symbol(Symbol::Redirect(_))
));
assert!(matches!(lex.next_token().unwrap(), Token::Word(_)));
assert!(matches!(lex.next_token().unwrap(), Token::Eof));
}
}