use crate::shell::ast::Arg;
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Symbol {
Pipe,
OrOr,
AndAnd,
Ampersand,
Semicolon,
Redirect(String),
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Token {
Word(Arg),
Symbol(Symbol),
Eof,
}
#[derive(Clone, Debug)]
pub struct Lexer {
data: Vec<u8>,
pos: usize,
win32_escapes: bool,
}
impl Lexer {
pub fn new(input: &str, win32_escapes: bool) -> Self {
Self {
data: input.as_bytes().to_vec(),
pos: 0,
win32_escapes,
}
}
pub fn next_token(&mut self) -> Result<Token, String> {
self.skip_ws();
if self.pos >= self.data.len() {
return Ok(Token::Eof);
}
let ch = self.data[self.pos] as char;
if Self::is_meta(ch) {
return Ok(Token::Symbol(self.read_symbol()?));
}
if ch == '\'' || ch == '"' {
self.pos += 1;
return Ok(Token::Word(Arg::Text(self.lex_quoted(ch))));
}
self.lex_word()
}
fn is_meta(c: char) -> bool {
matches!(c, ';' | '|' | '&' | '>' | '<')
}
fn skip_ws(&mut self) {
while self.pos < self.data.len() {
if (self.data[self.pos] as char).is_whitespace() {
self.pos += 1;
} else {
break;
}
}
}
fn read_symbol(&mut self) -> Result<Symbol, String> {
let ch = self.bump();
match ch {
';' => Ok(Symbol::Semicolon),
'|' => {
if self.matches('|') {
Ok(Symbol::OrOr)
} else {
Ok(Symbol::Pipe)
}
}
'&' => {
if self.matches('&') {
Ok(Symbol::AndAnd)
} else if self.matches('>') {
Ok(Symbol::Redirect("&>".to_string()))
} else {
Ok(Symbol::Ampersand)
}
}
'>' => {
if self.matches('&') {
Ok(Symbol::Redirect(">&".into()))
} else if self.matches('>') {
Ok(Symbol::Redirect(">>".into()))
} else {
Ok(Symbol::Redirect(">".into()))
}
}
'<' => {
if self.matches('&') {
Ok(Symbol::Redirect("<&".into()))
} else if self.matches('<') {
Ok(Symbol::Redirect("<<".into()))
} else {
Ok(Symbol::Redirect("<".into()))
}
}
_ => Err(format!("unexpected token start '{ch}'")),
}
}
fn matches(&mut self, expected: char) -> bool {
if self.pos < self.data.len() && self.data[self.pos] as char == expected {
self.pos += 1;
true
} else {
false
}
}
fn bump(&mut self) -> char {
let ch = self.data[self.pos] as char;
self.pos += 1;
ch
}
fn lex_word(&mut self) -> Result<Token, String> {
let mut out = String::new();
let mut has_glob = false;
let mut quoted_has_glob = false;
while self.pos < self.data.len() {
let ch = self.data[self.pos] as char;
if ch.is_whitespace() || Self::is_meta(ch) {
break;
}
self.pos += 1;
if ch == '\\' && !self.win32_escapes {
if self.pos >= self.data.len() {
return Err("escape at end of argument".to_string());
}
let next = self.bump();
out.push(next);
continue;
}
if ch == '\'' || ch == '"' {
let quoted = self.lex_quoted(ch);
if quoted.contains('*') || quoted.contains('?') {
quoted_has_glob = true;
}
out.push_str("ed);
continue;
}
if ch == '*' || ch == '?' {
has_glob = true;
}
out.push(ch);
}
if out.is_empty() {
return Err("empty token".to_string());
}
if has_glob && !quoted_has_glob {
Ok(Token::Word(Arg::Glob(out)))
} else {
Ok(Token::Word(Arg::Text(out)))
}
}
fn lex_quoted(&mut self, delim: char) -> String {
let mut out = String::new();
while self.pos < self.data.len() {
let ch = self.bump();
if ch == delim {
return out;
}
if ch == '\\' && delim == '"' {
if self.pos >= self.data.len() {
out.push('\\');
return out;
}
let next = self.bump();
let translated = match next {
'\\' | '"' | '$' => next,
_ => {
out.push('\\');
next
}
};
out.push(translated);
continue;
}
out.push(ch);
}
out
}
}
#[cfg(test)]
mod tests {
use super::{Lexer, Symbol, Token};
use crate::shell::ast::Arg;
#[test]
fn quoted_tokens() {
let mut lex = Lexer::new("\"a b\" 'c d' e", false);
assert_eq!(
lex.next_token().unwrap(),
Token::Word(Arg::Text("a b".into()))
);
assert_eq!(
lex.next_token().unwrap(),
Token::Word(Arg::Text("c d".into()))
);
assert_eq!(
lex.next_token().unwrap(),
Token::Word(Arg::Text("e".into()))
);
assert_eq!(lex.next_token().unwrap(), Token::Eof);
}
#[test]
fn redirections_are_tokens() {
let mut lex = Lexer::new("echo a | grep b > out 2>&1", false);
assert!(matches!(lex.next_token().unwrap(), Token::Word(_)));
assert!(matches!(lex.next_token().unwrap(), Token::Word(_)));
assert!(matches!(
lex.next_token().unwrap(),
Token::Symbol(Symbol::Pipe)
));
assert!(matches!(lex.next_token().unwrap(), Token::Word(_)));
assert!(matches!(lex.next_token().unwrap(), Token::Word(_)));
assert!(matches!(
lex.next_token().unwrap(),
Token::Symbol(Symbol::Redirect(_))
));
assert!(matches!(lex.next_token().unwrap(), Token::Word(_)));
assert!(matches!(lex.next_token().unwrap(), Token::Word(_)));
assert!(matches!(
lex.next_token().unwrap(),
Token::Symbol(Symbol::Redirect(_))
));
assert!(matches!(lex.next_token().unwrap(), Token::Word(_)));
assert!(matches!(lex.next_token().unwrap(), Token::Eof));
}
}