litnik 0.2.1

An embedded Lit and FileCheck runner for compiler test suites.
Documentation
use crate::shell::ast::Arg;

/// A shell operator recognized outside quoted arguments.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Symbol {
    /// `|`
    Pipe,
    /// `||`
    OrOr,
    /// `&&`
    AndAnd,
    /// `&`
    Ampersand,
    /// `;`
    Semicolon,
    /// A redirection operator.
    Redirect(String),
}

/// One token produced by [`Lexer`].
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Token {
    /// An argument with quote removal already applied.
    Word(Arg),
    /// A control or redirection operator.
    Symbol(Symbol),
    /// The end of the input.
    Eof,
}

/// Tokenizes the restricted shell language used by Lit `RUN:` directives.
///
/// The lexer performs quote removal and escape processing but deliberately
/// leaves pathname expansion to the executor.
#[derive(Clone, Debug)]
pub struct Lexer {
    data: Vec<char>,
    pos: usize,
    win32_escapes: bool,
}

impl Lexer {
    /// Creates a lexer over `input`.
    ///
    /// When `win32_escapes` is true, backslashes outside quotes are retained as
    /// path separators instead of consuming the following character.
    pub fn new(input: &str, win32_escapes: bool) -> Self {
        Self {
            data: input.chars().collect(),
            pos: 0,
            win32_escapes,
        }
    }

    /// Returns the next token.
    ///
    /// # Errors
    ///
    /// Returns an error for an incomplete escape or unterminated quote.
    pub fn next_token(&mut self) -> Result<Token, String> {
        self.skip_ws();
        if self.pos >= self.data.len() {
            return Ok(Token::Eof);
        }

        let ch = self.data[self.pos];
        if Self::is_meta(ch) {
            return Ok(Token::Symbol(self.read_symbol()?));
        }

        self.lex_word()
    }

    fn is_meta(c: char) -> bool {
        matches!(c, ';' | '|' | '&' | '>' | '<')
    }

    fn skip_ws(&mut self) {
        while self.pos < self.data.len() {
            if self.data[self.pos].is_whitespace() {
                self.pos += 1;
            } else {
                break;
            }
        }
    }

    fn read_symbol(&mut self) -> Result<Symbol, String> {
        let ch = self.bump();

        match ch {
            ';' => Ok(Symbol::Semicolon),
            '|' => {
                if self.matches('|') {
                    Ok(Symbol::OrOr)
                } else {
                    Ok(Symbol::Pipe)
                }
            }
            '&' => {
                if self.matches('&') {
                    Ok(Symbol::AndAnd)
                } else if self.matches('>') {
                    Ok(Symbol::Redirect("&>".to_string()))
                } else {
                    Ok(Symbol::Ampersand)
                }
            }
            '>' => {
                if self.matches('&') {
                    Ok(Symbol::Redirect(">&".into()))
                } else if self.matches('>') {
                    Ok(Symbol::Redirect(">>".into()))
                } else {
                    Ok(Symbol::Redirect(">".into()))
                }
            }
            '<' => {
                if self.matches('&') {
                    Ok(Symbol::Redirect("<&".into()))
                } else if self.matches('<') {
                    Ok(Symbol::Redirect("<<".into()))
                } else {
                    Ok(Symbol::Redirect("<".into()))
                }
            }
            _ => Err(format!("unexpected token start '{ch}'")),
        }
    }

    fn matches(&mut self, expected: char) -> bool {
        if self.pos < self.data.len() && self.data[self.pos] == expected {
            self.pos += 1;
            true
        } else {
            false
        }
    }

    fn bump(&mut self) -> char {
        let ch = self.data[self.pos];
        self.pos += 1;
        ch
    }

    fn lex_word(&mut self) -> Result<Token, String> {
        let mut out = String::new();
        let mut active_metacharacters = Vec::new();
        let mut consumed = false;

        while self.pos < self.data.len() {
            let ch = self.data[self.pos];
            if ch.is_whitespace() || Self::is_meta(ch) {
                break;
            }

            self.pos += 1;
            consumed = true;
            if ch == '\\' && !self.win32_escapes {
                if self.pos >= self.data.len() {
                    return Err("escape at end of argument".to_string());
                }
                let next = self.bump();
                out.push(next);
                continue;
            }

            if ch == '\'' || ch == '"' {
                let quoted = self.lex_quoted(ch)?;
                out.push_str(&quoted);
                continue;
            }

            if matches!(ch, '*' | '?' | '[' | ']') {
                active_metacharacters.push(out.chars().count());
            }
            out.push(ch);
        }

        if !consumed {
            return Err("empty token".to_string());
        }

        Ok(Token::Word(Arg::from_glob_parts(
            out,
            active_metacharacters,
        )))
    }

    fn lex_quoted(&mut self, delim: char) -> Result<String, String> {
        let mut out = String::new();
        while self.pos < self.data.len() {
            let ch = self.bump();
            if ch == delim {
                return Ok(out);
            }
            if ch == '\\' && delim == '"' {
                if self.pos >= self.data.len() {
                    return Err("escape at end of quoted argument".to_string());
                }
                let next = self.bump();
                let translated = match next {
                    '\\' | '"' | '$' => next,
                    _ => {
                        out.push('\\');
                        next
                    }
                };
                out.push(translated);
                continue;
            }
            out.push(ch);
        }
        Err(format!("unterminated {delim} quote"))
    }
}

#[cfg(test)]
mod tests {
    use super::{Lexer, Symbol, Token};
    use crate::shell::ast::Arg;

    #[test]
    fn quoted_tokens() {
        let mut lex = Lexer::new("\"a b\" 'c d' e", false);
        assert_eq!(
            lex.next_token().unwrap(),
            Token::Word(Arg::Text("a b".into()))
        );
        assert_eq!(
            lex.next_token().unwrap(),
            Token::Word(Arg::Text("c d".into()))
        );
        assert_eq!(
            lex.next_token().unwrap(),
            Token::Word(Arg::Text("e".into()))
        );
        assert_eq!(lex.next_token().unwrap(), Token::Eof);
    }

    #[test]
    fn redirections_are_tokens() {
        let mut lex = Lexer::new("echo a | grep b > out 2>&1", false);
        assert!(matches!(lex.next_token().unwrap(), Token::Word(_)));
        assert!(matches!(lex.next_token().unwrap(), Token::Word(_)));
        assert!(matches!(
            lex.next_token().unwrap(),
            Token::Symbol(Symbol::Pipe)
        ));
        assert!(matches!(lex.next_token().unwrap(), Token::Word(_)));
        assert!(matches!(lex.next_token().unwrap(), Token::Word(_)));
        assert!(matches!(
            lex.next_token().unwrap(),
            Token::Symbol(Symbol::Redirect(_))
        ));
        assert!(matches!(lex.next_token().unwrap(), Token::Word(_)));
        assert!(matches!(lex.next_token().unwrap(), Token::Word(_)));
        assert!(matches!(
            lex.next_token().unwrap(),
            Token::Symbol(Symbol::Redirect(_))
        ));
        assert!(matches!(lex.next_token().unwrap(), Token::Word(_)));
        assert!(matches!(lex.next_token().unwrap(), Token::Eof));
    }
}