use std::{error::Error, fmt::Display};
use crate::tokenizer::{Keyword, Token, TokenTag};
pub struct Parser<'a> {
tokens: &'a [Token<'a>],
idx: usize,
}
#[derive(Clone, PartialEq, Debug, Default)]
pub struct ParseError {
pub message: String,
pub line: usize,
pub col: usize,
pub len: usize,
}
impl From<(TokenTag<'_>, Token<'_>)> for ParseError {
fn from(value: (TokenTag<'_>, Token<'_>)) -> Self {
let (
want,
Token {
line,
col,
len,
tag,
},
) = value;
Self {
message: format!("Expected `{want}` after `{tag}`"),
line,
col,
len,
}
}
}
impl Display for ParseError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "parser error: {}", self.message)
}
}
impl Error for ParseError {}
impl<'a> Parser<'a> {
pub fn with_tokens(tokens: &'a [Token<'a>]) -> Self {
Self { tokens, idx: 0 }
}
#[inline]
fn peek(&self) -> Token<'a> {
self.tokens[self.idx]
}
#[inline]
fn peek_back(&self) -> Token<'a> {
self.tokens[self.idx - 1]
}
fn peek_n(&self, n: usize) -> Option<TokenTag<'a>> {
if self.idx + n >= self.tokens.len() {
None
} else {
Some(self.tokens[self.idx + n].tag)
}
}
fn consume(&mut self, token: &TokenTag<'_>) -> Result<(), ParseError> {
if &self.peek().tag == token {
self.idx += 1;
Ok(())
} else {
Err((*token, self.peek_back()).into())
}
}
fn at_end(&self) -> bool {
self.peek().tag == TokenTag::EOF
}
fn advance(&mut self) -> Token<'a> {
if !self.at_end() {
self.idx += 1
}
if self.idx == 0 {
self.tokens[0]
} else {
self.tokens[self.idx - 1]
}
}
pub fn parse_many(&mut self) -> Result<Vec<Expr<'a>>, ParseError> {
let mut expressions = vec![];
while !self.at_end() {
let expr = self.parse()?;
expressions.push(expr);
}
Ok(expressions)
}
pub fn parse(&mut self) -> Result<Expr<'a>, ParseError> {
let expr = self.statement()?;
Ok(expr)
}
fn statement(&mut self) -> Result<Expr<'a>, ParseError> {
match self.peek().tag {
TokenTag::Keyword(Keyword::Roar) => {
self.advance();
let next = self.expression()?;
self.consume(&TokenTag::Bang)?;
Ok(Expr::Roar(Box::new(next)))
}
TokenTag::Keyword(Keyword::Print) => {
self.advance();
let next = self.expression()?;
self.consume(&TokenTag::Semicolon)?;
Ok(Expr::Print(Box::new(next)))
}
TokenTag::OpenBrace => self.block(),
TokenTag::Keyword(Keyword::For) => self.for_statement(),
TokenTag::Keyword(Keyword::If) => self.if_statement(),
TokenTag::Keyword(Keyword::While) => self.while_statement(),
_ => {
let res = self.expression()?;
self.consume(&TokenTag::Semicolon)?;
Ok(res)
}
}
}
fn while_statement(&mut self) -> Result<Expr<'a>, ParseError> {
self.consume(&TokenTag::Keyword(Keyword::While))?;
self.consume(&TokenTag::OpenParen)?;
let condition = Box::new(self.equality()?);
self.consume(&TokenTag::CloseParen)?;
let exec = Box::new(self.statement()?);
Ok(Expr::WhileLoop { condition, exec })
}
fn for_statement(&mut self) -> Result<Expr<'a>, ParseError> {
self.consume(&TokenTag::Keyword(Keyword::For))?;
self.consume(&TokenTag::OpenParen)?;
let init = Box::new(self.expression()?);
self.consume(&TokenTag::Semicolon)?;
let check = Box::new(self.equality()?);
self.consume(&TokenTag::Semicolon)?;
let update = Box::new(self.expression()?);
self.consume(&TokenTag::CloseParen)?;
let exec = Box::new(self.statement()?);
Ok(Expr::ForLoop {
init,
check,
update,
exec,
})
}
fn block(&mut self) -> Result<Expr<'a>, ParseError> {
self.consume(&TokenTag::OpenBrace)?;
let mut block_items = vec![];
while self.peek().tag != TokenTag::CloseBrace {
let expr = self.statement()?;
block_items.push(expr);
}
self.consume(&TokenTag::CloseBrace)?;
Ok(Expr::Block(block_items))
}
fn if_statement(&mut self) -> Result<Expr<'a>, ParseError> {
self.consume(&TokenTag::Keyword(Keyword::If))?;
self.consume(&TokenTag::OpenParen)?;
let check = self.equality()?;
self.consume(&TokenTag::CloseParen)?;
let block = self.block()?;
let mut else_branch = None;
if self.peek().tag == TokenTag::Keyword(Keyword::Else) {
self.advance();
else_branch = Some(Box::new(self.statement()?));
}
Ok(Expr::Conditional {
condition: Box::new(check),
true_branch: Box::new(block),
else_branch,
})
}
fn expression(&mut self) -> Result<Expr<'a>, ParseError> {
match (self.peek().tag, self.peek_n(1)) {
(TokenTag::Keyword(Keyword::Var), _) => {
self.advance();
let advance = self.advance();
if let TokenTag::Identifier(name) = advance.tag {
self.consume(&TokenTag::Equal)?;
let val = self.equality()?;
Ok(Expr::Assignment {
name,
val: Box::new(val),
})
} else {
Err(ParseError {
message: format!("Expected Expression, found `{}`", advance.tag),
line: advance.line,
col: advance.col,
len: advance.len,
})
}
}
(TokenTag::Identifier(name), Some(TokenTag::Equal)) => {
self.advance();
self.advance();
let assignment = self.equality()?;
Ok(Expr::Reassignment {
name,
val: Box::new(assignment),
})
}
(TokenTag::Identifier(name), Some(TokenTag::PlusEq)) => {
self.advance();
self.advance();
let add = Box::new(self.equality()?);
Ok(Expr::AddAssign { name, add })
}
(TokenTag::Identifier(ident), Some(TokenTag::PlusPlus)) => {
self.advance();
self.advance();
Ok(Expr::Inc(ident, true))
}
(TokenTag::PlusPlus, Some(TokenTag::Identifier(ident))) => {
self.advance();
self.advance();
Ok(Expr::Inc(ident, false))
}
_ => self.equality(),
}
}
fn equality(&mut self) -> Result<Expr<'a>, ParseError> {
let mut expr = self.comparison()?;
while matches!(self.peek().tag, TokenTag::BangEqual | TokenTag::EqualEqual) {
let op = match self.advance().tag {
TokenTag::BangEqual => BinaryOperator::Neq,
TokenTag::EqualEqual => BinaryOperator::Eq,
_ => unreachable!(),
};
let right = self.comparison()?;
expr = Expr::Binary {
op,
left: Box::new(expr),
right: Box::new(right),
}
}
Ok(expr)
}
fn comparison(&mut self) -> Result<Expr<'a>, ParseError> {
let mut expr = self.term()?;
while matches!(
self.peek().tag,
TokenTag::Greater | TokenTag::GreaterEqual | TokenTag::Less | TokenTag::LessEqual
) {
let op = match self.advance().tag {
TokenTag::GreaterEqual => BinaryOperator::Gte,
TokenTag::Greater => BinaryOperator::Gt,
TokenTag::LessEqual => BinaryOperator::Lte,
TokenTag::Less => BinaryOperator::Lt,
_ => unreachable!(),
};
let right = self.term()?;
expr = Expr::Binary {
op,
left: Box::new(expr),
right: Box::new(right),
}
}
Ok(expr)
}
fn term(&mut self) -> Result<Expr<'a>, ParseError> {
let mut expr = self.factor()?;
while matches!(self.peek().tag, TokenTag::Plus | TokenTag::Minus) {
let op = match self.advance().tag {
TokenTag::Minus => BinaryOperator::Sub,
TokenTag::Plus => BinaryOperator::Add,
_ => unreachable!(),
};
let right = self.factor()?;
expr = Expr::Binary {
op,
left: Box::new(expr),
right: Box::new(right),
}
}
Ok(expr)
}
fn factor(&mut self) -> Result<Expr<'a>, ParseError> {
let mut expr = self.unary()?;
while matches!(self.peek().tag, TokenTag::Slash | TokenTag::Star) {
let op = match self.advance().tag {
TokenTag::Slash => BinaryOperator::Div,
TokenTag::Star => BinaryOperator::Mul,
_ => unreachable!(),
};
let right = self.unary()?;
expr = Expr::Binary {
op,
left: Box::new(expr),
right: Box::new(right),
}
}
Ok(expr)
}
fn unary(&mut self) -> Result<Expr<'a>, ParseError> {
if matches!(self.peek().tag, TokenTag::Bang | TokenTag::Minus) {
let op = match self.advance().tag {
TokenTag::Bang => UnaryOperator::Not,
TokenTag::Minus => UnaryOperator::Neg,
_ => unreachable!(),
};
let unary = self.unary()?;
Ok(Expr::Unary {
op,
node: Box::new(unary),
})
} else {
self.primary()
}
}
fn primary(&mut self) -> Result<Expr<'a>, ParseError> {
let advance = self.advance();
let mut prim = match advance.tag {
TokenTag::Number(n) => Expr::Literal(Literal::Number(n)),
TokenTag::Keyword(Keyword::True) => Expr::Literal(Literal::True),
TokenTag::Keyword(Keyword::False) => Expr::Literal(Literal::False),
TokenTag::Keyword(Keyword::Nil) => Expr::Literal(Literal::Nil),
TokenTag::Identifier(ident) => Expr::Variable(ident),
TokenTag::String(s) => Expr::Literal(Literal::String(s)),
TokenTag::OpenBracket => self.list()?,
TokenTag::OpenParen => {
let expr = self.expression()?;
self.consume(&TokenTag::CloseParen)?;
Expr::Grouping(Box::new(expr))
}
_ => {
return Err(ParseError {
message: format!("Unexpected token: `{}`", advance.tag),
line: advance.line,
col: advance.col,
len: advance.len,
});
}
};
while self.peek().tag == TokenTag::OpenBracket {
self.advance();
let index = Box::new(self.primary()?);
self.consume(&TokenTag::CloseBracket)?;
prim = Expr::Index {
item: Box::new(prim),
index,
};
}
Ok(prim)
}
fn list(&mut self) -> Result<Expr<'a>, ParseError> {
let mut items = vec![];
while self.peek().tag != TokenTag::CloseBracket {
items.push(self.equality()?);
if self.peek().tag != TokenTag::CloseBracket {
self.consume(&TokenTag::Comma)?;
}
}
self.consume(&TokenTag::CloseBracket)?;
Ok(Expr::List(items))
}
}
#[derive(Debug, PartialEq, Clone)]
pub enum Expr<'a> {
Index {
item: Box<Expr<'a>>,
index: Box<Expr<'a>>,
},
ForLoop {
init: Box<Expr<'a>>,
check: Box<Expr<'a>>,
update: Box<Expr<'a>>,
exec: Box<Expr<'a>>,
},
WhileLoop {
condition: Box<Expr<'a>>,
exec: Box<Expr<'a>>,
},
Conditional {
condition: Box<Expr<'a>>,
true_branch: Box<Expr<'a>>,
else_branch: Option<Box<Expr<'a>>>,
},
Block(Vec<Expr<'a>>),
Literal(Literal<'a>),
List(Vec<Expr<'a>>),
Variable(&'a str),
Unary {
op: UnaryOperator,
node: Box<Expr<'a>>,
},
Roar(Box<Expr<'a>>),
Print(Box<Expr<'a>>),
Reassignment {
name: &'a str,
val: Box<Expr<'a>>,
},
Inc(&'a str, bool),
AddAssign {
name: &'a str,
add: Box<Expr<'a>>,
},
Assignment {
name: &'a str,
val: Box<Expr<'a>>,
},
Binary {
op: BinaryOperator,
left: Box<Expr<'a>>,
right: Box<Expr<'a>>,
},
Grouping(Box<Expr<'a>>),
}
#[derive(PartialEq, Debug, Clone)]
pub enum Literal<'a> {
Number(f64),
String(&'a str),
Concat(Box<Literal<'a>>, Box<Literal<'a>>),
True,
False,
Nil,
Void,
List(Vec<Literal<'a>>),
}
impl Display for Literal<'_> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Number(n) => write!(f, "{n}"),
Self::String(s) => write!(f, "{s}"),
Self::True => write!(f, "true"),
Self::False => write!(f, "false"),
Self::Nil => write!(f, "nil"),
Self::Concat(a, b) => write!(f, "{a}{b}"),
Self::Void => write!(f, ""),
Self::List(vals) => write!(f, "{vals:?}"),
}
}
}
#[derive(Debug, PartialEq, Clone, Copy)]
pub enum UnaryOperator {
Neg,
Not,
}
#[derive(Debug, PartialEq, Clone, Copy)]
pub enum BinaryOperator {
Eq,
Neq,
Gt,
Gte,
Lt,
Lte,
Add,
Sub,
Mul,
Div,
}
#[cfg(test)]
mod tests {
use crate::{
parser::{BinaryOperator, Expr, Literal, Parser, UnaryOperator},
tokenizer::Tokenizable,
};
#[test]
fn assignment() {
let tokens = "var foo = 100;".tokenize().expect("Tokenize");
let mut parser = Parser::with_tokens(&tokens);
let ast = parser.parse().expect("Failed to parse");
assert_eq!(
ast,
Expr::Assignment {
name: "foo",
val: Box::new(Expr::Literal(Literal::Number(100.0)))
}
)
}
#[test]
fn basic_ast_generated() {
let tokens = "100 + 100;".tokenize().expect("Tokenize");
let mut parser = Parser::with_tokens(&tokens);
let ast = parser.parse().expect("Failed to parse");
assert_eq!(
ast,
Expr::Binary {
op: BinaryOperator::Add,
left: Box::new(Expr::Literal(Literal::Number(100.0))),
right: Box::new(Expr::Literal(Literal::Number(100.0)))
}
)
}
#[test]
fn while_loop_structure() {
let tokens = "while (true) {}".tokenize().expect("Tokenize");
let mut parser = Parser::with_tokens(&tokens);
let ast = parser.parse().expect("Failed to parse");
assert_eq!(
ast,
Expr::WhileLoop {
condition: Box::new(Expr::Literal(Literal::True)),
exec: Box::new(Expr::Block(vec![]))
}
)
}
#[test]
fn list_with_expressions() {
let tokens = "[false, 2 + 4];".tokenize().expect("Tokenize");
let mut parser = Parser::with_tokens(&tokens);
let ast = parser.parse().expect("Failed to parse");
assert_eq!(
ast,
Expr::List(vec![
Expr::Literal(Literal::False),
Expr::Binary {
op: BinaryOperator::Add,
left: Box::new(Expr::Literal(Literal::Number(2.0))),
right: Box::new(Expr::Literal(Literal::Number(4.0)))
}
])
);
}
#[test]
fn list() {
let tokens = "[1, 2.4];".tokenize().expect("Tokenize");
let mut parser = Parser::with_tokens(&tokens);
let ast = parser.parse().expect("Failed to parse");
assert_eq!(
ast,
Expr::List(vec![
Expr::Literal(Literal::Number(1.0)),
Expr::Literal(Literal::Number(2.4))
])
);
}
#[test]
fn simple_list_construction() {
let tokens = "[];".tokenize().expect("Tokenize");
let mut parser = Parser::with_tokens(&tokens);
let ast = parser.parse().expect("Failed to parse");
assert_eq!(ast, Expr::List(vec![]));
}
#[test]
fn test_literal_number() {
let tokens = "42;".tokenize().expect("Tokenize");
let mut parser = Parser::with_tokens(&tokens);
let ast = parser.parse().expect("Failed to parse");
assert_eq!(ast, Expr::Literal(Literal::Number(42.0)));
}
#[test]
fn test_literal_string() {
let tokens = "\"hello\";".tokenize().expect("Tokenize");
let mut parser = Parser::with_tokens(&tokens);
let ast = parser.parse().expect("Failed to parse");
assert_eq!(ast, Expr::Literal(Literal::String("hello")));
}
#[test]
fn test_literal_true() {
let tokens = "true;".tokenize().expect("Tokenize");
let mut parser = Parser::with_tokens(&tokens);
let ast = parser.parse().expect("Failed to parse");
assert_eq!(ast, Expr::Literal(Literal::True));
}
#[test]
fn empty_block() {
let tokens = "{};".tokenize().expect("Tokenize");
let mut parser = Parser::with_tokens(&tokens);
let ast = parser.parse().expect("Failed to parse");
assert_eq!(ast, Expr::Block(vec![]));
}
#[test]
fn if_block_simple() {
let tokens = r#"
if (true) {
print "Hello!";
}
"#
.tokenize()
.expect("Tokenize");
let mut parser = Parser::with_tokens(&tokens);
let ast = parser.parse().expect("Failed to parse");
assert_eq!(
ast,
Expr::Conditional {
condition: Box::new(Expr::Literal(Literal::True)),
true_branch: Box::new(Expr::Block(vec![Expr::Print(Box::new(Expr::Literal(
Literal::String("Hello!")
)))])),
else_branch: None,
}
);
}
#[test]
fn block_with_stuff() {
let tokens = r#"
{
var foo = 10;
print foo;
}"#
.tokenize()
.expect("Tokenize");
let mut parser = Parser::with_tokens(&tokens);
let ast = parser.parse().expect("Failed to parse");
assert_eq!(
ast,
Expr::Block(vec![
Expr::Assignment {
name: "foo",
val: Box::new(Expr::Literal(Literal::Number(10.0)))
},
Expr::Print(Box::new(Expr::Variable("foo")))
])
);
}
#[test]
fn test_literal_false() {
let tokens = "false;".tokenize().expect("Tokenize");
let mut parser = Parser::with_tokens(&tokens);
let ast = parser.parse().expect("Failed to parse");
assert_eq!(ast, Expr::Literal(Literal::False));
}
#[test]
fn test_literal_nil() {
let tokens = "nil;".tokenize().expect("Tokenize");
let mut parser = Parser::with_tokens(&tokens);
let ast = parser.parse().expect("Failed to parse");
assert_eq!(ast, Expr::Literal(Literal::Nil));
}
#[test]
fn test_unary_negation() {
let tokens = "-42;".tokenize().expect("Tokenize");
let mut parser = Parser::with_tokens(&tokens);
let ast = parser.parse().expect("Failed to parse");
assert_eq!(
ast,
Expr::Unary {
op: UnaryOperator::Neg,
node: Box::new(Expr::Literal(Literal::Number(42.0))),
}
);
}
#[test]
fn test_unary_not() {
let tokens = "!true;".tokenize().expect("Tokenize");
let mut parser = Parser::with_tokens(&tokens);
let ast = parser.parse().expect("Failed to parse");
assert_eq!(
ast,
Expr::Unary {
op: UnaryOperator::Not,
node: Box::new(Expr::Literal(Literal::True)),
}
);
}
#[test]
fn test_binary_addition() {
let tokens = "100 + 200;".tokenize().expect("Tokenize");
let mut parser = Parser::with_tokens(&tokens);
let ast = parser.parse().expect("Failed to parse");
assert_eq!(
ast,
Expr::Binary {
op: BinaryOperator::Add,
left: Box::new(Expr::Literal(Literal::Number(100.0))),
right: Box::new(Expr::Literal(Literal::Number(200.0))),
}
);
}
#[test]
fn test_binary_multiplication() {
let tokens = "3 * 4;".tokenize().expect("Tokenize");
let mut parser = Parser::with_tokens(&tokens);
let ast = parser.parse().expect("Failed to parse");
assert_eq!(
ast,
Expr::Binary {
op: BinaryOperator::Mul,
left: Box::new(Expr::Literal(Literal::Number(3.0))),
right: Box::new(Expr::Literal(Literal::Number(4.0))),
}
);
}
#[test]
fn test_binary_equality() {
let tokens = "true == false;".tokenize().expect("Tokenize");
let mut parser = Parser::with_tokens(&tokens);
let ast = parser.parse().expect("Failed to parse");
assert_eq!(
ast,
Expr::Binary {
op: BinaryOperator::Eq,
left: Box::new(Expr::Literal(Literal::True)),
right: Box::new(Expr::Literal(Literal::False)),
}
);
}
#[test]
fn test_binary_inequality() {
let tokens = "true != false;".tokenize().expect("Tokenize");
let mut parser = Parser::with_tokens(&tokens);
let ast = parser.parse().expect("Failed to parse");
assert_eq!(
ast,
Expr::Binary {
op: BinaryOperator::Neq,
left: Box::new(Expr::Literal(Literal::True)),
right: Box::new(Expr::Literal(Literal::False)),
}
);
}
#[test]
fn test_grouping() {
let tokens = "(42 + 10) * 2;".tokenize().expect("Tokenize");
let mut parser = Parser::with_tokens(&tokens);
let ast = parser.parse().expect("Failed to parse");
assert_eq!(
ast,
Expr::Binary {
op: BinaryOperator::Mul,
left: Box::new(Expr::Grouping(Box::new(Expr::Binary {
op: BinaryOperator::Add,
left: Box::new(Expr::Literal(Literal::Number(42.0))),
right: Box::new(Expr::Literal(Literal::Number(10.0))),
}))),
right: Box::new(Expr::Literal(Literal::Number(2.0))),
}
);
}
#[test]
fn test_complex_expression() {
let tokens = "(3 + 5) * (10 - 2) / 4;".tokenize().expect("Tokenize");
let mut parser = Parser::with_tokens(&tokens);
let ast = parser.parse().expect("Failed to parse");
assert_eq!(
ast,
Expr::Binary {
op: BinaryOperator::Div,
left: Box::new(Expr::Binary {
op: BinaryOperator::Mul,
left: Box::new(Expr::Grouping(Box::new(Expr::Binary {
op: BinaryOperator::Add,
left: Box::new(Expr::Literal(Literal::Number(3.0))),
right: Box::new(Expr::Literal(Literal::Number(5.0))),
}))),
right: Box::new(Expr::Grouping(Box::new(Expr::Binary {
op: BinaryOperator::Sub,
left: Box::new(Expr::Literal(Literal::Number(10.0))),
right: Box::new(Expr::Literal(Literal::Number(2.0))),
}))),
}),
right: Box::new(Expr::Literal(Literal::Number(4.0))),
}
);
}
#[test]
fn test_invalid_expression() {
let tokens = "42 +;".tokenize().expect("Tokenize");
let mut parser = Parser::with_tokens(&tokens);
let result = parser.parse();
assert!(result.is_err());
}
#[test]
fn test_unexpected_token() {
let tokens = "+ 42;".tokenize().expect("Tokenize");
let mut parser = Parser::with_tokens(&tokens);
let result = parser.parse();
assert!(result.is_err());
}
#[test]
fn test_empty_input() {
let tokens = "".tokenize().expect("Tokenize");
let mut parser = Parser::with_tokens(&tokens);
let result = parser.parse();
assert!(result.is_err());
}
}