use std::{
error::Error as STDError,
fmt::{Display, Formatter, Result as FMTResult},
iter::Peekable,
result::Result as STDResult
};
pub type Result<T> = STDResult<T, Error>;
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum Error {
UnexpectedCharacter(char),
IllegalEscapeCode(char),
NumberTooLarge(Box<str>)
}
impl STDError for Error {}
impl Display for Error {
fn fmt(&self, f: &mut Formatter<'_>) -> FMTResult {
match self {
Self::UnexpectedCharacter(character) =>
write!(f, "unexpected symbol {:?}", character),
Self::IllegalEscapeCode(character) =>
write!(f, "invalid escape sequence '\\{}'", character),
Self::NumberTooLarge(number) =>
write!(f, "number too large {}", number)
}
}
}
pub struct Lexer<T>
where T: Iterator<Item = char> {
pub source: Peekable<T>
}
impl<T> Lexer<T>
where T: Iterator<Item = char> {
#[cfg_attr(test, visibility::make(pub))]
fn eat(&mut self) {
self.peeked_next();
}
#[must_use = "all characters should be consumed, if you already peeked this, you should use `eat`"]
#[cfg_attr(test, visibility::make(pub))]
fn next(&mut self) -> Option<char> {
self.source.next()
}
#[cfg_attr(test, visibility::make(pub))]
fn peeked_next(&mut self) -> char {
match self.next() {
Some(next) => next,
None => unreachable!("called peeked_next when there wasn't anything next")
}
}
#[cfg_attr(test, visibility::make(pub))]
fn peek(&mut self) -> Option<char> {
self.source.peek().map(Clone::clone)
}
#[cfg_attr(test, visibility::make(pub))]
fn parse_whitespace(&mut self) -> Option<char> {
loop {
match self.peek()? {
' ' | '\n' | '\r' | '\t' => self.eat(),
character => break Some(character)
}
}
}
#[cfg_attr(test, visibility::make(pub))]
fn parse_identifier(&mut self) -> Token {
let mut identifier = String::new();
while let Some('a'..='z' | 'A'..='Z' | '0'..='9' | '_') = self.peek()
{identifier.push(self.peeked_next())}
match &identifier as &str {
"and" => Token::KeywordAnd,
"true" => Token::LiteralTrue,
"false" => Token::LiteralFalse,
"nil" => Token::LiteralNil,
"break" => Token::KeywordBreak,
"do" => Token::KeywordDo,
"else" => Token::KeywordElse,
"elseif" => Token::KeywordElseIf,
"end" => Token::KeywordEnd,
"for" => Token::KeywordFor,
"function" => Token::KeywordFunction,
"goto" => Token::KeywordGoto,
"if" => Token::KeywordIf,
"in" => Token::KeywordIn,
"local" => Token::KeywordLocal,
"not" => Token::KeywordNot,
"or" => Token::KeywordOr,
"repeat" => Token::KeywordRepeat,
"return" => Token::KeywordReturn,
"then" => Token::KeywordThen,
"until" => Token::KeywordUntil,
"while" => Token::KeywordWhile,
_ => Token::Identifier(identifier)
}
}
#[cfg_attr(test, visibility::make(pub))]
fn parse_string(&mut self) -> Option<Result<Token>> {
let delimiter = self.peeked_next();
let mut string = String::new();
loop {
match self.peek()? {
'\\' => match {self.eat(); self.peek()?} {
'a' => {self.eat(); string.push('\x07')},
'b' => {self.eat(); string.push('\x08')},
'f' => {self.eat(); string.push('\x0C')},
'n' => {self.eat(); string.push('\n')},
'r' => {self.eat(); string.push('\r')},
't' => {self.eat(); string.push('\t')},
'v' => {self.eat(); string.push('\x0B')},
'\\' => {self.eat(); string.push('\\')},
'"' => {self.eat(); string.push('"')},
'\'' => {self.eat(); string.push('\'')},
'[' => {self.eat(); string.push('[')},
']' => {self.eat(); string.push(']')},
character => break Some(Err(Error::IllegalEscapeCode(character)))
},
character if character == delimiter =>
{self.eat(); break Some(Ok(Token::String(string)))},
_ => string.push(self.peeked_next())
}
}
}
#[cfg_attr(test, visibility::make(pub))]
fn parse_bracketed_string(&mut self) -> Option<Token> {
self.parse_bracketed().map(Token::String)
}
#[cfg_attr(test, visibility::make(pub))]
fn parse_number(&mut self) -> Result<Token> {
let mut number = String::new();
while let Some('0'..='9') = self.peek()
{number.push(self.peeked_next())}
number.parse()
.map(|number| Token::Integer(number))
.map_err(|_| Error::NumberTooLarge(number.into_boxed_str()))
}
#[cfg_attr(test, visibility::make(pub))]
fn parse_comment(&mut self) -> Option<Token> {
match self.peek()? {
'[' => self.parse_bracketed().map(Token::Comment),
_ => {
let mut comment = String::new();
loop {
match self.peek()? {
'\n' => {self.eat(); break Some(Token::Comment(comment))},
_ => comment.push(self.peeked_next())
}
}
}
}
}
#[cfg_attr(test, visibility::make(pub))]
fn parse_bracketed(&mut self) -> Option<String> {
let mut string = String::new();
let length = {
let mut length = 0usize;
loop {
match self.peek()? {
'=' => {self.eat(); length += 1},
'[' => {self.eat(); break length},
_ => return None
}
}
};
let mut first = true;
loop {
match self.peek()? {
']' => {
string.push(self.peeked_next());
let mut end_length = length;
if loop {
match self.peek()? {
'=' => {
string.push(self.peeked_next());
match end_length.checked_sub(1) {
Some(new) => end_length = new,
None => break false
}
},
']' => {
string.push(self.peeked_next());
if end_length == 0 {break true}
},
_ => {
string.push(self.peeked_next());
break false
}
}
} {
string.truncate(string.len() - length - 2);
break Some(string)
}
},
'\n' if first => self.eat(),
_ => string.push(self.peeked_next())
}
first = false;
}
}
}
impl<T> Iterator for Lexer<T>
where T: Iterator<Item = char> {
type Item = Result<Token>;
fn next(&mut self) -> Option<Result<Token>> {
match self.parse_whitespace()? {
'-' => match {self.eat(); self.peek()} {
Some('-') => {self.eat(); self.parse_comment().map(Ok)},
_ => Some(Ok(Token::Minus))
},
'[' => match {self.eat(); self.peek()} {
Some('=' | '[') => self.parse_bracketed_string().map(Ok),
_ => Some(Ok(Token::OpenBracket))
},
'=' => match {self.eat(); self.peek()} {
Some('=') => {self.eat(); Some(Ok(Token::Equal))},
_ => Some(Ok(Token::Assign))
},
'<' => match {self.eat(); self.peek()} {
Some('=') => {self.eat(); Some(Ok(Token::LessThanOrEqual))},
Some('<') => {self.eat(); Some(Ok(Token::ShiftLeft))},
_ => Some(Ok(Token::LessThan))
},
'>' => match {self.eat(); self.peek()} {
Some('=') => {self.eat(); Some(Ok(Token::GreaterThanOrEqual))},
Some('>') => {self.eat(); Some(Ok(Token::ShiftRight))},
_ => Some(Ok(Token::GreaterThan))
},
'~' => match {self.eat(); self.peek()} {
Some('=') => {self.eat(); Some(Ok(Token::NotEqual))},
_ => Some(Ok(Token::BitwiseNotOrXOr))
},
'/' => match {self.eat(); self.peek()} {
Some('/') => {self.eat(); Some(Ok(Token::FloorDivide))},
_ => Some(Ok(Token::FloorDivide))
},
'.' => match {self.eat(); self.peek()} {
Some('.') => {self.eat(); Some(Ok(Token::Concat))},
_ => Some(Ok(Token::Period))
},
'+' => {self.eat(); Some(Ok(Token::Add))},
'*' => {self.eat(); Some(Ok(Token::Multiply))},
'%' => {self.eat(); Some(Ok(Token::Modulo))},
'^' => {self.eat(); Some(Ok(Token::Exponent))},
'&' => {self.eat(); Some(Ok(Token::BitwiseAnd))},
'|' => {self.eat(); Some(Ok(Token::BitwiseOr))},
':' => {self.eat(); Some(Ok(Token::Colon))},
',' => {self.eat(); Some(Ok(Token::Comma))},
';' => {self.eat(); Some(Ok(Token::SemiColon))},
'#' => {self.eat(); Some(Ok(Token::Length))},
'(' => {self.eat(); Some(Ok(Token::OpenParen))},
')' => {self.eat(); Some(Ok(Token::CloseParen))},
'{' => {self.eat(); Some(Ok(Token::OpenCurly))},
'}' => {self.eat(); Some(Ok(Token::CloseCurly))},
']' => {self.eat(); Some(Ok(Token::CloseBracket))},
'"' => self.parse_string(), '\'' => self.parse_string(), '0'..='9' => Some(self.parse_number()), 'a'..='z' | 'A'..='Z' | '_' => Some(Ok(self.parse_identifier())),
character => Some(Err(Error::UnexpectedCharacter(character)))
}
}
fn size_hint(&self) -> (usize, Option<usize>) {
(0, self.source.size_hint().1)
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum Token {
Comment(String),
Identifier(String),
Integer(i64),
String(String),
LiteralTrue,
LiteralFalse,
LiteralNil,
Add,
Minus,
Multiply,
Divide,
FloorDivide,
Modulo,
Exponent,
BitwiseAnd,
BitwiseOr,
BitwiseNotOrXOr,
ShiftLeft,
ShiftRight,
Equal,
NotEqual,
LessThan,
LessThanOrEqual,
GreaterThan,
GreaterThanOrEqual,
Assign,
Colon,
Comma,
Period,
SemiColon,
Concat,
Length,
OpenParen,
CloseParen,
OpenCurly,
CloseCurly,
OpenBracket,
CloseBracket,
KeywordAnd,
KeywordBreak,
KeywordDo,
KeywordElse,
KeywordElseIf,
KeywordEnd,
KeywordFor,
KeywordFunction,
KeywordGoto,
KeywordIf,
KeywordIn,
KeywordLocal,
KeywordNot,
KeywordOr,
KeywordRepeat,
KeywordReturn,
KeywordThen,
KeywordUntil,
KeywordWhile
}
impl std::fmt::Display for Token {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Comment(comment) => write!(f, "--[===[{}]===]", comment),
Self::Identifier(identifier) => write!(f, "{}", identifier),
Self::Integer(integer) => write!(f, "{}", integer),
Self::String(string) => write!(f, "{:?}", string),
Self::LiteralTrue => write!(f, "true"),
Self::LiteralFalse => write!(f, "false"),
Self::LiteralNil => write!(f, "nil"),
Self::Add => write!(f, "+"),
Self::Minus => write!(f, "-"),
Self::Multiply => write!(f, "*"),
Self::Divide => write!(f, "/"),
Self::FloorDivide => write!(f, "//"),
Self::Modulo => write!(f, "%"),
Self::Exponent => write!(f, "^"),
Self::BitwiseAnd => write!(f, "&"),
Self::BitwiseOr => write!(f, "|"),
Self::BitwiseNotOrXOr => write!(f, "~"),
Self::ShiftLeft => write!(f, "<<"),
Self::ShiftRight => write!(f, ">>"),
Self::Equal => write!(f, "=="),
Self::NotEqual => write!(f, "~="),
Self::LessThan => write!(f, "<"),
Self::LessThanOrEqual => write!(f, "<="),
Self::GreaterThan => write!(f, ">"),
Self::GreaterThanOrEqual => write!(f, ">="),
Self::Assign => write!(f, "="),
Self::Colon => write!(f, ":"),
Self::Comma => write!(f, ","),
Self::Period => write!(f, "."),
Self::SemiColon => write!(f, ";"),
Self::Concat => write!(f, ".."),
Self::Length => write!(f, "#"),
Self::OpenParen => write!(f, "("),
Self::CloseParen => write!(f, ")"),
Self::OpenCurly => write!(f, "{{"),
Self::CloseCurly => write!(f, "}}"),
Self::OpenBracket => write!(f, "["),
Self::CloseBracket => write!(f, "]"),
Self::KeywordAnd => write!(f, "and"),
Self::KeywordBreak => write!(f, "break"),
Self::KeywordDo => write!(f, "do"),
Self::KeywordElse => write!(f, "else"),
Self::KeywordElseIf => write!(f, "elseif"),
Self::KeywordEnd => write!(f, "end"),
Self::KeywordFor => write!(f, "for"),
Self::KeywordFunction => write!(f, "function"),
Self::KeywordGoto => write!(f, "goto"),
Self::KeywordIf => write!(f, "if"),
Self::KeywordIn => write!(f, "in"),
Self::KeywordLocal => write!(f, "local"),
Self::KeywordNot => write!(f, "not"),
Self::KeywordOr => write!(f, "or"),
Self::KeywordRepeat => write!(f, "repeat"),
Self::KeywordReturn => write!(f, "return"),
Self::KeywordThen => write!(f, "then"),
Self::KeywordUntil => write!(f, "until"),
Self::KeywordWhile => write!(f, "while")
}
}
}