mod cursor;
pub mod span;
use cursor::Cursor;
use span::FilePosition;
pub use span::Span;
pub mod token;
use token::{Token, TokenKind};
use crate::Result;
pub fn tokenize(src: &str) -> Result<Vec<Token>> {
Lexer {
c: Cursor::new(src)
}.tokenize()
}
struct Lexer<'a> {
c: Cursor<'a>,
}
impl Lexer<'_> {
fn tokenize(&mut self) -> Result<Vec<Token>> {
let mut tokens: Vec<Token> = Vec::new();
while !self.c.is_finished() {
self.c.step();
if let Some(t) = self.next_token()? {
tokens.push(t);
}
}
Ok(tokens)
}
#[allow(clippy::unnecessary_wraps)]
fn add_token(&self, token_type: TokenKind) -> Result<Option<Token>> {
Ok(Some(Token::new(token_type, self.c.get_span())))
}
fn next_token(&mut self) -> Result<Option<Token>> {
let c = self.c.advance();
match c {
'*' => self.add_token(TokenKind::Mul),
'-' => self.add_token(TokenKind::Minus),
'+' => self.add_token(TokenKind::Plus),
'/' => self.add_token(TokenKind::Slash),
'%' => self.add_token(TokenKind::Mod),
'(' => self.add_token(TokenKind::LParen),
')' => self.add_token(TokenKind::RParen),
' ' | '\n' | '\r' | '\t' => Ok(None), c => {
if c.is_numeric() {
self.number()
} else {
self.error(&format!("Unexpected character '{c}'"))?;
Ok(None)
}
}
}
}
fn number(&mut self) -> Result<Option<Token>> {
self.c.advance_while(|c| c.is_numeric());
if self.c.peek() == '.' && self.c.peek_next().is_numeric() {
self.c.advance();
self.c.advance_while(|c| c.is_numeric());
}
self.add_token(TokenKind::Number)
}
fn error(&mut self, msg: &str) -> Result<Option<Token>> {
let FilePosition {
start_line,
start_col,
..
} = self.c.file_pos();
let msg = format!("[{start_line}:{start_col}] {msg}");
Err(msg.into())
}
}