use crate::{
Result,
diagnostic::{AeriError, Span},
};
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Token {
pub kind: TokenKind,
pub span: Span,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum TokenKind {
Identifier(String),
Integer(i64),
String(String),
ByteArray(String),
Module,
Type,
Const,
Test,
Validator,
Fn,
Let,
Require,
Trace,
Return,
If,
Else,
Match,
Fail,
True,
False,
LParen,
RParen,
LBrace,
RBrace,
LBracket,
RBracket,
Comma,
Colon,
Semicolon,
Arrow,
FatArrow,
Equal,
EqualEqual,
Bang,
BangEqual,
Less,
LessEqual,
Greater,
GreaterEqual,
AmpAmp,
PipePipe,
Plus,
Minus,
Star,
Slash,
Percent,
Eof,
}
pub fn lex(file: &str, source: &str) -> Result<Vec<Token>> {
Lexer::new(file, source).tokens()
}
struct Lexer<'a> {
file: &'a str,
source: &'a str,
offset: usize,
line: usize,
column: usize,
}
impl<'a> Lexer<'a> {
fn new(file: &'a str, source: &'a str) -> Self {
Self {
file,
source,
offset: 0,
line: 1,
column: 1,
}
}
fn tokens(mut self) -> Result<Vec<Token>> {
let mut tokens = Vec::new();
while !self.is_at_end() {
self.skip_whitespace_and_comments()?;
if self.is_at_end() {
break;
}
tokens.push(self.token()?);
}
tokens.push(Token {
kind: TokenKind::Eof,
span: self.span_at_current(),
});
Ok(tokens)
}
fn token(&mut self) -> Result<Token> {
let start = self.offset;
let line = self.line;
let column = self.column;
let ch = self.advance().expect("lexer advanced past end");
let kind = match ch {
'(' => TokenKind::LParen,
')' => TokenKind::RParen,
'{' => TokenKind::LBrace,
'}' => TokenKind::RBrace,
'[' => TokenKind::LBracket,
']' => TokenKind::RBracket,
',' => TokenKind::Comma,
':' => TokenKind::Colon,
';' => TokenKind::Semicolon,
'+' => TokenKind::Plus,
'*' => TokenKind::Star,
'%' => TokenKind::Percent,
'-' => {
if self.match_char('>') {
TokenKind::Arrow
} else {
TokenKind::Minus
}
}
'/' => TokenKind::Slash,
'=' => {
if self.match_char('>') {
TokenKind::FatArrow
} else if self.match_char('=') {
TokenKind::EqualEqual
} else {
TokenKind::Equal
}
}
'!' => {
if self.match_char('=') {
TokenKind::BangEqual
} else {
TokenKind::Bang
}
}
'<' => {
if self.match_char('=') {
TokenKind::LessEqual
} else {
TokenKind::Less
}
}
'>' => {
if self.match_char('=') {
TokenKind::GreaterEqual
} else {
TokenKind::Greater
}
}
'&' => {
if self.match_char('&') {
TokenKind::AmpAmp
} else {
return Err(self.error_at(line, column, "expected '&' after '&'"));
}
}
'|' => {
if self.match_char('|') {
TokenKind::PipePipe
} else {
return Err(self.error_at(line, column, "expected '|' after '|'"));
}
}
'"' => return self.string(start, line, column),
'#' => return self.byte_array(start, line, column),
ch if ch.is_ascii_digit() => return self.integer(start, line, column),
ch if is_identifier_start(ch) => return Ok(self.identifier(start, line, column)),
_ => return Err(self.error_at(line, column, format!("unexpected character '{ch}'"))),
};
Ok(Token {
kind,
span: Span::new(start, self.offset, line, column),
})
}
fn identifier(&mut self, start: usize, line: usize, column: usize) -> Token {
while matches!(self.peek(), Some(ch) if is_identifier_continue(ch)) {
self.advance();
}
let text = &self.source[start..self.offset];
let kind = match text {
"module" => TokenKind::Module,
"type" => TokenKind::Type,
"const" => TokenKind::Const,
"test" => TokenKind::Test,
"validator" => TokenKind::Validator,
"fn" => TokenKind::Fn,
"let" => TokenKind::Let,
"require" => TokenKind::Require,
"trace" => TokenKind::Trace,
"return" => TokenKind::Return,
"if" => TokenKind::If,
"else" => TokenKind::Else,
"match" => TokenKind::Match,
"fail" => TokenKind::Fail,
"true" => TokenKind::True,
"false" => TokenKind::False,
_ => TokenKind::Identifier(text.to_string()),
};
Token {
kind,
span: Span::new(start, self.offset, line, column),
}
}
fn integer(&mut self, start: usize, line: usize, column: usize) -> Result<Token> {
while matches!(self.peek(), Some(ch) if ch.is_ascii_digit() || ch == '_') {
self.advance();
}
let raw = self.source[start..self.offset].replace('_', "");
let value = raw
.parse::<i64>()
.map_err(|_| self.error_at(line, column, "integer literal is too large"))?;
Ok(Token {
kind: TokenKind::Integer(value),
span: Span::new(start, self.offset, line, column),
})
}
fn string(&mut self, start: usize, line: usize, column: usize) -> Result<Token> {
let mut value = String::new();
while let Some(ch) = self.peek() {
match ch {
'"' => {
self.advance();
return Ok(Token {
kind: TokenKind::String(value),
span: Span::new(start, self.offset, line, column),
});
}
'\\' => {
self.advance();
let escaped = self.advance().ok_or_else(|| {
self.error_at(line, column, "unterminated string escape sequence")
})?;
match escaped {
'n' => value.push('\n'),
'r' => value.push('\r'),
't' => value.push('\t'),
'\\' => value.push('\\'),
'"' => value.push('"'),
_ => {
return Err(self.error_at(
line,
column,
format!("unsupported string escape '\\{escaped}'"),
));
}
}
}
'\n' => return Err(self.error_at(line, column, "unterminated string literal")),
_ => {
value.push(ch);
self.advance();
}
}
}
Err(self.error_at(line, column, "unterminated string literal"))
}
fn byte_array(&mut self, start: usize, line: usize, column: usize) -> Result<Token> {
while matches!(self.peek(), Some(ch) if ch.is_ascii_hexdigit()) {
self.advance();
}
let hex = &self.source[start + 1..self.offset];
if hex.is_empty() {
return Err(self.error_at(
line,
column,
"byte array literal needs hex digits after '#'",
));
}
if !hex.len().is_multiple_of(2) {
return Err(self.error_at(
line,
column,
"byte array literal must have an even number of hex digits",
));
}
Ok(Token {
kind: TokenKind::ByteArray(hex.to_ascii_lowercase()),
span: Span::new(start, self.offset, line, column),
})
}
fn skip_whitespace_and_comments(&mut self) -> Result<()> {
loop {
match self.peek() {
Some(ch) if ch.is_whitespace() => {
self.advance();
}
Some('/') if self.peek_next() == Some('/') => {
while !matches!(self.peek(), None | Some('\n')) {
self.advance();
}
}
Some('/') if self.peek_next() == Some('*') => {
let line = self.line;
let column = self.column;
let mut closed = false;
self.advance();
self.advance();
while !self.is_at_end() {
if self.peek() == Some('*') && self.peek_next() == Some('/') {
self.advance();
self.advance();
closed = true;
break;
}
self.advance();
}
if !closed {
return Err(self.error_at(line, column, "unterminated block comment"));
}
}
_ => break,
}
}
Ok(())
}
fn span_at_current(&self) -> Span {
Span::new(self.offset, self.offset, self.line, self.column)
}
fn error_at(&self, line: usize, column: usize, message: impl Into<String>) -> AeriError {
AeriError::at(self.file, line, column, message)
}
fn match_char(&mut self, expected: char) -> bool {
if self.peek() == Some(expected) {
self.advance();
true
} else {
false
}
}
fn advance(&mut self) -> Option<char> {
let ch = self.peek()?;
self.offset += ch.len_utf8();
if ch == '\n' {
self.line += 1;
self.column = 1;
} else {
self.column += 1;
}
Some(ch)
}
fn peek(&self) -> Option<char> {
self.source[self.offset..].chars().next()
}
fn peek_next(&self) -> Option<char> {
let mut chars = self.source[self.offset..].chars();
chars.next()?;
chars.next()
}
fn is_at_end(&self) -> bool {
self.offset >= self.source.len()
}
}
fn is_identifier_start(ch: char) -> bool {
ch.is_ascii_alphabetic() || ch == '_'
}
fn is_identifier_continue(ch: char) -> bool {
is_identifier_start(ch) || ch.is_ascii_digit()
}