use crate::diagnostic::{Diagnostic, DiagnosticKind};
use crate::span::{BytePos, Span};
use crate::token::{BuiltinType, Keyword, Token, TokenKind};
use thiserror::Error;
#[derive(Debug, Error, Clone, PartialEq, Eq)]
pub enum LexerError {
#[error("{}", .0.message)]
Diagnostic(Diagnostic),
}
impl LexerError {
pub fn diagnostic(&self) -> &Diagnostic {
match self {
Self::Diagnostic(d) => d,
}
}
}
pub fn lex(source: &str) -> Result<Vec<Token>, LexerError> {
Lexer::new(source).tokenize_all()
}
pub struct Lexer<'src> {
src: &'src str,
bytes: &'src [u8],
pos: usize,
}
impl<'src> Lexer<'src> {
pub fn new(src: &'src str) -> Self {
Self {
src,
bytes: src.as_bytes(),
pos: 0,
}
}
pub fn tokenize_all(mut self) -> Result<Vec<Token>, LexerError> {
let mut tokens = Vec::new();
loop {
let tok = self.next_token()?;
let is_eof = tok.kind.is_eof();
tokens.push(tok);
if is_eof {
break;
}
}
Ok(tokens)
}
pub fn next_token(&mut self) -> Result<Token, LexerError> {
self.skip_trivia()?;
let start = self.pos;
let Some(ch) = self.peek_char() else {
return Ok(Token::new(
TokenKind::Eof,
Span::from_offsets(start, start),
));
};
match ch {
'{' => self.simple(TokenKind::LBrace, 1),
'}' => self.simple(TokenKind::RBrace, 1),
'(' => self.simple(TokenKind::LParen, 1),
')' => self.simple(TokenKind::RParen, 1),
'[' => self.simple(TokenKind::LBracket, 1),
']' => self.simple(TokenKind::RBracket, 1),
'|' => self.simple(TokenKind::Pipe, 1),
',' => self.simple(TokenKind::Comma, 1),
';' => self.simple(TokenKind::Semi, 1),
':' => self.simple(TokenKind::Colon, 1),
'=' => self.simple(TokenKind::Eq, 1),
'-' if self.peek_at(1) == Some('>') => self.simple(TokenKind::Arrow, 2),
'.' if self.peek_at(1) == Some('.') => self.simple(TokenKind::DotDot, 2),
'"' => self.string_literal(),
'`' => self.raw_string(),
c if c.is_ascii_digit() => self.integer(),
c if is_ident_start(c) => self.ident_or_keyword(),
_ => {
let end = start + ch.len_utf8();
Err(self.err(
start,
end,
format!("unexpected character `{ch}`"),
Some("expected a keyword, identifier, string, or punctuation"),
))
}
}
}
fn simple(&mut self, kind: TokenKind, len: usize) -> Result<Token, LexerError> {
let start = self.pos;
self.pos += len;
Ok(Token::new(kind, Span::from_offsets(start, self.pos)))
}
fn skip_trivia(&mut self) -> Result<(), LexerError> {
loop {
self.skip_whitespace();
if self.starts_with("//") {
self.skip_line_comment();
continue;
}
if self.starts_with("/*") {
self.skip_block_comment()?;
continue;
}
break;
}
Ok(())
}
fn skip_whitespace(&mut self) {
while let Some(ch) = self.peek_char() {
if ch.is_whitespace() {
self.bump();
} else {
break;
}
}
}
fn skip_line_comment(&mut self) {
debug_assert!(self.starts_with("//"));
self.pos += 2;
while let Some(ch) = self.peek_char() {
if ch == '\n' {
break;
}
self.bump();
}
}
fn skip_block_comment(&mut self) -> Result<(), LexerError> {
let start = self.pos;
debug_assert!(self.starts_with("/*"));
self.pos += 2;
while self.pos < self.bytes.len() {
if self.starts_with("*/") {
self.pos += 2;
return Ok(());
}
self.bump();
}
Err(self.err(
start,
self.pos,
"unterminated block comment",
Some("close the comment with `*/`"),
))
}
fn string_literal(&mut self) -> Result<Token, LexerError> {
let start = self.pos;
self.bump(); let mut value = String::new();
while let Some(ch) = self.peek_char() {
match ch {
'"' => {
self.bump();
return Ok(Token::new(
TokenKind::String(value),
Span::from_offsets(start, self.pos),
));
}
'\\' => {
self.bump();
let Some(esc) = self.peek_char() else {
return Err(self.err(
start,
self.pos,
"unterminated string literal",
Some("close the string with `\"`"),
));
};
let decoded = match esc {
'n' => '\n',
'r' => '\r',
't' => '\t',
'\\' => '\\',
'"' => '"',
'0' => '\0',
other => {
let esc_start = self.pos;
self.bump();
return Err(self.err(
esc_start,
self.pos,
format!("unknown escape sequence `\\{other}`"),
Some("supported escapes: \\\\ \\\" \\n \\r \\t \\0"),
));
}
};
self.bump();
value.push(decoded);
}
'\n' | '\r' => {
return Err(self.err(
start,
self.pos,
"unterminated string literal",
Some("close the string with `\"` before the newline"),
));
}
_ => {
value.push(ch);
self.bump();
}
}
}
Err(self.err(
start,
self.pos,
"unterminated string literal",
Some("close the string with `\"`"),
))
}
fn raw_string(&mut self) -> Result<Token, LexerError> {
let start = self.pos;
self.bump(); let content_start = self.pos;
while let Some(ch) = self.peek_char() {
if ch == '`' {
let value = self.src[content_start..self.pos].to_string();
self.bump();
return Ok(Token::new(
TokenKind::RawString(value),
Span::from_offsets(start, self.pos),
));
}
self.bump();
}
Err(self.err(
start,
self.pos,
"unterminated raw string",
Some("close the raw string with a backtick (`)"),
))
}
fn integer(&mut self) -> Result<Token, LexerError> {
let start = self.pos;
while matches!(self.peek_char(), Some(c) if c.is_ascii_digit()) {
self.bump();
}
let lexeme = &self.src[start..self.pos];
let value: u64 = lexeme.parse().map_err(|_| {
self.err(
start,
self.pos,
format!("integer literal `{lexeme}` is out of range"),
None,
)
})?;
Ok(Token::new(
TokenKind::Integer(value),
Span::from_offsets(start, self.pos),
))
}
fn ident_or_keyword(&mut self) -> Result<Token, LexerError> {
let start = self.pos;
self.bump();
while matches!(self.peek_char(), Some(c) if is_ident_continue(c)) {
self.bump();
}
let lexeme = &self.src[start..self.pos];
let span = Span::from_offsets(start, self.pos);
let kind = if let Some(kw) = Keyword::from_ident(lexeme) {
TokenKind::Keyword(kw)
} else if let Some(builtin) = BuiltinType::from_ident(lexeme) {
TokenKind::Builtin(builtin)
} else {
TokenKind::Ident(lexeme.to_string())
};
Ok(Token::new(kind, span))
}
fn err(
&self,
start: usize,
end: usize,
message: impl Into<String>,
help: Option<&str>,
) -> LexerError {
let mut diag = Diagnostic::error(
DiagnosticKind::LexError,
message,
Span::new(BytePos(start as u32), BytePos(end as u32)),
);
if let Some(help) = help {
diag = diag.with_help(help);
}
LexerError::Diagnostic(diag)
}
fn peek_char(&self) -> Option<char> {
self.src[self.pos..].chars().next()
}
fn peek_at(&self, offset: usize) -> Option<char> {
let idx = self.pos + offset;
if idx >= self.bytes.len() {
return None;
}
self.src[idx..].chars().next()
}
fn bump(&mut self) -> Option<char> {
let ch = self.peek_char()?;
self.pos += ch.len_utf8();
Some(ch)
}
fn starts_with(&self, s: &str) -> bool {
self.src[self.pos..].starts_with(s)
}
}
fn is_ident_start(c: char) -> bool {
c.is_ascii_alphabetic() || c == '_'
}
fn is_ident_continue(c: char) -> bool {
c.is_ascii_alphanumeric() || c == '_'
}
#[cfg(test)]
mod tests {
use super::*;
use crate::token::{BuiltinType, Keyword, TokenKind};
fn kinds(src: &str) -> Vec<TokenKind> {
lex(src)
.expect("lex ok")
.into_iter()
.map(|t| t.kind)
.collect()
}
#[test]
fn lexes_grammar_header_and_rule() {
let src = r#"
grammar Calculator;
token Num = Number;
rule expr {
term
repeat {
match "+" | "-"
term
}
}
"#;
let toks = kinds(src);
assert!(toks.starts_with(&[
TokenKind::Keyword(Keyword::Grammar),
TokenKind::Ident("Calculator".into()),
TokenKind::Semi,
TokenKind::Keyword(Keyword::Token),
TokenKind::Ident("Num".into()),
TokenKind::Eq,
TokenKind::Builtin(BuiltinType::Number),
TokenKind::Semi,
TokenKind::Keyword(Keyword::Rule),
TokenKind::Ident("expr".into()),
TokenKind::LBrace,
]));
assert_eq!(toks.last(), Some(&TokenKind::Eof));
assert!(toks.contains(&TokenKind::Keyword(Keyword::Repeat)));
assert!(toks.contains(&TokenKind::Keyword(Keyword::Match)));
assert!(toks.contains(&TokenKind::String("+".into())));
assert!(toks.contains(&TokenKind::Pipe));
}
#[test]
fn lexes_optional_repeat_bounds_and_arrow_skip() {
let src = r#"
token Ws = " " -> skip;
rule items {
optional { item }
repeat(0..) { "," item }
}
"#;
let toks = kinds(src);
assert!(toks.contains(&TokenKind::Arrow));
assert!(toks.contains(&TokenKind::Keyword(Keyword::Skip)));
assert!(toks.contains(&TokenKind::Keyword(Keyword::Optional)));
assert!(toks.contains(&TokenKind::Integer(0)));
assert!(toks.contains(&TokenKind::DotDot));
}
#[test]
fn lexes_example_block_with_raw_string() {
let src = r#"
example "object" {
input: `{"a": 1}`
expect: object
}
"#;
let toks = kinds(src);
assert!(toks.contains(&TokenKind::Keyword(Keyword::Example)));
assert!(toks.contains(&TokenKind::String("object".into())));
assert!(toks.contains(&TokenKind::Keyword(Keyword::Input)));
assert!(toks.contains(&TokenKind::RawString(r#"{"a": 1}"#.into())));
assert!(toks.contains(&TokenKind::Keyword(Keyword::Expect)));
}
#[test]
fn skips_line_and_block_comments() {
let src = r#"
// line comment
grammar /* block */ Mini;
"#;
assert_eq!(
kinds(src),
vec![
TokenKind::Keyword(Keyword::Grammar),
TokenKind::Ident("Mini".into()),
TokenKind::Semi,
TokenKind::Eof,
]
);
}
#[test]
fn string_escapes() {
let toks = kinds(r#""a\nb\t\"c""#);
assert_eq!(
toks,
vec![TokenKind::String("a\nb\t\"c".into()), TokenKind::Eof]
);
}
#[test]
fn unterminated_string_is_error() {
let err = lex(r#"rule x { "oops"#).unwrap_err();
assert!(err.diagnostic().message.contains("unterminated"));
}
#[test]
fn unknown_character_is_error() {
let err = lex("grammar @").unwrap_err();
assert!(err.diagnostic().message.contains("unexpected character"));
}
#[test]
fn tracks_spans() {
let tokens = lex("rule").unwrap();
assert_eq!(tokens[0].span, Span::from_offsets(0, 4));
}
}