use crate::{
error::{Error, Result},
span::Span,
};
#[derive(Debug, Clone, PartialEq)]
pub enum TokenKind {
Ident(String),
Number(String),
StringLit(String),
Semi,
Eq,
Slash,
Star,
Amp,
Plus,
Minus,
Dot,
LBracket,
RBracket,
LParen,
RParen,
Comma,
Question,
}
#[derive(Debug, Clone)]
pub struct Token {
pub kind: TokenKind,
pub span: Span,
}
pub struct Lexer<'a> {
input: &'a str,
i: usize,
file: Option<String>,
}
impl<'a> Lexer<'a> {
pub fn new(input: &'a str, file: Option<String>) -> Self {
Self { input, i: 0, file }
}
pub fn tokenize(mut self) -> Result<Vec<Token>> {
let mut out = Vec::new();
while self.skip_ws_and_comments() {
if self.i >= self.input.len() {
break;
}
let c = self.peek().unwrap();
let start = self.i;
let kind = match c {
';' => {
self.i += 1;
TokenKind::Semi
}
'=' => {
self.i += 1;
TokenKind::Eq
}
'/' => {
self.i += 1;
TokenKind::Slash
}
'*' => {
self.i += 1;
TokenKind::Star
}
'&' => {
self.i += 1;
TokenKind::Amp
}
'+' => {
self.i += 1;
TokenKind::Plus
}
'-' => {
self.i += 1;
TokenKind::Minus
}
'.' => {
self.i += 1;
TokenKind::Dot
}
'[' => {
self.i += 1;
TokenKind::LBracket
}
']' => {
self.i += 1;
TokenKind::RBracket
}
'(' => {
self.i += 1;
TokenKind::LParen
}
')' => {
self.i += 1;
TokenKind::RParen
}
',' => {
self.i += 1;
TokenKind::Comma
}
'?' => {
self.i += 1;
TokenKind::Question
}
'\'' => self.lex_string_lit()?,
c if is_ident_start(c) => self.lex_ident(),
c if c.is_ascii_digit() => self.lex_number(),
other => {
return Err(Error::lex(
format!("unexpected character: {other:?}"),
self.file.clone(),
Some(Span::new(start, start + 1)),
));
}
};
let end = self.i;
out.push(Token { kind, span: Span::new(start, end) });
}
Ok(out)
}
fn peek(&self) -> Option<char> {
self.input[self.i..].chars().next()
}
fn bump(&mut self) -> Option<char> {
let c = self.peek()?;
self.i += c.len_utf8();
Some(c)
}
fn skip_ws_and_comments(&mut self) -> bool {
while self.i < self.input.len() {
let rest = &self.input[self.i..];
if rest.starts_with('{') {
if let Some(end) = rest.find('}') {
self.i += end + 1;
continue;
} else {
return true;
}
}
let c = self.peek().unwrap();
if c.is_whitespace() {
self.bump();
continue;
}
break;
}
true
}
fn lex_ident(&mut self) -> TokenKind {
let mut s = String::new();
while let Some(c) = self.peek() {
if is_ident_continue(c) {
s.push(c);
self.bump();
} else {
break;
}
}
TokenKind::Ident(s)
}
fn lex_number(&mut self) -> TokenKind {
let mut s = String::new();
while let Some(c) = self.peek() {
if c.is_ascii_digit() {
s.push(c);
self.bump();
} else {
break;
}
}
TokenKind::Number(s)
}
fn lex_string_lit(&mut self) -> Result<TokenKind> {
let quote_start = self.i;
self.bump();
let mut s = String::new();
while let Some(c) = self.peek() {
if c == '\'' {
self.bump();
return Ok(TokenKind::StringLit(s));
} else {
s.push(c);
self.bump();
}
}
Err(Error::lex(
"unterminated string literal".to_string(),
self.file.clone(),
Some(Span::new(quote_start, self.i)),
))
}
}
fn is_ident_start(c: char) -> bool {
c.is_ascii_alphabetic() || c == '_'
}
fn is_ident_continue(c: char) -> bool {
c.is_ascii_alphanumeric() || c == '_' || c == '.'
}