mod comment;
mod cursor;
pub mod error;
mod identifier;
mod number;
mod operator;
mod regex;
mod spread;
mod string;
mod template;
pub mod token;
#[cfg(test)]
mod tests;
use self::{
comment::{MultiLineComment, SingleLineComment},
cursor::Cursor,
identifier::Identifier,
number::NumberLiteral,
operator::Operator,
regex::RegexLiteral,
spread::SpreadLiteral,
string::StringLiteral,
template::TemplateLiteral,
};
use crate::syntax::ast::{Punctuator, Span};
pub use crate::{profiler::BoaProfiler, syntax::ast::Position};
pub use error::Error;
use std::io::Read;
pub use token::{Token, TokenKind};
trait Tokenizer<R> {
fn lex(&mut self, cursor: &mut Cursor<R>, start_pos: Position) -> Result<Token, Error>
where
R: Read;
}
#[derive(Debug)]
pub struct Lexer<R> {
cursor: Cursor<R>,
goal_symbol: InputElement,
}
impl<R> Lexer<R> {
fn is_whitespace(ch: char) -> bool {
matches!(
ch,
'\u{0020}' | '\u{0009}' | '\u{000B}' | '\u{000C}' | '\u{00A0}' | '\u{FEFF}' |
'\u{1680}' | '\u{2000}'..='\u{200A}' | '\u{202F}' | '\u{205F}' | '\u{3000}'
)
}
#[inline]
pub(crate) fn set_goal(&mut self, elm: InputElement) {
self.goal_symbol = elm;
}
#[inline]
pub(crate) fn get_goal(&self) -> InputElement {
self.goal_symbol
}
#[inline]
pub fn new(reader: R) -> Self
where
R: Read,
{
Self {
cursor: Cursor::new(reader),
goal_symbol: Default::default(),
}
}
pub(crate) fn lex_slash_token(&mut self, start: Position) -> Result<Token, Error>
where
R: Read,
{
let _timer = BoaProfiler::global().start_event("lex_slash_token", "Lexing");
if let Some(c) = self.cursor.peek()? {
match c {
'/' => {
self.cursor.next_char()?.expect("/ token vanished");
SingleLineComment.lex(&mut self.cursor, start)
}
'*' => {
self.cursor.next_char()?.expect("* token vanished");
MultiLineComment.lex(&mut self.cursor, start)
}
ch => {
match self.get_goal() {
InputElement::Div | InputElement::TemplateTail => {
if ch == '=' {
self.cursor.next_char()?.expect("= token vanished");
Ok(Token::new(
Punctuator::AssignDiv.into(),
Span::new(start, self.cursor.pos()),
))
} else {
Ok(Token::new(
Punctuator::Div.into(),
Span::new(start, self.cursor.pos()),
))
}
}
InputElement::RegExp | InputElement::RegExpOrTemplateTail => {
RegexLiteral.lex(&mut self.cursor, start)
}
}
}
}
} else {
Err(Error::syntax(
"Abrupt end: Expecting Token /,*,= or regex",
start,
))
}
}
#[allow(clippy::should_implement_trait)]
pub fn next(&mut self) -> Result<Option<Token>, Error>
where
R: Read,
{
let _timer = BoaProfiler::global().start_event("next()", "Lexing");
let (start, next_chr) = loop {
let start = self.cursor.pos();
if let Some(next_chr) = self.cursor.next_char()? {
if !Self::is_whitespace(next_chr) {
break (start, next_chr);
}
} else {
return Ok(None);
}
};
let strict_mode = false;
let token = match next_chr {
'\r' | '\n' | '\u{2028}' | '\u{2029}' => Ok(Token::new(
TokenKind::LineTerminator,
Span::new(start, self.cursor.pos()),
)),
'"' | '\'' => StringLiteral::new(next_chr).lex(&mut self.cursor, start),
'`' => TemplateLiteral.lex(&mut self.cursor, start),
_ if next_chr.is_digit(10) => {
NumberLiteral::new(next_chr, strict_mode).lex(&mut self.cursor, start)
}
_ if next_chr.is_alphabetic() || next_chr == '$' || next_chr == '_' => {
Identifier::new(next_chr).lex(&mut self.cursor, start)
}
';' => Ok(Token::new(
Punctuator::Semicolon.into(),
Span::new(start, self.cursor.pos()),
)),
':' => Ok(Token::new(
Punctuator::Colon.into(),
Span::new(start, self.cursor.pos()),
)),
'.' => SpreadLiteral::new().lex(&mut self.cursor, start),
'(' => Ok(Token::new(
Punctuator::OpenParen.into(),
Span::new(start, self.cursor.pos()),
)),
')' => Ok(Token::new(
Punctuator::CloseParen.into(),
Span::new(start, self.cursor.pos()),
)),
',' => Ok(Token::new(
Punctuator::Comma.into(),
Span::new(start, self.cursor.pos()),
)),
'{' => Ok(Token::new(
Punctuator::OpenBlock.into(),
Span::new(start, self.cursor.pos()),
)),
'}' => Ok(Token::new(
Punctuator::CloseBlock.into(),
Span::new(start, self.cursor.pos()),
)),
'[' => Ok(Token::new(
Punctuator::OpenBracket.into(),
Span::new(start, self.cursor.pos()),
)),
']' => Ok(Token::new(
Punctuator::CloseBracket.into(),
Span::new(start, self.cursor.pos()),
)),
'?' => Ok(Token::new(
Punctuator::Question.into(),
Span::new(start, self.cursor.pos()),
)),
'/' => self.lex_slash_token(start),
'=' | '*' | '+' | '-' | '%' | '|' | '&' | '^' | '<' | '>' | '!' | '~' => {
Operator::new(next_chr).lex(&mut self.cursor, start)
}
_ => {
let details = format!(
"unexpected '{}' at line {}, column {}",
next_chr,
start.line_number(),
start.column_number()
);
Err(Error::syntax(details, start))
}
}?;
if token.kind() == &TokenKind::Comment {
self.next()
} else {
Ok(Some(token))
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum InputElement {
Div,
RegExp,
RegExpOrTemplateTail,
TemplateTail,
}
impl Default for InputElement {
fn default() -> Self {
InputElement::RegExp
}
}