use oxc_ast::CommentKind;
use oxc_data_structures::assert_unchecked;
use crate::{
config::{LexerConfig as Config, NoTokensLexerConfig, RuntimeLexerConfig, TokensLexerConfig},
diagnostics,
};
use super::{Kind, Lexer, SourcePosition};
impl<C: Config> Lexer<'_, C> {
#[expect(clippy::inline_always)]
#[inline(always)]
pub(super) unsafe fn handle_byte(&mut self, byte: u8) -> Kind {
if let Some(kind) = single_byte_token(byte) {
let next = unsafe { self.source.position().add(1) };
self.source.set_position(next);
return kind;
}
let byte_handlers = self.config.byte_handlers();
unsafe { byte_handlers[byte as usize](self, self.source.position()).0 }
}
}
#[expect(clippy::inline_always)]
#[inline(always)]
pub(super) fn single_byte_token(byte: u8) -> Option<Kind> {
let kind = SINGLE_BYTE_TOKENS[byte as usize];
if kind == Kind::Eof { None } else { Some(kind) }
}
static SINGLE_BYTE_TOKENS: [Kind; 256] = {
let mut table = [Kind::Eof; 256];
table[b'(' as usize] = Kind::LParen;
table[b')' as usize] = Kind::RParen;
table[b';' as usize] = Kind::Semicolon;
table[b',' as usize] = Kind::Comma;
table[b':' as usize] = Kind::Colon;
table[b'[' as usize] = Kind::LBrack;
table[b']' as usize] = Kind::RBrack;
table[b'{' as usize] = Kind::LCurly;
table[b'}' as usize] = Kind::RCurly;
table[b'~' as usize] = Kind::Tilde;
table[b'@' as usize] = Kind::At;
table[b'>' as usize] = Kind::RAngle;
table
};
pub type ByteHandler<C> =
for<'a> unsafe fn(&mut Lexer<'a, C>, SourcePosition<'a>) -> (Kind, SourcePosition<'a>);
pub type ByteHandlers<C> = [ByteHandler<C>; 256];
#[rustfmt::skip]
macro_rules! byte_handlers {
() => {
[
ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, SPS, LIN, ISP, ISP, LIN, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, SPS, EXL, QOD, HAS, IDT, PRC, AMP, QOS, UER, UER, ATR, PLS, UER, MIN, PRD, SLH, ZER, DIG, DIG, DIG, DIG, DIG, DIG, DIG, DIG, DIG, UER, UER, LSS, EQL, UER, QST, UER, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, UER, ESC, UER, CRT, IDT, TPL, KWD, KWD, KWD, KWD, KWD, KWD, KWD, IDT, KWD, IDT, KWD, KWD, KWD, KWD, KWD, KWD, IDT, KWD, KWD, KWD, KWD, KWD, KWD, IDT, KWD, IDT, UER, PIP, UER, UER, ERR, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, UER, ]
};
}
pub mod byte_handler_tables {
use super::*;
pub static NO_TOKENS: ByteHandlers<NoTokensLexerConfig> = byte_handlers!();
pub static WITH_TOKENS: ByteHandlers<TokensLexerConfig> = byte_handlers!();
pub static RUNTIME_TOKENS: ByteHandlers<RuntimeLexerConfig> = byte_handlers!();
}
macro_rules! ascii_byte_handler {
($id:ident($lex:ident) $body:expr) => {
#[expect(non_snake_case)]
fn $id<'a, C: Config>(
$lex: &mut Lexer<'a, C>,
pos: SourcePosition<'a>,
) -> (Kind, SourcePosition<'a>) {
$lex.source.set_position(pos);
unsafe {
assert_unchecked!(!$lex.source.is_eof());
assert_unchecked!($lex.source.peek_byte_unchecked() < 128);
}
let kind = $body;
(kind, $lex.source.position())
}
};
}
ascii_byte_handler!(ERR(lexer) {
let c = lexer.consume_char();
lexer.error(diagnostics::invalid_character(c, lexer.unterminated_range()));
Kind::Undetermined
});
ascii_byte_handler!(SPS(lexer) {
lexer.whitespace_handler()
});
ascii_byte_handler!(ISP(lexer) {
lexer.consume_char();
lexer.trivia_builder.add_irregular_whitespace(lexer.token_start, lexer.offset());
Kind::Skip
});
#[expect(non_snake_case)]
fn LIN<'a, C: Config>(
lexer: &mut Lexer<'a, C>,
pos: SourcePosition<'a>,
) -> (Kind, SourcePosition<'a>) {
lexer.source.set_position(pos);
unsafe {
assert_unchecked!(!lexer.source.is_eof());
assert_unchecked!(lexer.source.peek_byte_unchecked() < 128);
}
lexer.consume_char();
let byte = lexer.line_break_handler();
let pos = lexer.source.position();
lexer.token_start = lexer.source.offset_of(pos);
let Some(byte) = byte else {
return (Kind::Eof, pos);
};
if matches!(byte, b'"' | b'\'') {
return (Kind::Skip, pos);
}
if let Some(kind) = single_byte_token(byte) {
let end = unsafe { pos.add(1) };
lexer.source.set_position(end);
return (kind, end);
}
let byte_handlers = lexer.config.byte_handlers();
unsafe { byte_handlers[byte as usize](lexer, pos) }
}
ascii_byte_handler!(EXL(lexer) {
lexer.consume_char();
match lexer.peek_byte() {
Some(b'=') => {
lexer.consume_char();
if lexer.peek_byte() == Some(b'=') {
lexer.consume_char();
Kind::Neq2
} else {
Kind::Neq
}
}
_ => Kind::Bang
}
});
ascii_byte_handler!(QOD(lexer) {
unsafe { lexer.read_string_literal_double_quote() }
});
ascii_byte_handler!(QOS(lexer) {
unsafe { lexer.read_string_literal_single_quote() }
});
ascii_byte_handler!(HAS(lexer) {
lexer.consume_char();
lexer.private_identifier()
});
#[expect(non_snake_case)]
fn IDT<'a, C: Config>(
lexer: &mut Lexer<'a, C>,
pos: SourcePosition<'a>,
) -> (Kind, SourcePosition<'a>) {
unsafe { lexer.identifier_name_handler::<false>(pos) }
}
#[expect(non_snake_case)]
fn KWD<'a, C: Config>(
lexer: &mut Lexer<'a, C>,
pos: SourcePosition<'a>,
) -> (Kind, SourcePosition<'a>) {
unsafe { lexer.identifier_name_handler::<true>(pos) }
}
ascii_byte_handler!(PRC(lexer) {
lexer.consume_char();
if lexer.next_ascii_byte_eq(b'=') {
Kind::PercentEq
} else {
Kind::Percent
}
});
ascii_byte_handler!(AMP(lexer) {
lexer.consume_char();
if lexer.next_ascii_byte_eq(b'&') {
if lexer.next_ascii_byte_eq(b'=') {
Kind::Amp2Eq
} else {
Kind::Amp2
}
} else if lexer.next_ascii_byte_eq(b'=') {
Kind::AmpEq
} else {
Kind::Amp
}
});
ascii_byte_handler!(ATR(lexer) {
lexer.consume_char();
if lexer.next_ascii_byte_eq(b'*') {
if lexer.next_ascii_byte_eq(b'=') {
Kind::Star2Eq
} else {
Kind::Star2
}
} else if lexer.next_ascii_byte_eq(b'=') {
Kind::StarEq
} else {
Kind::Star
}
});
ascii_byte_handler!(PLS(lexer) {
lexer.consume_char();
if lexer.next_ascii_byte_eq(b'+') {
Kind::Plus2
} else if lexer.next_ascii_byte_eq(b'=') {
Kind::PlusEq
} else {
Kind::Plus
}
});
ascii_byte_handler!(MIN(lexer) {
lexer.consume_char();
lexer.read_minus().unwrap_or_else(|| lexer.skip_single_line_comment(CommentKind::HtmlClose))
});
ascii_byte_handler!(PRD(lexer) {
lexer.consume_char();
lexer.read_dot()
});
ascii_byte_handler!(SLH(lexer) {
lexer.consume_char();
match lexer.peek_byte() {
Some(b'/') => {
lexer.consume_char();
lexer.skip_single_line_comment(CommentKind::Line)
}
Some(b'*') => {
lexer.consume_char();
lexer.skip_multi_line_comment()
}
_ => {
if lexer.next_ascii_byte_eq(b'=') {
Kind::SlashEq
} else {
Kind::Slash
}
}
}
});
ascii_byte_handler!(ZER(lexer) {
lexer.consume_char();
lexer.read_zero()
});
ascii_byte_handler!(DIG(lexer) {
lexer.consume_char();
lexer.decimal_literal_after_first_digit()
});
ascii_byte_handler!(LSS(lexer) {
lexer.consume_char();
lexer.read_left_angle().unwrap_or_else(|| lexer.skip_single_line_comment(CommentKind::HtmlOpen))
});
ascii_byte_handler!(EQL(lexer) {
lexer.consume_char();
if lexer.next_ascii_byte_eq(b'=') {
if lexer.next_ascii_byte_eq(b'=') {
Kind::Eq3
} else {
Kind::Eq2
}
} else if lexer.next_ascii_byte_eq(b'>') {
Kind::Arrow
} else {
Kind::Eq
}
});
ascii_byte_handler!(QST(lexer) {
lexer.consume_char();
match lexer.peek_byte() {
Some(b'?') => {
lexer.consume_char();
if lexer.peek_byte() == Some(b'=') {
lexer.consume_char();
Kind::Question2Eq
} else {
Kind::Question2
}
}
Some(b'.') => {
if lexer.peek_2_bytes().is_none_or(|bytes| !bytes[1].is_ascii_digit()) {
lexer.consume_char();
Kind::QuestionDot
} else {
Kind::Question
}
}
_ => Kind::Question,
}
});
ascii_byte_handler!(ESC(lexer) {
lexer.identifier_backslash_handler()
});
ascii_byte_handler!(CRT(lexer) {
lexer.consume_char();
if lexer.next_ascii_byte_eq(b'=') {
Kind::CaretEq
} else {
Kind::Caret
}
});
ascii_byte_handler!(TPL(lexer) {
lexer.consume_char();
lexer.read_template_literal(Kind::TemplateHead, Kind::NoSubstitutionTemplate)
});
ascii_byte_handler!(PIP(lexer) {
lexer.consume_char();
match lexer.peek_byte() {
Some(b'|') => {
lexer.consume_char();
if lexer.next_ascii_byte_eq(b'=') {
Kind::Pipe2Eq
} else {
Kind::Pipe2
}
}
Some(b'=') => {
lexer.consume_char();
Kind::PipeEq
}
_ => Kind::Pipe
}
});
#[expect(non_snake_case)]
fn UNI<'a, C: Config>(
lexer: &mut Lexer<'a, C>,
_pos: SourcePosition<'a>,
) -> (Kind, SourcePosition<'a>) {
let kind = lexer.unicode_char_handler();
(kind, lexer.source.position())
}
#[expect(non_snake_case)]
fn UER<'a, C: Config>(
_lexer: &mut Lexer<'a, C>,
_pos: SourcePosition<'a>,
) -> (Kind, SourcePosition<'a>) {
unreachable!();
}