use super::{ReservedWord, TemplatePart};
use crate::span::Span;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Token {
pub kind: TokenKind,
pub span: Span,
pub newline_before: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum TokenKind {
Eof,
Identifier {
contains_escape: bool,
},
PrivateIdentifier {
contains_escape: bool,
},
Keyword(ReservedWord),
Number {
legacy: bool,
},
String {
legacy_escape: bool,
},
Template {
part: TemplatePart,
cooked_undefined: bool,
},
RegExp,
BigInt,
LBrace,
RBrace,
LParen,
RParen,
LBracket,
RBracket,
Dot,
DotDotDot,
Semicolon,
Comma,
Colon,
Arrow,
QuestionDot,
Question,
Lt,
Gt,
LtEq,
GtEq,
EqEq,
BangEq,
EqEqEq,
BangEqEq,
Plus,
Minus,
Star,
Slash,
Percent,
StarStar,
PlusPlus,
MinusMinus,
LtLt,
GtGt,
GtGtGt,
Amp,
Pipe,
Caret,
Bang,
Tilde,
AmpAmp,
PipePipe,
QuestionQuestion,
Eq,
PlusEq,
MinusEq,
StarEq,
SlashEq,
PercentEq,
StarStarEq,
LtLtEq,
GtGtEq,
GtGtGtEq,
AmpEq,
PipeEq,
CaretEq,
AmpAmpEq,
PipePipeEq,
QuestionQuestionEq,
}
impl TokenKind {
pub fn as_str(&self) -> Option<&'static str> {
Some(match self {
Self::Eof => "",
Self::Identifier { .. }
| Self::PrivateIdentifier { .. }
| Self::Number { .. }
| Self::BigInt
| Self::String { .. }
| Self::RegExp
| Self::Template { .. } => return None,
Self::Keyword(word) => word.as_str(),
Self::LBrace => "{",
Self::RBrace => "}",
Self::LParen => "(",
Self::RParen => ")",
Self::LBracket => "[",
Self::RBracket => "]",
Self::Dot => ".",
Self::DotDotDot => "...",
Self::Semicolon => ";",
Self::Comma => ",",
Self::Colon => ":",
Self::Arrow => "=>",
Self::QuestionDot => "?.",
Self::Question => "?",
Self::Lt => "<",
Self::Gt => ">",
Self::LtEq => "<=",
Self::GtEq => ">=",
Self::EqEq => "==",
Self::BangEq => "!=",
Self::EqEqEq => "===",
Self::BangEqEq => "!==",
Self::Plus => "+",
Self::Minus => "-",
Self::Star => "*",
Self::Slash => "/",
Self::Percent => "%",
Self::StarStar => "**",
Self::PlusPlus => "++",
Self::MinusMinus => "--",
Self::LtLt => "<<",
Self::GtGt => ">>",
Self::GtGtGt => ">>>",
Self::Amp => "&",
Self::Pipe => "|",
Self::Caret => "^",
Self::Bang => "!",
Self::Tilde => "~",
Self::AmpAmp => "&&",
Self::PipePipe => "||",
Self::QuestionQuestion => "??",
Self::Eq => "=",
Self::PlusEq => "+=",
Self::MinusEq => "-=",
Self::StarEq => "*=",
Self::SlashEq => "/=",
Self::PercentEq => "%=",
Self::StarStarEq => "**=",
Self::LtLtEq => "<<=",
Self::GtGtEq => ">>=",
Self::GtGtGtEq => ">>>=",
Self::AmpEq => "&=",
Self::PipeEq => "|=",
Self::CaretEq => "^=",
Self::AmpAmpEq => "&&=",
Self::PipePipeEq => "||=",
Self::QuestionQuestionEq => "??=",
})
}
}
pub(super) const PUNCTUATORS: &[(&str, TokenKind)] = &[
(">>>=", TokenKind::GtGtGtEq),
("...", TokenKind::DotDotDot),
("===", TokenKind::EqEqEq),
("!==", TokenKind::BangEqEq),
("**=", TokenKind::StarStarEq),
("<<=", TokenKind::LtLtEq),
(">>=", TokenKind::GtGtEq),
(">>>", TokenKind::GtGtGt),
("&&=", TokenKind::AmpAmpEq),
("||=", TokenKind::PipePipeEq),
("??=", TokenKind::QuestionQuestionEq),
("=>", TokenKind::Arrow),
("==", TokenKind::EqEq),
("!=", TokenKind::BangEq),
("<=", TokenKind::LtEq),
(">=", TokenKind::GtEq),
("+=", TokenKind::PlusEq),
("-=", TokenKind::MinusEq),
("*=", TokenKind::StarEq),
("/=", TokenKind::SlashEq),
("%=", TokenKind::PercentEq),
("&=", TokenKind::AmpEq),
("|=", TokenKind::PipeEq),
("^=", TokenKind::CaretEq),
("**", TokenKind::StarStar),
("++", TokenKind::PlusPlus),
("--", TokenKind::MinusMinus),
("<<", TokenKind::LtLt),
(">>", TokenKind::GtGt),
("&&", TokenKind::AmpAmp),
("||", TokenKind::PipePipe),
("??", TokenKind::QuestionQuestion),
("?.", TokenKind::QuestionDot),
("{", TokenKind::LBrace),
("}", TokenKind::RBrace),
("(", TokenKind::LParen),
(")", TokenKind::RParen),
("[", TokenKind::LBracket),
("]", TokenKind::RBracket),
(".", TokenKind::Dot),
(";", TokenKind::Semicolon),
(",", TokenKind::Comma),
(":", TokenKind::Colon),
("?", TokenKind::Question),
("<", TokenKind::Lt),
(">", TokenKind::Gt),
("+", TokenKind::Plus),
("-", TokenKind::Minus),
("*", TokenKind::Star),
("/", TokenKind::Slash),
("%", TokenKind::Percent),
("&", TokenKind::Amp),
("|", TokenKind::Pipe),
("^", TokenKind::Caret),
("!", TokenKind::Bang),
("~", TokenKind::Tilde),
("=", TokenKind::Eq),
];
#[cfg(test)]
mod tests {
use super::*;
use crate::lexer::test_support::*;
#[test]
fn every_punctuator_lexes_as_itself_and_the_table_is_ordered_longest_first() {
for pair in PUNCTUATORS.windows(2) {
let [(before, _), (after, _)] = pair else {
continue;
};
assert!(
before.len() >= after.len(),
"{before:?} must not precede the longer {after:?}"
);
}
let mut seen = std::collections::HashSet::new();
for &(text, kind) in PUNCTUATORS {
assert_eq!(
kind.as_str(),
Some(text),
"table row {text:?} disagrees with as_str"
);
assert!(seen.insert(kind), "{text:?} appears twice in the table");
if kind != TokenKind::QuestionDot {
assert_eq!(kinds(text), [kind, TokenKind::Eof], "lexing {text:?}");
}
}
assert_eq!(seen.len(), 57, "ECMA-262 §12.8 has 57 punctuators");
assert_eq!(TokenKind::Eof.as_str(), Some(""));
assert_eq!(
TokenKind::Identifier {
contains_escape: false
}
.as_str(),
None
);
assert_eq!(
TokenKind::PrivateIdentifier {
contains_escape: false
}
.as_str(),
None
);
}
}