use std::fmt;
use crate::ascii::ascii_display;
use crate::ast_names::{AstName, LexemeType};
use crate::location::Location;
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum Token<'source, 'ast> {
Eof,
Ident(AstName<'ast>),
Comment(&'source [u8]),
BlockComment(usize),
BrokenComment,
Reserved(ReservedWord),
InterpStringBegin(&'source [u8]),
InterpStringMid(&'source [u8]),
InterpStringEnd(&'source [u8]),
InterpStringSimple(&'source [u8]),
BrokenInterpDoubleBrace(&'source [u8]),
Number(&'source [u8]),
QuotedString {
value: &'source [u8],
quote_style: QuoteStyle,
},
RawString {
value: &'source [u8],
block_depth: usize,
},
BrokenString,
Error,
BrokenUnicode {
codepoint: u32,
},
Attribute(AstName<'ast>),
AttributeOpen,
Ampersand,
Bang,
Hash,
LeftBrace,
LeftBracket,
Plus,
PlusEqual,
Minus,
MinusEqual,
SkinnyArrow,
Star,
StarEqual,
Slash,
SlashEqual,
SlashSlash,
SlashSlashEqual,
Percent,
PercentEqual,
Question,
Caret,
CaretEqual,
Colon,
DoubleColon,
Comma,
Dot,
DotDot,
DotDotEqual,
Equal,
EqualEqual,
Ellipsis,
Greater,
GreaterEqual,
Less,
LessEqual,
LeftParen,
Pipe,
RightBrace,
RightBracket,
RightParen,
Semicolon,
TildeEqual,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum TokenKind {
Eof,
Ident,
Comment,
BlockComment,
BrokenComment,
Reserved(ReservedWord),
InterpStringBegin,
InterpStringMid,
InterpStringEnd,
InterpStringSimple,
BrokenInterpDoubleBrace,
Number,
QuotedString,
RawString,
BrokenString,
Error,
BrokenUnicode,
Attribute,
AttributeOpen,
Ampersand,
Bang,
Hash,
LeftBrace,
LeftBracket,
Plus,
PlusEqual,
Minus,
MinusEqual,
SkinnyArrow,
Star,
StarEqual,
Slash,
SlashEqual,
SlashSlash,
SlashSlashEqual,
Percent,
PercentEqual,
Question,
Caret,
CaretEqual,
Colon,
DoubleColon,
Comma,
Dot,
DotDot,
DotDotEqual,
Equal,
EqualEqual,
Ellipsis,
Greater,
GreaterEqual,
Less,
LessEqual,
LeftParen,
Pipe,
RightBrace,
RightBracket,
RightParen,
Semicolon,
TildeEqual,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum QuoteStyle {
Single,
Double,
}
impl QuoteStyle {
pub(super) const fn from_delimiter(delimiter: u8) -> Self {
match delimiter {
b'\'' => Self::Single,
b'"' => Self::Double,
_ => unreachable!(),
}
}
}
impl Token<'_, '_> {
#[inline(always)]
pub const fn kind(self) -> TokenKind {
match self {
Self::Eof => TokenKind::Eof,
Self::Ident(_) => TokenKind::Ident,
Self::Comment(_) => TokenKind::Comment,
Self::BlockComment(_) => TokenKind::BlockComment,
Self::BrokenComment => TokenKind::BrokenComment,
Self::Reserved(word) => TokenKind::Reserved(word),
Self::InterpStringBegin(_) => TokenKind::InterpStringBegin,
Self::InterpStringMid(_) => TokenKind::InterpStringMid,
Self::InterpStringEnd(_) => TokenKind::InterpStringEnd,
Self::InterpStringSimple(_) => TokenKind::InterpStringSimple,
Self::BrokenInterpDoubleBrace(_) => TokenKind::BrokenInterpDoubleBrace,
Self::Number(_) => TokenKind::Number,
Self::QuotedString { .. } => TokenKind::QuotedString,
Self::RawString { .. } => TokenKind::RawString,
Self::BrokenString => TokenKind::BrokenString,
Self::Error => TokenKind::Error,
Self::BrokenUnicode { .. } => TokenKind::BrokenUnicode,
Self::Attribute(_) => TokenKind::Attribute,
Self::AttributeOpen => TokenKind::AttributeOpen,
Self::Ampersand => TokenKind::Ampersand,
Self::Bang => TokenKind::Bang,
Self::Hash => TokenKind::Hash,
Self::LeftBrace => TokenKind::LeftBrace,
Self::LeftBracket => TokenKind::LeftBracket,
Self::Plus => TokenKind::Plus,
Self::PlusEqual => TokenKind::PlusEqual,
Self::Minus => TokenKind::Minus,
Self::MinusEqual => TokenKind::MinusEqual,
Self::SkinnyArrow => TokenKind::SkinnyArrow,
Self::Star => TokenKind::Star,
Self::StarEqual => TokenKind::StarEqual,
Self::Slash => TokenKind::Slash,
Self::SlashEqual => TokenKind::SlashEqual,
Self::SlashSlash => TokenKind::SlashSlash,
Self::SlashSlashEqual => TokenKind::SlashSlashEqual,
Self::Percent => TokenKind::Percent,
Self::PercentEqual => TokenKind::PercentEqual,
Self::Question => TokenKind::Question,
Self::Caret => TokenKind::Caret,
Self::CaretEqual => TokenKind::CaretEqual,
Self::Colon => TokenKind::Colon,
Self::DoubleColon => TokenKind::DoubleColon,
Self::Comma => TokenKind::Comma,
Self::Dot => TokenKind::Dot,
Self::DotDot => TokenKind::DotDot,
Self::DotDotEqual => TokenKind::DotDotEqual,
Self::Equal => TokenKind::Equal,
Self::EqualEqual => TokenKind::EqualEqual,
Self::Ellipsis => TokenKind::Ellipsis,
Self::Greater => TokenKind::Greater,
Self::GreaterEqual => TokenKind::GreaterEqual,
Self::Less => TokenKind::Less,
Self::LessEqual => TokenKind::LessEqual,
Self::LeftParen => TokenKind::LeftParen,
Self::Pipe => TokenKind::Pipe,
Self::RightBrace => TokenKind::RightBrace,
Self::RightBracket => TokenKind::RightBracket,
Self::RightParen => TokenKind::RightParen,
Self::Semicolon => TokenKind::Semicolon,
Self::TildeEqual => TokenKind::TildeEqual,
}
}
pub fn name(&self) -> &'static str {
match self {
Self::Eof => "end of input",
Self::Ident(_) => "identifier",
Self::Comment(_) => "comment",
Self::BlockComment(_) => "block comment",
Self::BrokenComment => "unfinished comment",
Self::Reserved(word) => word.as_str(),
Self::InterpStringBegin(_)
| Self::InterpStringMid(_)
| Self::InterpStringEnd(_)
| Self::InterpStringSimple(_) => "interpolated string",
Self::BrokenInterpDoubleBrace(_) => "broken interpolated string",
Self::Number(_) => "number",
Self::QuotedString { .. } => "string",
Self::RawString { .. } => "string",
Self::BrokenString => "unterminated string",
Self::Error => "invalid token",
Self::BrokenUnicode { .. } => "invalid unicode",
Self::Attribute(_) => "attribute",
Self::AttributeOpen => "@[",
Self::Ampersand => "&",
Self::Bang => "!",
Self::Hash => "#",
Self::LeftBrace => "{",
Self::LeftBracket => "[",
Self::Plus => "+",
Self::PlusEqual => "+=",
Self::Minus => "-",
Self::MinusEqual => "-=",
Self::SkinnyArrow => "->",
Self::Star => "*",
Self::StarEqual => "*=",
Self::Slash => "/",
Self::SlashEqual => "/=",
Self::SlashSlash => "//",
Self::SlashSlashEqual => "//=",
Self::Percent => "%",
Self::PercentEqual => "%=",
Self::Question => "?",
Self::Caret => "^",
Self::CaretEqual => "^=",
Self::Colon => ":",
Self::DoubleColon => "::",
Self::Comma => ",",
Self::Dot => ".",
Self::DotDot => "..",
Self::DotDotEqual => "..=",
Self::Equal => "=",
Self::EqualEqual => "==",
Self::Ellipsis => "...",
Self::Greater => ">",
Self::GreaterEqual => ">=",
Self::Less => "<",
Self::LessEqual => "<=",
Self::LeftParen => "(",
Self::Pipe => "|",
Self::RightBrace => "}",
Self::RightBracket => "]",
Self::RightParen => ")",
Self::Semicolon => ";",
Self::TildeEqual => "~=",
}
}
}
impl fmt::Display for Token<'_, '_> {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Token::Eof => formatter.write_str("<eof>"),
Token::Attribute(name) => write!(formatter, "'@{}'", ascii_display(name.bytes())),
Token::AttributeOpen => formatter.write_str("'@['"),
Token::Ident(value) => write!(formatter, "'{}'", ascii_display(value.bytes())),
Token::Number(value) => write!(formatter, "'{}'", ascii_display(value)),
Token::QuotedString { value, .. } | Token::RawString { value, .. } => {
write!(formatter, "\"{}\"", ascii_display(value))
}
Token::InterpStringBegin(_) => formatter.write_str("`{"),
Token::InterpStringMid(_) => formatter.write_str("}{"),
Token::InterpStringEnd(_) => formatter.write_str("}`"),
Token::InterpStringSimple(_) => formatter.write_str("`"),
Token::Reserved(word) => write!(formatter, "'{}'", word.as_str()),
Token::BrokenComment => formatter.write_str("unfinished comment"),
token => write!(formatter, "'{}'", token.name()),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ReservedWord {
And,
Break,
Do,
Else,
Elseif,
End,
False,
For,
Function,
If,
In,
Local,
Nil,
Not,
Or,
Repeat,
Return,
Then,
True,
Until,
While,
}
impl ReservedWord {
pub const fn as_str(self) -> &'static str {
match self {
Self::And => "and",
Self::Break => "break",
Self::Do => "do",
Self::Else => "else",
Self::Elseif => "elseif",
Self::End => "end",
Self::False => "false",
Self::For => "for",
Self::Function => "function",
Self::If => "if",
Self::In => "in",
Self::Local => "local",
Self::Nil => "nil",
Self::Not => "not",
Self::Or => "or",
Self::Repeat => "repeat",
Self::Return => "return",
Self::Then => "then",
Self::True => "true",
Self::Until => "until",
Self::While => "while",
}
}
pub(super) const fn from_lexeme_type(kind: LexemeType) -> Option<Self> {
Some(match kind {
LexemeType::ReservedAnd => Self::And,
LexemeType::ReservedBreak => Self::Break,
LexemeType::ReservedDo => Self::Do,
LexemeType::ReservedElse => Self::Else,
LexemeType::ReservedElseif => Self::Elseif,
LexemeType::ReservedEnd => Self::End,
LexemeType::ReservedFalse => Self::False,
LexemeType::ReservedFor => Self::For,
LexemeType::ReservedFunction => Self::Function,
LexemeType::ReservedIf => Self::If,
LexemeType::ReservedIn => Self::In,
LexemeType::ReservedLocal => Self::Local,
LexemeType::ReservedNil => Self::Nil,
LexemeType::ReservedNot => Self::Not,
LexemeType::ReservedOr => Self::Or,
LexemeType::ReservedRepeat => Self::Repeat,
LexemeType::ReservedReturn => Self::Return,
LexemeType::ReservedThen => Self::Then,
LexemeType::ReservedTrue => Self::True,
LexemeType::ReservedUntil => Self::Until,
LexemeType::ReservedWhile => Self::While,
_ => return None,
})
}
}
#[derive(Clone, Copy)]
pub struct LexedToken<'source, 'ast> {
pub token: Token<'source, 'ast>,
pub start: usize,
pub end: usize,
pub location: Location,
}