use std::fmt;
use std::hash::{Hash, Hasher};
use erg_common::error::Location;
use erg_common::impl_displayable_stream_for_wrapper;
use erg_common::str::Str;
use erg_common::traits::{Locational, Stream};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
#[repr(u8)]
pub enum TokenKind {
Symbol,
NatLit,
IntLit,
RatioLit,
BoolLit,
StrLit,
NoneLit,
NoImplLit,
EllipsisLit,
InfLit,
PrePlus,
PreMinus,
PreBitNot,
Mutate,
Try,
Plus,
Minus,
Star,
Slash,
FloorDiv,
Pow,
Mod,
Closed,
RightOpen,
LeftOpen,
Open,
BitAnd,
BitOr,
BitXor,
Shl,
Shr,
Less,
Gre,
LessEq,
GreEq,
DblEq,
NotEq,
InOp,
NotInOp,
SubOp,
IsOp,
IsNotOp,
AndOp,
OrOp,
DotOp,
CrossOp,
RefOp,
RefMutOp,
Equal,
Walrus,
FuncArrow,
ProcArrow,
LParen,
RParen,
LSqBr,
RSqBr,
LBrace,
RBrace,
Indent,
Dedent,
Dot,
Pipe,
Colon,
DblColon,
SupertypeOf,
SubtypeOf,
Comma,
Caret,
Amper,
AtSign,
VBar,
UBar,
Spread,
Newline,
Semi,
Illegal,
BOF,
EOF,
}
use TokenKind::*;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum TokenCategory {
Symbol,
Literal,
BinOp,
UnaryOp,
PostfixOp,
LEnclosure,
REnclosure,
SpecialBinOp,
DefOp,
LambdaOp,
Separator,
Reserved,
AtSign,
VBar,
UBar,
BOF,
EOF,
Illegal,
}
impl TokenCategory {
pub const fn is_block_op(&self) -> bool {
matches!(self, Self::DefOp | Self::LambdaOp)
}
}
impl TokenKind {
pub const fn category(&self) -> TokenCategory {
match self {
Symbol => TokenCategory::Symbol,
NatLit | IntLit | RatioLit | StrLit | BoolLit | NoneLit | EllipsisLit | NoImplLit
| InfLit => TokenCategory::Literal,
PrePlus | PreMinus | PreBitNot | Mutate | RefOp | RefMutOp => TokenCategory::UnaryOp,
Try => TokenCategory::PostfixOp,
Comma | Colon | DblColon | SupertypeOf | SubtypeOf | Dot | Pipe | Walrus => {
TokenCategory::SpecialBinOp
}
Equal => TokenCategory::DefOp,
FuncArrow | ProcArrow => TokenCategory::LambdaOp,
Semi | Newline => TokenCategory::Separator,
LParen | LBrace | LSqBr | Indent => TokenCategory::LEnclosure,
RParen | RBrace | RSqBr | Dedent => TokenCategory::REnclosure,
Caret | Amper => TokenCategory::Reserved,
AtSign => TokenCategory::AtSign,
VBar => TokenCategory::VBar,
UBar => TokenCategory::UBar,
BOF => TokenCategory::BOF,
EOF => TokenCategory::EOF,
Illegal => TokenCategory::Illegal,
_ => TokenCategory::BinOp,
}
}
pub const fn precedence(&self) -> Option<usize> {
let prec = match self {
Dot | DblColon => 200, Pow => 190, PrePlus | PreMinus | PreBitNot | RefOp | RefMutOp => 180, Star | Slash | FloorDiv | Mod | CrossOp | DotOp => 170, Plus | Minus => 160, Shl | Shr => 150, BitAnd => 140, BitXor => 130, BitOr => 120, Closed | LeftOpen | RightOpen | Open => 100, Less | Gre | LessEq | GreEq | DblEq | NotEq | InOp | NotInOp | IsOp | IsNotOp => 90, AndOp => 80, OrOp => 70, FuncArrow | ProcArrow => 60, Colon | SupertypeOf | SubtypeOf => 50, Comma => 40, Equal | Walrus => 20, Newline | Semi => 10, LParen | LBrace | LSqBr | Indent => 0, _ => return None,
};
Some(prec)
}
pub const fn is_right_associative(&self) -> bool {
matches!(
self,
FuncArrow | ProcArrow | Equal
)
}
}
impl fmt::Display for TokenKind {
#[inline]
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{self:?}")
}
}
#[derive(Clone, Eq)]
pub struct Token {
pub kind: TokenKind,
pub content: Str,
pub lineno: usize,
pub col_begin: usize,
}
impl fmt::Debug for Token {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Token")
.field("kind", &self.kind)
.field("content", &self.content.replace('\n', "\\n"))
.field("lineno", &self.lineno)
.field("col_begin", &self.col_begin)
.finish()
}
}
impl fmt::Display for Token {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{:?} {}", self.kind, self.content.replace('\n', "\\n"))
}
}
impl PartialEq for Token {
#[inline]
fn eq(&self, other: &Self) -> bool {
self.is(other.kind) && self.content == other.content
}
}
impl Hash for Token {
fn hash<H: Hasher>(&self, state: &mut H) {
self.kind.hash(state);
self.content.hash(state);
}
}
impl Locational for Token {
fn loc(&self) -> Location {
if self.lineno == 0 {
Location::Unknown
} else {
Location::range(
self.lineno,
self.col_begin,
self.lineno,
self.col_begin + self.content.len(),
)
}
}
#[inline]
fn col_end(&self) -> Option<usize> {
Some(self.col_begin + self.content.len())
}
}
impl Token {
#[inline]
pub fn dummy() -> Self {
Token {
kind: TokenKind::Illegal,
content: "DUMMY".into(),
lineno: 1,
col_begin: 0,
}
}
#[inline]
pub fn new<S: Into<Str>>(kind: TokenKind, cont: S, lineno: usize, col_begin: usize) -> Self {
Token {
kind,
content: cont.into(),
lineno,
col_begin,
}
}
#[inline]
pub fn from_str(kind: TokenKind, cont: &str) -> Self {
Token {
kind,
content: Str::rc(cont),
lineno: 0,
col_begin: 0,
}
}
#[inline]
pub fn symbol(cont: &str) -> Self {
Self::from_str(TokenKind::Symbol, cont)
}
#[inline]
pub fn symbol_with_line(cont: &str, line: usize) -> Self {
Token {
kind: TokenKind::Symbol,
content: Str::rc(cont),
lineno: line,
col_begin: 0,
}
}
pub const fn static_symbol(s: &'static str) -> Self {
Token {
kind: TokenKind::Symbol,
content: Str::ever(s),
lineno: 0,
col_begin: 0,
}
}
pub const fn category(&self) -> TokenCategory {
self.kind.category()
}
pub fn category_is(&self, category: TokenCategory) -> bool {
self.kind.category() == category
}
pub fn is(&self, kind: TokenKind) -> bool {
self.kind == kind
}
pub const fn is_block_op(&self) -> bool {
self.category().is_block_op()
}
pub const fn inspect(&self) -> &Str {
&self.content
}
pub fn is_procedural(&self) -> bool {
self.inspect().ends_with('!')
}
}
#[derive(Debug, Clone)]
pub struct TokenStream(Vec<Token>);
impl_displayable_stream_for_wrapper!(TokenStream, Token);