use crate::language::{IntType, FloatType};
use crate::debug::DebugSymbol;
#[derive(Clone, Debug)]
pub enum Token {
OpenParen,
CloseParen,
OpenBrace,
CloseBrace,
OpenSquare,
CloseSquare,
Comma,
Colon,
Semicolon,
Ellipsis,
Decorator,
OpAdd, OpSub, OpMul, OpDiv, OpMod, OpExp,
OpInv, OpAnd, OpOr, OpXor, OpLShift, OpRShift,
OpAddAssign, OpSubAssign, OpMulAssign, OpDivAssign, OpModAssign,
OpAndAssign, OpOrAssign, OpXorAssign, OpLShiftAssign, OpRShiftAssign,
OpLT, OpLE, OpGT, OpGE, OpEQ, OpNE,
OpAssign, OpAccess,
And, Or, Not,
True, False, Nil,
Let, Var, Local, NonLocal, Del,
If, Then, Elif, Else,
Begin, Loop, While, For, In, Do,
Continue, Break, Return,
Fun, Class,
Assert,
End,
Identifier(String),
StringLiteral(String),
IntegerLiteral(IntType),
FloatLiteral(FloatType),
Label(String),
Comment,
EOF,
}
#[derive(Clone, Debug)]
pub struct TokenMeta {
pub token: Token,
pub symbol: DebugSymbol,
pub newline: bool, }