1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62

use crate::language::{IntType, FloatType};
use crate::debug::DebugSymbol;

// Token Types

#[derive(Clone, Debug)]
pub enum Token {
    // Delimiters, Separators, punctuation
    OpenParen,
    CloseParen,
    OpenBrace,
    CloseBrace,
    OpenSquare,
    CloseSquare,
    Comma,
    Colon,
    Semicolon,
    Ellipsis,
    Decorator,
    
    // Operator Symbols
    OpAdd, OpSub, OpMul, OpDiv, OpMod, OpExp,
    OpInv, OpAnd, OpOr, OpXor, OpLShift, OpRShift,
    
    OpAddAssign, OpSubAssign, OpMulAssign, OpDivAssign, OpModAssign,
    OpAndAssign, OpOrAssign, OpXorAssign, OpLShiftAssign, OpRShiftAssign,
    
    OpLT, OpLE, OpGT, OpGE, OpEQ, OpNE,
    OpAssign, OpAccess,
    
    // Keywords
    And, Or, Not,
    True, False, Nil,
    Let, Var, Local, NonLocal, Del,
    If, Then, Elif, Else,
    Begin, Loop, While, For, In, Do,
    Continue, Break, Return,
    Fun, Class,
    // Self_, Super,
    Echo, Assert,
    End,
    
    // Literals
    Identifier(String),
    StringLiteral(String),
    IntegerLiteral(IntType),
    FloatLiteral(FloatType),
    
    // Misc
    Label(String),
    Comment,
    EOF,
}


/// Token Output
#[derive(Clone, Debug)]
pub struct TokenMeta {
    pub token: Token,
    pub symbol: DebugSymbol,
    pub newline: bool,  // true if this is the first token after the start of a new line
}