sphinx/lexer/
token.rs

1use crate::language::{IntType, FloatType};
2use crate::debug::DebugSymbol;
3
4// Token Types
5
6#[derive(Clone, Debug)]
7pub enum Token {
8    // Delimiters, Separators, punctuation
9    OpenParen,
10    CloseParen,
11    OpenBrace,
12    CloseBrace,
13    OpenSquare,
14    CloseSquare,
15    Comma,
16    Colon,
17    Semicolon,
18    Ellipsis,
19    Decorator,
20    
21    // Operator Symbols
22    OpAdd, OpSub, OpMul, OpDiv, OpMod, OpExp,
23    OpInv, OpAnd, OpOr, OpXor, OpLShift, OpRShift,
24    
25    OpAddAssign, OpSubAssign, OpMulAssign, OpDivAssign, OpModAssign,
26    OpAndAssign, OpOrAssign, OpXorAssign, OpLShiftAssign, OpRShiftAssign,
27    
28    OpLT, OpLE, OpGT, OpGE, OpEQ, OpNE,
29    OpAssign, OpAccess,
30    
31    // Keywords
32    And, Or, Not,
33    True, False, Nil,
34    Let, Var, Local, NonLocal, Del,
35    If, Then, Elif, Else,
36    Begin, Loop, While, For, In, Do,
37    Continue, Break, Return,
38    Fun, Class,
39    // Self_, Super,
40    Assert,
41    End,
42    
43    // Literals
44    Identifier(String),
45    StringLiteral(String),
46    IntegerLiteral(IntType),
47    FloatLiteral(FloatType),
48    
49    // Misc
50    Label(String),
51    Comment,
52    EOF,
53}
54
55
56/// Token Output
57#[derive(Clone, Debug)]
58pub struct TokenMeta {
59    pub token: Token,
60    pub symbol: DebugSymbol,
61    pub newline: bool,  // true if this is the first token after the start of a new line
62}
sphinx/lexer/token.rs

sphinx/lexer/
token.rs
