1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90

use crate::interner::InternedStr;
use crate::span::SpanId;

#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
pub struct Token {
    pub span: SpanId,
    pub kind: TokenKind,
}

#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
pub enum TokenKind {
    EndOfFile,
    Whitespace,
    Comment,

    Simple(STokenKind),
    OtherOp(Box<str>),
    Ident(InternedStr),
    Number(Number),
    String(Box<str>),
    TextBlock(Box<str>),
}

#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
pub enum STokenKind {
    // Keywords
    Assert,
    Else,
    Error,
    False,
    For,
    Function,
    If,
    Import,
    Importstr,
    Importbin,
    In,
    Local,
    Null,
    Tailstrict,
    Then,
    Self_,
    Super,
    True,

    // Symbols
    Exclam,
    ExclamEq,
    Dollar,
    Percent,
    Amp,
    AmpAmp,
    LeftParen,
    RightParen,
    Asterisk,
    Plus,
    PlusColon,
    PlusColonColon,
    PlusColonColonColon,
    Comma,
    Minus,
    Dot,
    Slash,
    Colon,
    ColonColon,
    ColonColonColon,
    Semicolon,
    Lt,
    LtLt,
    LtEq,
    Eq,
    EqEq,
    Gt,
    GtEq,
    GtGt,
    LeftBracket,
    RightBracket,
    Hat,
    LeftBrace,
    Pipe,
    PipePipe,
    RightBrace,
    Tilde,
}

#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
pub struct Number {
    pub digits: Box<str>,
    pub exp: i64,
}