1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22

#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub struct Token {
    pub len: usize,
    pub kind: TokenKind,
}

#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub enum TokenKind {
    Unknown,
    BranchKeyword,
    Comment,
    Constant,
    Delimiter,
    Identifier,
    LoopKeyword,
    OtherKeyword,
    Number,
    Punctuator,
    Typename,
    String,
    Whitespace,
}