rye_grain/
tokens.rs

1#[allow(non_camel_case_types)]
2#[derive(Debug, Copy, Clone, PartialEq, Eq)]
3pub enum TokenType {
4    ENDMARKER,
5    NAME,
6    // experimental
7    INTEGER,
8    BININT,
9    OCTINT,
10    HEXINT,
11    FLOAT,
12    IMAGINARY,
13    // experimental
14    NUMBER,
15    STRING,
16    NEWLINE,
17    INDENT,
18    DEDENT,
19    LPAR,
20    RPAR,
21    LSQB,
22    RSQB,
23    COLON,
24    COMMA,
25    SEMI,
26    PLUS,
27    MINUS,
28    STAR,
29    SLASH,
30    VBAR,
31    AMPER,
32    LESS,
33    GREATER,
34    EQUAL,
35    DOT,
36    PERCENT,
37    LBRACE,
38    RBRACE,
39    EQEQUAL,
40    NOTEQUAL,
41    LESSEQUAL,
42    GREATEREQUAL,
43    TILDE,
44    CIRCUMFLEX,
45    LEFTSHIFT,
46    RIGHTSHIFT,
47    DOUBLESTAR,
48    PLUSEQUAL,
49    MINEQUAL,
50    STAREQUAL,
51    SLASHEQUAL,
52    PERCENTEQUAL,
53    AMPEREQUAL,
54    VBAREQUAL,
55    CIRCUMFLEXEQUAL,
56    LEFTSHIFTEQUAL,
57    RIGHTSHIFTEQUAL,
58    DOUBLESTAREQUAL,
59    DOUBLESLASH,
60    DOUBLESLASHEQUAL,
61    AT,
62    ATEQUAL,
63    RARROW,
64    ELLIPSIS,
65    COLONEQUAL,
66    OP,
67    AWAIT,
68    ASYNC,
69    TYPE_IGNORE,
70    TYPE_COMMENT,
71    ERRORTOKEN,
72    COMMENT,
73    NL,
74    ENCODING,
75}
76
77#[derive(Debug, Clone, PartialEq, Eq)]
78pub struct Token {
79    pub token_type: TokenType,
80    pub exact_token_type: TokenType,
81    pub token_contents: String,
82    // these are slice indices
83    pub col_start: usize,
84    pub col_end: usize,
85}
rye_grain/tokens.rs

rye_grain/
tokens.rs
