arrow_parser/
token.rs

1use super::error::SyntaxError;
2use super::error::SyntaxErrorType;
3use super::source::SourceRange;
4
5#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)]
6#[repr(u8)]
7pub enum TokenType {
8  // Special token used to represent the end of the source code. Easier than using and handling Option everywhere.
9  EOF,
10
11  // Only used during lexing.
12  _LiteralNumberBin,
13  _LiteralNumberDec,
14  _LiteralNumberHex,
15  _LiteralNumberOct,
16
17  Ampersand,
18  AmpersandAmpersand,
19  AmpersandAmpersandEquals,
20  AmpersandEquals,
21  Asterisk,
22  AsteriskAsterisk,
23  AsteriskAsteriskEquals,
24  AsteriskEquals,
25  Bar,
26  BarBar,
27  BarBarEquals,
28  BarEquals,
29  BraceClose,
30  BraceOpen,
31  BracketClose,
32  BracketOpen,
33  Caret,
34  CaretEquals,
35  ChevronLeft,
36  ChevronLeftChevronLeft,
37  ChevronLeftChevronLeftEquals,
38  ChevronLeftEquals,
39  ChevronLeftSlash,
40  ChevronRight,
41  ChevronRightChevronRight,
42  ChevronRightChevronRightChevronRight,
43  ChevronRightChevronRightChevronRightEquals,
44  ChevronRightChevronRightEquals,
45  ChevronRightEquals,
46  Colon,
47  ColonColon,
48  Comma,
49  CommentMultiple,
50  CommentSingle,
51  Dot,
52  DotDotDot,
53  Equals,
54  EqualsEquals,
55  Exclamation,
56  ExclamationEquals,
57  Hyphen,
58  HyphenChevronRight,
59  HyphenEquals,
60  Identifier,
61  KeywordAs,
62  KeywordBreak,
63  KeywordContinue,
64  KeywordCrate,
65  KeywordElse,
66  KeywordFor,
67  KeywordIf,
68  KeywordImpl,
69  KeywordIn,
70  KeywordLet,
71  KeywordLoop,
72  KeywordMut,
73  KeywordPub,
74  KeywordReturn,
75  KeywordStruct,
76  LiteralBigInt,
77  LiteralDecimal,
78  LiteralFalse,
79  LiteralFloat,
80  LiteralInt,
81  LiteralNone,
82  LiteralTemplatePartString,
83  LiteralTemplatePartStringEnd,
84  LiteralTrue,
85  ParenthesisClose,
86  ParenthesisOpen,
87  Percent,
88  PercentEquals,
89  Plus,
90  PlusEquals,
91  QuestionBracketOpen,
92  QuestionDot,
93  QuestionParenthesisOpen,
94  QuestionQuestion,
95  QuestionQuestionEquals,
96  Semicolon,
97  Slash,
98  SlashEquals,
99}
100
101#[derive(Clone, Copy)]
102pub struct TokenTypeSet(u128);
103
104impl TokenTypeSet {
105  pub fn new(vals: &[TokenType]) -> TokenTypeSet {
106    let mut set = 0u128;
107    for t in vals.iter().cloned() {
108      set |= 1u128 << (t as u8);
109    }
110    TokenTypeSet(set)
111  }
112
113  pub fn contains(self, t: TokenType) -> bool {
114    (self.0 & (1u128 << (t as u8))) != 0
115  }
116}
117
118#[derive(Clone)]
119pub struct Token {
120  pub loc: SourceRange,
121  pub typ: TokenType,
122}
123
124impl Token {
125  pub fn new(loc: SourceRange, typ: TokenType) -> Token {
126    Token { loc, typ }
127  }
128
129  pub fn error(&self, typ: SyntaxErrorType) -> SyntaxError {
130    SyntaxError::from_loc(self.loc, typ, Some(self.typ.clone()))
131  }
132}
arrow_parser/token.rs

arrow_parser/
token.rs
