1use alloc::string::String;
2use core::fmt;
3#[derive(Debug, Clone, PartialEq)]
4pub struct Token {
5 pub kind: TokenKind,
6 pub lexeme: String,
7 pub line: usize,
8 pub column: usize,
9}
10
11impl Token {
12 pub fn new(kind: TokenKind, lexeme: String, line: usize, column: usize) -> Self {
13 Self {
14 kind,
15 lexeme,
16 line,
17 column,
18 }
19 }
20}
21
22#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
23pub enum TokenKind {
24 Integer,
25 Float,
26 String,
27 True,
28 False,
29 Identifier,
30 Local,
31 Mut,
32 Function,
33 Return,
34 If,
35 Then,
36 Else,
37 Elseif,
38 End,
39 While,
40 Do,
41 For,
42 In,
43 Break,
44 Continue,
45 Struct,
46 Enum,
47 Trait,
48 Impl,
49 Match,
50 Case,
51 As,
52 Is,
53 Extern,
54 Unsafe,
55 Pub,
56 Use,
57 Module,
58 Const,
59 Static,
60 Type,
61 Plus,
62 Minus,
63 Star,
64 Slash,
65 Percent,
66 Caret,
67 DoubleDot,
68 Concat,
69 Equal,
70 DoubleEqual,
71 NotEqual,
72 Less,
73 LessEqual,
74 Greater,
75 GreaterEqual,
76 And,
77 Or,
78 Not,
79 PlusEqual,
80 MinusEqual,
81 StarEqual,
82 SlashEqual,
83 Ampersand,
84 Pipe,
85 Question,
86 LeftParen,
87 RightParen,
88 LeftBrace,
89 RightBrace,
90 LeftBracket,
91 RightBracket,
92 Comma,
93 Colon,
94 DoubleColon,
95 Semicolon,
96 Dot,
97 Arrow,
98 FatArrow,
99 Comment,
100 Whitespace,
101 Newline,
102 Eof,
103}
104
105impl fmt::Display for TokenKind {
106 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
107 match self {
108 TokenKind::Integer => write!(f, "integer"),
109 TokenKind::Float => write!(f, "float"),
110 TokenKind::String => write!(f, "string"),
111 TokenKind::True => write!(f, "true"),
112 TokenKind::False => write!(f, "false"),
113 TokenKind::Identifier => write!(f, "identifier"),
114 TokenKind::Local => write!(f, "local"),
115 TokenKind::Mut => write!(f, "mut"),
116 TokenKind::Function => write!(f, "function"),
117 TokenKind::Return => write!(f, "return"),
118 TokenKind::If => write!(f, "if"),
119 TokenKind::Then => write!(f, "then"),
120 TokenKind::Else => write!(f, "else"),
121 TokenKind::Elseif => write!(f, "elseif"),
122 TokenKind::End => write!(f, "end"),
123 TokenKind::While => write!(f, "while"),
124 TokenKind::Do => write!(f, "do"),
125 TokenKind::For => write!(f, "for"),
126 TokenKind::In => write!(f, "in"),
127 TokenKind::Break => write!(f, "break"),
128 TokenKind::Continue => write!(f, "continue"),
129 TokenKind::Struct => write!(f, "struct"),
130 TokenKind::Enum => write!(f, "enum"),
131 TokenKind::Trait => write!(f, "trait"),
132 TokenKind::Impl => write!(f, "impl"),
133 TokenKind::Match => write!(f, "match"),
134 TokenKind::Case => write!(f, "case"),
135 TokenKind::As => write!(f, "as"),
136 TokenKind::Is => write!(f, "is"),
137 TokenKind::Extern => write!(f, "extern"),
138 TokenKind::Unsafe => write!(f, "unsafe"),
139 TokenKind::Pub => write!(f, "pub"),
140 TokenKind::Use => write!(f, "use"),
141 TokenKind::Module => write!(f, "module"),
142 TokenKind::Const => write!(f, "const"),
143 TokenKind::Static => write!(f, "static"),
144 TokenKind::Type => write!(f, "type"),
145 TokenKind::Plus => write!(f, "+"),
146 TokenKind::Minus => write!(f, "-"),
147 TokenKind::Star => write!(f, "*"),
148 TokenKind::Slash => write!(f, "/"),
149 TokenKind::Percent => write!(f, "%"),
150 TokenKind::Caret => write!(f, "^"),
151 TokenKind::DoubleDot => write!(f, ".."),
152 TokenKind::Concat => write!(f, ".."),
153 TokenKind::Equal => write!(f, "="),
154 TokenKind::DoubleEqual => write!(f, "=="),
155 TokenKind::NotEqual => write!(f, "~="),
156 TokenKind::Less => write!(f, "<"),
157 TokenKind::LessEqual => write!(f, "<="),
158 TokenKind::Greater => write!(f, ">"),
159 TokenKind::GreaterEqual => write!(f, ">="),
160 TokenKind::And => write!(f, "and"),
161 TokenKind::Or => write!(f, "or"),
162 TokenKind::Not => write!(f, "not"),
163 TokenKind::PlusEqual => write!(f, "+="),
164 TokenKind::MinusEqual => write!(f, "-="),
165 TokenKind::StarEqual => write!(f, "*="),
166 TokenKind::SlashEqual => write!(f, "/="),
167 TokenKind::Ampersand => write!(f, "&"),
168 TokenKind::Pipe => write!(f, "|"),
169 TokenKind::Question => write!(f, "?"),
170 TokenKind::LeftParen => write!(f, "("),
171 TokenKind::RightParen => write!(f, ")"),
172 TokenKind::LeftBrace => write!(f, "{{"),
173 TokenKind::RightBrace => write!(f, "}}"),
174 TokenKind::LeftBracket => write!(f, "["),
175 TokenKind::RightBracket => write!(f, "]"),
176 TokenKind::Comma => write!(f, ","),
177 TokenKind::Colon => write!(f, ":"),
178 TokenKind::DoubleColon => write!(f, "::"),
179 TokenKind::Semicolon => write!(f, ";"),
180 TokenKind::Dot => write!(f, "."),
181 TokenKind::Arrow => write!(f, "->"),
182 TokenKind::FatArrow => write!(f, "=>"),
183 TokenKind::Comment => write!(f, "comment"),
184 TokenKind::Whitespace => write!(f, "whitespace"),
185 TokenKind::Newline => write!(f, "newline"),
186 TokenKind::Eof => write!(f, "EOF"),
187 }
188 }
189}
190
191impl TokenKind {
192 pub fn keyword(s: &str) -> Option<TokenKind> {
193 match s {
194 "local" => Some(TokenKind::Local),
195 "mut" => Some(TokenKind::Mut),
196 "function" => Some(TokenKind::Function),
197 "return" => Some(TokenKind::Return),
198 "if" => Some(TokenKind::If),
199 "then" => Some(TokenKind::Then),
200 "else" => Some(TokenKind::Else),
201 "elseif" => Some(TokenKind::Elseif),
202 "end" => Some(TokenKind::End),
203 "while" => Some(TokenKind::While),
204 "do" => Some(TokenKind::Do),
205 "for" => Some(TokenKind::For),
206 "in" => Some(TokenKind::In),
207 "break" => Some(TokenKind::Break),
208 "continue" => Some(TokenKind::Continue),
209 "struct" => Some(TokenKind::Struct),
210 "enum" => Some(TokenKind::Enum),
211 "trait" => Some(TokenKind::Trait),
212 "impl" => Some(TokenKind::Impl),
213 "match" => Some(TokenKind::Match),
214 "case" => Some(TokenKind::Case),
215 "as" => Some(TokenKind::As),
216 "is" => Some(TokenKind::Is),
217 "true" => Some(TokenKind::True),
218 "false" => Some(TokenKind::False),
219 "and" => Some(TokenKind::And),
220 "or" => Some(TokenKind::Or),
221 "not" => Some(TokenKind::Not),
222 "extern" => Some(TokenKind::Extern),
223 "unsafe" => Some(TokenKind::Unsafe),
224 "pub" => Some(TokenKind::Pub),
225 "use" => Some(TokenKind::Use),
226 "module" => Some(TokenKind::Module),
227 "const" => Some(TokenKind::Const),
228 "static" => Some(TokenKind::Static),
229 "type" => Some(TokenKind::Type),
230 _ => None,
231 }
232 }
233}