oak_fsharp/lexer/token_type.rs
1use oak_core::{Source, Token, TokenType, UniversalElementRole, UniversalTokenRole};
2
3pub type FSharpToken = Token<FSharpTokenType>;
4
5/// F# token types
6#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
7#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
8pub enum FSharpTokenType {
9 /// Root node
10 Root,
11 /// Expression
12 Expression,
13 /// Whitespace
14 Whitespace,
15 /// Newline
16 Newline,
17
18 /// Identifier
19 Identifier,
20 /// Integer literal
21 IntegerLiteral,
22 /// Float literal
23 FloatLiteral,
24 /// String literal
25 StringLiteral,
26 /// Character literal
27 CharLiteral,
28 /// Boolean literal
29 BooleanLiteral,
30 /// Unit literal
31 UnitLiteral,
32
33 /// The 'let' keyword
34 Let,
35 /// The 'rec' keyword
36 Rec,
37 /// The 'and' keyword
38 And,
39 /// The 'in' keyword
40 In,
41 /// The 'if' keyword
42 If,
43 /// The 'then' keyword
44 Then,
45 /// The 'else' keyword
46 Else,
47 /// The 'elif' keyword
48 Elif,
49 /// The 'match' keyword
50 Match,
51 /// The 'with' keyword
52 With,
53 /// The 'when' keyword
54 When,
55 /// The 'function' keyword
56 Function,
57 /// The 'fun' keyword
58 Fun,
59
60 /// The 'type' keyword
61 Type,
62 /// The 'val' keyword
63 Val,
64 /// The 'mutable' keyword
65 Mutable,
66 /// The 'of' keyword
67 Of,
68 /// The 'as' keyword
69 As,
70
71 /// The 'module' keyword
72 Module,
73 /// The 'namespace' keyword
74 Namespace,
75 /// The 'open' keyword
76 Open,
77
78 /// The 'try' keyword
79 Try,
80 /// The 'finally' keyword
81 Finally,
82 /// The 'exception' keyword
83 Exception,
84 /// The 'raise' keyword
85 Raise,
86 /// The 'failwith' keyword
87 Failwith,
88
89 /// The 'for' keyword
90 For,
91 /// The 'to' keyword
92 To,
93 /// The 'downto' keyword
94 Downto,
95 /// The 'do' keyword
96 Do,
97 /// The 'done' keyword
98 Done,
99 /// The 'while' keyword
100 While,
101 /// The 'yield' keyword
102 Yield,
103 /// The 'return' keyword
104 Return,
105
106 /// The 'class' keyword
107 Class,
108 /// The 'interface' keyword
109 Interface,
110 /// The 'inherit' keyword
111 Inherit,
112 /// The 'abstract' keyword
113 Abstract,
114 /// The 'override' keyword
115 Override,
116 /// The 'default' keyword
117 Default,
118 /// The 'member' keyword
119 Member,
120 /// The 'static' keyword
121 Static,
122 /// The 'new' keyword
123 New,
124
125 /// The 'lazy' keyword
126 Lazy,
127 /// The 'async' keyword
128 Async,
129 /// The 'seq' keyword
130 Seq,
131 /// The 'use' keyword
132 Use,
133 /// The 'begin' keyword
134 Begin,
135 /// The 'end' keyword
136 End,
137 /// The 'struct' keyword
138 Struct,
139 /// The 'sig' keyword
140 Sig,
141
142 /// The 'true' keyword
143 True,
144 /// The 'false' keyword
145 False,
146 /// The 'null' keyword
147 Null,
148 /// The 'or' keyword
149 Or,
150
151 /// The 'public' keyword
152 Public,
153 /// The 'private' keyword
154 Private,
155 /// The 'internal' keyword
156 Internal,
157
158 /// The 'inline' keyword
159 Inline,
160 /// The 'extern' keyword
161 Extern,
162 /// The 'upcast' keyword
163 Upcast,
164 /// The 'downcast' keyword
165 Downcast,
166 /// The 'assert' keyword
167 Assert,
168 /// The 'global' keyword
169 Global,
170 /// The 'base' keyword
171 Base,
172 /// The 'this' keyword
173 This,
174 /// The 'void' keyword
175 Void,
176 /// The 'delegate' keyword
177 Delegate,
178 /// The 'select' keyword
179 Select,
180
181 /// The 'obj' keyword
182 Obj,
183 /// The 'unit' keyword
184 Unit,
185 /// The 'int' keyword
186 Int,
187 /// The 'float' keyword
188 Float,
189 /// The 'string' keyword
190 String,
191 /// The 'bool' keyword
192 Bool,
193 /// The 'char' keyword
194 Char,
195 /// The 'byte' keyword
196 Byte,
197 /// The 'sbyte' keyword
198 SByte,
199 /// The 'int16' keyword
200 Int16,
201 /// The 'uint16' keyword
202 UInt16,
203 /// The 'int32' keyword
204 Int32,
205 /// The 'uint32' keyword
206 UInt32,
207 /// The 'int64' keyword
208 Int64,
209 /// The 'uint64' keyword
210 UInt64,
211 /// The 'nativeint' keyword
212 NativeInt,
213 /// The 'unativeint' keyword
214 UNativeInt,
215 /// The 'decimal' keyword
216 Decimal,
217 /// The 'bigint' keyword
218 BigInt,
219
220 /// The '+' operator
221 Plus,
222 /// The '-' operator
223 Minus,
224 /// The '*' operator
225 Star,
226 /// The '/' operator
227 Slash,
228 /// The '%' operator
229 Percent,
230 /// The '**' operator
231 StarStar,
232
233 /// The '=' operator
234 Equal,
235 /// The '<>' operator
236 NotEqual,
237 /// The '<' operator
238 LessThan,
239 /// The '<=' operator
240 LessEqual,
241 /// The '>' operator
242 GreaterThan,
243 /// The '>=' operator
244 GreaterEqual,
245
246 /// The '&&' operator
247 AndAnd,
248 /// The '||' operator
249 OrOr,
250 /// The 'not' operator
251 Not,
252
253 /// The '&&&' operator
254 BitwiseAnd,
255 /// The '|||' operator
256 BitwiseOr,
257 /// The '^^^' operator
258 BitwiseXor,
259 /// The '~~~' operator
260 BitwiseNot,
261 /// The '<<<' operator
262 LeftShift,
263 /// The '>>>' operator
264 RightShift,
265
266 /// The '->' operator
267 Arrow,
268 /// The '=>' operator
269 DoubleArrow,
270 /// The '|' operator
271 Pipe,
272 /// The '|>' operator
273 PipeRight,
274 /// The '||' operator
275 DoublePipe,
276 /// The '::' operator
277 Cons,
278 /// The '@' operator
279 At,
280 /// The '>>' operator
281 Compose,
282 /// The '<<' operator
283 ComposeBack,
284 /// The '$' operator
285 Dollar,
286 /// The '|>' operator
287 PipeForward,
288 /// The '<|' operator
289 PipeBackward,
290
291 /// The '&&' logical operator
292 LogicalAnd,
293 /// The '||' logical operator
294 LogicalOr,
295 /// The '&' operator
296 Ampersand,
297 /// The '^' operator
298 Caret,
299 /// The '~' operator
300 Tilde,
301 /// The '<' operator
302 Less,
303 /// The '>' operator
304 Greater,
305 /// The '|>' operator
306 PipeGreater,
307 /// The '!' operator
308 Exclamation,
309 /// The ':=' operator
310 ColonEqual,
311 /// The '<-' operator
312 LArrow,
313 /// The '++' operator
314 PlusPlus,
315 /// The '--' operator
316 MinusMinus,
317
318 /// The '(' delimiter
319 LeftParen,
320 /// The ')' delimiter
321 RightParen,
322 /// The '[' delimiter
323 LeftBracket,
324 /// The ']' delimiter
325 RightBracket,
326 /// The '[|' delimiter
327 LeftArrayBracket,
328 /// The '|]' delimiter
329 RightArrayBracket,
330 /// The '[<' delimiter
331 LeftBracketBar,
332 /// The '>]' delimiter
333 RightBracketBar,
334 /// The '[ <' delimiter
335 LeftBracketAngle,
336 /// The '> ]' delimiter
337 RightBracketAngle,
338 /// The '{' delimiter
339 LeftBrace,
340 /// The '}' delimiter
341 RightBrace,
342 /// The '<' delimiter
343 LeftAngle,
344 /// The '>' delimiter
345 RightAngle,
346
347 /// The ',' punctuation
348 Comma,
349 /// The ';' punctuation
350 Semicolon,
351 /// The ':' punctuation
352 Colon,
353 /// The '::' punctuation
354 DoubleColon,
355 /// The '.' punctuation
356 Dot,
357 /// The '..' punctuation
358 DotDot,
359 /// The '?' punctuation
360 Question,
361 /// The '_' punctuation
362 Underscore,
363 /// The ''' punctuation
364 Apostrophe,
365 /// The '`' punctuation
366 Backtick,
367 /// The '#' punctuation
368 Hash,
369
370 /// Line comment
371 LineComment,
372 /// Block comment
373 BlockComment,
374
375 /// Error
376 Error,
377 /// End of file
378 Eof,
379}
380
381impl FSharpTokenType {
382 /// Checks if it is a keyword
383 pub fn is_keyword(&self) -> bool {
384 matches!(
385 self,
386 Self::Let
387 | Self::Rec
388 | Self::And
389 | Self::In
390 | Self::If
391 | Self::Then
392 | Self::Else
393 | Self::Elif
394 | Self::Match
395 | Self::With
396 | Self::When
397 | Self::Function
398 | Self::Fun
399 | Self::Type
400 | Self::Val
401 | Self::Mutable
402 | Self::Of
403 | Self::As
404 | Self::Module
405 | Self::Namespace
406 | Self::Open
407 | Self::Try
408 | Self::Finally
409 | Self::Exception
410 | Self::Raise
411 | Self::Failwith
412 | Self::For
413 | Self::To
414 | Self::Downto
415 | Self::Do
416 | Self::Done
417 | Self::While
418 | Self::Yield
419 | Self::Return
420 | Self::Class
421 | Self::Interface
422 | Self::Inherit
423 | Self::Abstract
424 | Self::Override
425 | Self::Default
426 | Self::Member
427 | Self::Static
428 | Self::New
429 | Self::Lazy
430 | Self::Async
431 | Self::Seq
432 | Self::Use
433 | Self::Begin
434 | Self::End
435 | Self::Struct
436 | Self::Sig
437 | Self::True
438 | Self::False
439 | Self::Null
440 | Self::Or
441 | Self::Public
442 | Self::Private
443 | Self::Internal
444 | Self::Inline
445 | Self::Extern
446 | Self::Upcast
447 | Self::Downcast
448 | Self::Assert
449 | Self::Delegate
450 | Self::Select
451 )
452 }
453}
454
455impl TokenType for FSharpTokenType {
456 type Role = UniversalTokenRole;
457 const END_OF_STREAM: Self = Self::Eof;
458
459 fn is_ignored(&self) -> bool {
460 matches!(self, Self::Whitespace | Self::Newline | Self::LineComment | Self::BlockComment)
461 }
462
463 fn role(&self) -> Self::Role {
464 match self {
465 Self::Error => UniversalTokenRole::Error,
466 _ if self.is_keyword() => UniversalTokenRole::Keyword,
467 Self::Identifier => UniversalTokenRole::Name,
468 Self::IntegerLiteral | Self::FloatLiteral | Self::StringLiteral | Self::CharLiteral | Self::BooleanLiteral => UniversalTokenRole::Literal,
469 _ => UniversalTokenRole::None,
470 }
471 }
472}