oak_fsharp/lexer/token_type.rs
1use oak_core::{Source, Token, TokenType, UniversalElementRole, UniversalTokenRole};
2#[cfg(feature = "serde")]
3use serde::{Deserialize, Serialize};
4
5pub type FSharpToken = Token<FSharpTokenType>;
6
7/// F# token types
8#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
9#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
10pub enum FSharpTokenType {
11 /// Root node
12 Root,
13 /// Expression
14 Expression,
15 /// Whitespace
16 Whitespace,
17 /// Newline
18 Newline,
19
20 /// Identifier
21 Identifier,
22 /// Integer literal
23 IntegerLiteral,
24 /// Float literal
25 FloatLiteral,
26 /// String literal
27 StringLiteral,
28 /// Character literal
29 CharLiteral,
30 /// Boolean literal
31 BooleanLiteral,
32 /// Unit literal
33 UnitLiteral,
34
35 /// The 'let' keyword
36 Let,
37 /// The 'rec' keyword
38 Rec,
39 /// The 'and' keyword
40 And,
41 /// The 'in' keyword
42 In,
43 /// The 'if' keyword
44 If,
45 /// The 'then' keyword
46 Then,
47 /// The 'else' keyword
48 Else,
49 /// The 'elif' keyword
50 Elif,
51 /// The 'match' keyword
52 Match,
53 /// The 'with' keyword
54 With,
55 /// The 'when' keyword
56 When,
57 /// The 'function' keyword
58 Function,
59 /// The 'fun' keyword
60 Fun,
61
62 /// The 'type' keyword
63 Type,
64 /// The 'val' keyword
65 Val,
66 /// The 'mutable' keyword
67 Mutable,
68 /// The 'of' keyword
69 Of,
70 /// The 'as' keyword
71 As,
72
73 /// The 'module' keyword
74 Module,
75 /// The 'namespace' keyword
76 Namespace,
77 /// The 'open' keyword
78 Open,
79
80 /// The 'try' keyword
81 Try,
82 /// The 'finally' keyword
83 Finally,
84 /// The 'exception' keyword
85 Exception,
86 /// The 'raise' keyword
87 Raise,
88 /// The 'failwith' keyword
89 Failwith,
90
91 /// The 'for' keyword
92 For,
93 /// The 'to' keyword
94 To,
95 /// The 'downto' keyword
96 Downto,
97 /// The 'do' keyword
98 Do,
99 /// The 'done' keyword
100 Done,
101 /// The 'while' keyword
102 While,
103 /// The 'yield' keyword
104 Yield,
105 /// The 'return' keyword
106 Return,
107
108 /// The 'class' keyword
109 Class,
110 /// The 'interface' keyword
111 Interface,
112 /// The 'inherit' keyword
113 Inherit,
114 /// The 'abstract' keyword
115 Abstract,
116 /// The 'override' keyword
117 Override,
118 /// The 'default' keyword
119 Default,
120 /// The 'member' keyword
121 Member,
122 /// The 'static' keyword
123 Static,
124 /// The 'new' keyword
125 New,
126
127 /// The 'lazy' keyword
128 Lazy,
129 /// The 'async' keyword
130 Async,
131 /// The 'seq' keyword
132 Seq,
133 /// The 'use' keyword
134 Use,
135 /// The 'begin' keyword
136 Begin,
137 /// The 'end' keyword
138 End,
139 /// The 'struct' keyword
140 Struct,
141 /// The 'sig' keyword
142 Sig,
143
144 /// The 'true' keyword
145 True,
146 /// The 'false' keyword
147 False,
148 /// The 'null' keyword
149 Null,
150 /// The 'or' keyword
151 Or,
152
153 /// The 'public' keyword
154 Public,
155 /// The 'private' keyword
156 Private,
157 /// The 'internal' keyword
158 Internal,
159
160 /// The 'inline' keyword
161 Inline,
162 /// The 'extern' keyword
163 Extern,
164 /// The 'upcast' keyword
165 Upcast,
166 /// The 'downcast' keyword
167 Downcast,
168 /// The 'assert' keyword
169 Assert,
170 /// The 'global' keyword
171 Global,
172 /// The 'base' keyword
173 Base,
174 /// The 'this' keyword
175 This,
176 /// The 'void' keyword
177 Void,
178 /// The 'delegate' keyword
179 Delegate,
180 /// The 'select' keyword
181 Select,
182
183 /// The 'obj' keyword
184 Obj,
185 /// The 'unit' keyword
186 Unit,
187 /// The 'int' keyword
188 Int,
189 /// The 'float' keyword
190 Float,
191 /// The 'string' keyword
192 String,
193 /// The 'bool' keyword
194 Bool,
195 /// The 'char' keyword
196 Char,
197 /// The 'byte' keyword
198 Byte,
199 /// The 'sbyte' keyword
200 SByte,
201 /// The 'int16' keyword
202 Int16,
203 /// The 'uint16' keyword
204 UInt16,
205 /// The 'int32' keyword
206 Int32,
207 /// The 'uint32' keyword
208 UInt32,
209 /// The 'int64' keyword
210 Int64,
211 /// The 'uint64' keyword
212 UInt64,
213 /// The 'nativeint' keyword
214 NativeInt,
215 /// The 'unativeint' keyword
216 UNativeInt,
217 /// The 'decimal' keyword
218 Decimal,
219 /// The 'bigint' keyword
220 BigInt,
221
222 /// The '+' operator
223 Plus,
224 /// The '-' operator
225 Minus,
226 /// The '*' operator
227 Star,
228 /// The '/' operator
229 Slash,
230 /// The '%' operator
231 Percent,
232 /// The '**' operator
233 StarStar,
234
235 /// The '=' operator
236 Equal,
237 /// The '<>' operator
238 NotEqual,
239 /// The '<' operator
240 LessThan,
241 /// The '<=' operator
242 LessEqual,
243 /// The '>' operator
244 GreaterThan,
245 /// The '>=' operator
246 GreaterEqual,
247
248 /// The '&&' operator
249 AndAnd,
250 /// The '||' operator
251 OrOr,
252 /// The 'not' operator
253 Not,
254
255 /// The '&&&' operator
256 BitwiseAnd,
257 /// The '|||' operator
258 BitwiseOr,
259 /// The '^^^' operator
260 BitwiseXor,
261 /// The '~~~' operator
262 BitwiseNot,
263 /// The '<<<' operator
264 LeftShift,
265 /// The '>>>' operator
266 RightShift,
267
268 /// The '->' operator
269 Arrow,
270 /// The '=>' operator
271 DoubleArrow,
272 /// The '|' operator
273 Pipe,
274 /// The '|>' operator
275 PipeRight,
276 /// The '||' operator
277 DoublePipe,
278 /// The '::' operator
279 Cons,
280 /// The '@' operator
281 At,
282 /// The '>>' operator
283 Compose,
284 /// The '<<' operator
285 ComposeBack,
286 /// The '$' operator
287 Dollar,
288
289 /// The '&&' logical operator
290 LogicalAnd,
291 /// The '||' logical operator
292 LogicalOr,
293 /// The '&' operator
294 Ampersand,
295 /// The '^' operator
296 Caret,
297 /// The '~' operator
298 Tilde,
299 /// The '<' operator
300 Less,
301 /// The '>' operator
302 Greater,
303 /// The '|>' operator
304 PipeGreater,
305 /// The '!' operator
306 Exclamation,
307 /// The ':=' operator
308 ColonEqual,
309 /// The '<-' operator
310 LArrow,
311 /// The '++' operator
312 PlusPlus,
313 /// The '--' operator
314 MinusMinus,
315
316 /// The '(' delimiter
317 LeftParen,
318 /// The ')' delimiter
319 RightParen,
320 /// The '[' delimiter
321 LeftBracket,
322 /// The ']' delimiter
323 RightBracket,
324 /// The '[|' delimiter
325 LeftArrayBracket,
326 /// The '|]' delimiter
327 RightArrayBracket,
328 /// The '[<' delimiter
329 LeftBracketBar,
330 /// The '>]' delimiter
331 RightBracketBar,
332 /// The '[ <' delimiter
333 LeftBracketAngle,
334 /// The '> ]' delimiter
335 RightBracketAngle,
336 /// The '{' delimiter
337 LeftBrace,
338 /// The '}' delimiter
339 RightBrace,
340 /// The '<' delimiter
341 LeftAngle,
342 /// The '>' delimiter
343 RightAngle,
344
345 /// The ',' punctuation
346 Comma,
347 /// The ';' punctuation
348 Semicolon,
349 /// The ':' punctuation
350 Colon,
351 /// The '::' punctuation
352 DoubleColon,
353 /// The '.' punctuation
354 Dot,
355 /// The '..' punctuation
356 DotDot,
357 /// The '?' punctuation
358 Question,
359 /// The '_' punctuation
360 Underscore,
361 /// The ''' punctuation
362 Apostrophe,
363 /// The '`' punctuation
364 Backtick,
365 /// The '#' punctuation
366 Hash,
367
368 /// Line comment
369 LineComment,
370 /// Block comment
371 BlockComment,
372
373 /// Error
374 Error,
375 /// End of file
376 Eof,
377}
378
379impl FSharpTokenType {
380 /// Checks if it is a keyword
381 pub fn is_keyword(&self) -> bool {
382 matches!(
383 self,
384 Self::Let
385 | Self::Rec
386 | Self::And
387 | Self::In
388 | Self::If
389 | Self::Then
390 | Self::Else
391 | Self::Elif
392 | Self::Match
393 | Self::With
394 | Self::When
395 | Self::Function
396 | Self::Fun
397 | Self::Type
398 | Self::Val
399 | Self::Mutable
400 | Self::Of
401 | Self::As
402 | Self::Module
403 | Self::Namespace
404 | Self::Open
405 | Self::Try
406 | Self::Finally
407 | Self::Exception
408 | Self::Raise
409 | Self::Failwith
410 | Self::For
411 | Self::To
412 | Self::Downto
413 | Self::Do
414 | Self::Done
415 | Self::While
416 | Self::Yield
417 | Self::Return
418 | Self::Class
419 | Self::Interface
420 | Self::Inherit
421 | Self::Abstract
422 | Self::Override
423 | Self::Default
424 | Self::Member
425 | Self::Static
426 | Self::New
427 | Self::Lazy
428 | Self::Async
429 | Self::Seq
430 | Self::Use
431 | Self::Begin
432 | Self::End
433 | Self::Struct
434 | Self::Sig
435 | Self::True
436 | Self::False
437 | Self::Null
438 | Self::Or
439 | Self::Public
440 | Self::Private
441 | Self::Internal
442 | Self::Inline
443 | Self::Extern
444 | Self::Upcast
445 | Self::Downcast
446 | Self::Assert
447 | Self::Delegate
448 | Self::Select
449 )
450 }
451}
452
453impl TokenType for FSharpTokenType {
454 type Role = UniversalTokenRole;
455 const END_OF_STREAM: Self = Self::Eof;
456
457 fn is_ignored(&self) -> bool {
458 matches!(self, Self::Whitespace | Self::Newline | Self::LineComment | Self::BlockComment)
459 }
460
461 fn role(&self) -> Self::Role {
462 match self {
463 Self::Error => UniversalTokenRole::Error,
464 _ if self.is_keyword() => UniversalTokenRole::Keyword,
465 Self::Identifier => UniversalTokenRole::Name,
466 Self::IntegerLiteral | Self::FloatLiteral | Self::StringLiteral | Self::CharLiteral | Self::BooleanLiteral => UniversalTokenRole::Literal,
467 _ => UniversalTokenRole::None,
468 }
469 }
470}