oak_ruby/lexer/
token_type.rs

1use oak_core::{Source, Token, TokenType, UniversalElementRole, UniversalTokenRole};
2use serde::{Deserialize, Serialize};
3use std::fmt;
4
5/// Represents a token in a Ruby source file.
6pub type RubyToken = Token<RubyTokenType>;
7
8/// Token types for Ruby.
9#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
10#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
11#[repr(u8)]
12pub enum RubyTokenType {
13    /// An identifier.
14    Identifier,
15    /// A global variable (e.g., `$global`).
16    GlobalVariable,
17    /// An instance variable (e.g., `@instance`).
18    InstanceVariable,
19    /// A class variable (e.g., `@@class`).
20    ClassVariable,
21    /// A constant (e.g., `CONSTANT`).
22    Constant,
23    /// An integer literal.
24    IntegerLiteral,
25    /// A float literal.
26    FloatLiteral,
27    /// A string literal.
28    StringLiteral,
29    /// A literal value.
30    Literal,
31    /// A symbol (e.g., `:symbol`).
32    Symbol,
33    /// A regular expression literal.
34    RegexLiteral,
35
36    /// The `if` keyword.
37    If,
38    /// The `unless` keyword.
39    Unless,
40    /// The `elsif` keyword.
41    Elsif,
42    /// The `else` keyword.
43    Else,
44    /// The `case` keyword.
45    Case,
46    /// The `when` keyword.
47    When,
48    /// The `then` keyword.
49    Then,
50    /// The `for` keyword.
51    For,
52    /// The `while` keyword.
53    While,
54    /// The `until` keyword.
55    Until,
56    /// The `break` keyword.
57    Break,
58    /// The `next` keyword.
59    Next,
60    /// The `redo` keyword.
61    Redo,
62    /// The `retry` keyword.
63    Retry,
64    /// The `return` keyword.
65    Return,
66    /// The `yield` keyword.
67    Yield,
68    /// The `def` keyword.
69    Def,
70    /// The `class` keyword.
71    Class,
72    /// The `module` keyword.
73    Module,
74    /// The `end` keyword.
75    End,
76    /// The `lambda` keyword.
77    Lambda,
78    /// The `proc` keyword.
79    Proc,
80    /// The `begin` keyword.
81    Begin,
82    /// The `rescue` keyword.
83    Rescue,
84    /// The `ensure` keyword.
85    Ensure,
86    /// The `raise` keyword.
87    Raise,
88    /// The `require` keyword.
89    Require,
90    /// The `load` keyword.
91    Load,
92    /// The `include` keyword.
93    Include,
94    /// The `extend` keyword.
95    Extend,
96    /// The `prepend` keyword.
97    Prepend,
98    /// The `and` keyword.
99    And,
100    /// The `or` keyword.
101    Or,
102    /// The `=>` operator.
103    EqualGreater,
104    /// The `not` keyword.
105    Not,
106    /// The `in` keyword.
107    In,
108    /// The `true` keyword.
109    True,
110    /// The `false` keyword.
111    False,
112    /// The `nil` keyword.
113    Nil,
114    /// The `super` keyword.
115    Super,
116    /// The `self` keyword.
117    Self_,
118    /// The `alias` keyword.
119    Alias,
120    /// The `undef` keyword.
121    Undef,
122    /// The `defined?` keyword.
123    Defined,
124    /// The `do` keyword.
125    Do,
126
127    /// Plus operator `+`.
128    Plus,
129    /// Minus operator `-`.
130    Minus,
131    /// Multiply operator `*`.
132    Multiply,
133    /// Divide operator `/`.
134    Divide,
135    /// Modulo operator `%`.
136    Modulo,
137    /// Power operator `**`.
138    Power,
139    /// Equality operator `==`.
140    EqualEqual,
141    /// Inequality operator `!=`.
142    NotEqual,
143    /// Less than operator `<`.
144    Less,
145    /// Greater than operator `>`.
146    Greater,
147    /// Less than or equal operator `<=`.
148    LessEqual,
149    /// Greater than or equal operator `>=`.
150    GreaterEqual,
151    /// Case equality operator `===`.
152    EqualEqualEqual,
153    /// Spaceship operator `<=>`.
154    Spaceship,
155    /// Assignment operator `=`.
156    Assign,
157    /// Addition assignment operator `+=`.
158    PlusAssign,
159    /// Subtraction assignment operator `-=`.
160    MinusAssign,
161    /// Multiplication assignment operator `*=`.
162    MultiplyAssign,
163    /// Division assignment operator `/=`.
164    DivideAssign,
165    /// Modulo assignment operator `%=`.
166    ModuloAssign,
167    /// Power assignment operator `**=`.
168    PowerAssign,
169    /// Bitwise AND operator `&`.
170    BitAnd,
171    /// Bitwise OR operator `|`.
172    BitOr,
173    /// Bitwise XOR operator `^`.
174    Xor,
175    /// Logical NOT operator `!`.
176    LogicalNot,
177    /// Bitwise NOT operator `~`.
178    Tilde,
179    /// Left shift operator `<<`.
180    LeftShift,
181    /// Right shift operator `>>`.
182    RightShift,
183    /// Bitwise AND assignment operator `&=`.
184    AndAssign,
185    /// Bitwise OR assignment operator `|=`.
186    OrAssign,
187    /// Bitwise XOR assignment operator `^=`.
188    XorAssign,
189    /// Left shift assignment operator `<<=`.
190    LeftShiftAssign,
191    /// Right shift assignment operator `>>=`.
192    RightShiftAssign,
193    /// Logical AND operator `&&`.
194    AndAnd,
195    /// Logical OR operator `||`.
196    OrOr,
197    /// Logical OR assignment operator `||=`.
198    OrOrAssign,
199    /// Logical AND assignment operator `&&=`.
200    AndAndAssign,
201    /// Ternary operator `?`.
202    Question,
203    /// Range operator `..`.
204    DotDot,
205    /// Exclusive range operator `...`.
206    DotDotDot,
207    /// Match operator `=~`.
208    Match,
209    /// Not match operator `!~`.
210    NotMatch,
211
212    /// Left parenthesis `(`.
213    LeftParen,
214    /// Right parenthesis `)`.
215    RightParen,
216    /// Left bracket `[`.
217    LeftBracket,
218    /// Right bracket `]`.
219    RightBracket,
220    /// Left brace `{`.
221    LeftBrace,
222    /// Right brace `}`.
223    RightBrace,
224    /// Comma `,`.
225    Comma,
226    /// Colon `:`.
227    Colon,
228    /// Semicolon `;`.
229    Semicolon,
230    /// Dot `.`.
231    Dot,
232    /// Double colon `::`.
233    DoubleColon,
234    /// At symbol `@`.
235    At,
236    /// Dollar symbol `$`.
237    Dollar,
238
239    /// Whitespace.
240    Whitespace,
241    /// Newline.
242    Newline,
243    /// A comment.
244    Comment,
245    /// End of file.
246    Eof,
247    /// An invalid token.
248    Invalid,
249    /// The root of the syntax tree.
250    Root,
251    /// A binary expression.
252    BinaryExpression,
253    /// A unary expression.
254    UnaryExpression,
255    /// A literal expression.
256    LiteralExpression,
257    /// A parenthesized expression.
258    ParenExpression,
259    /// A parenthesized expression (alias).
260    ParenthesizedExpression,
261    /// A method definition.
262    MethodDefinition,
263    /// A class definition.
264    ClassDefinition,
265    /// A module definition.
266    ModuleDefinition,
267    /// An if statement.
268    IfStatement,
269    /// A while statement.
270    WhileStatement,
271    /// A return statement.
272    ReturnStatement,
273    /// An if expression.
274    IfExpression,
275    /// A method call expression.
276    CallExpression,
277    /// A member access expression.
278    MemberAccess,
279    /// A list of parameters.
280    ParameterList,
281    /// A list of arguments.
282    ArgumentList,
283    /// An error element.
284    Error,
285    /// Equality operator (alias).
286    Equal,
287}
288
289impl RubyTokenType {
290    /// Returns true if the token is ignored (whitespace, newline, or comment).
291    pub fn is_ignored(&self) -> bool {
292        matches!(self, Self::Whitespace | Self::Newline | Self::Comment)
293    }
294
295    /// Returns true if the token type is a keyword.
296    pub fn is_keyword(&self) -> bool {
297        matches!(
298            self,
299            Self::If
300                | Self::Unless
301                | Self::Elsif
302                | Self::Else
303                | Self::Case
304                | Self::When
305                | Self::Then
306                | Self::For
307                | Self::While
308                | Self::Until
309                | Self::Break
310                | Self::Next
311                | Self::Redo
312                | Self::Retry
313                | Self::Return
314                | Self::Yield
315                | Self::Def
316                | Self::Class
317                | Self::Module
318                | Self::End
319                | Self::Lambda
320                | Self::Proc
321                | Self::Begin
322                | Self::Rescue
323                | Self::Ensure
324                | Self::Raise
325                | Self::Require
326                | Self::Load
327                | Self::Include
328                | Self::Extend
329                | Self::Prepend
330                | Self::And
331                | Self::Or
332                | Self::Not
333                | Self::In
334                | Self::True
335                | Self::False
336                | Self::Nil
337                | Self::Super
338                | Self::Self_
339                | Self::Alias
340                | Self::Undef
341                | Self::Defined
342                | Self::Do
343        )
344    }
345}
346
347impl fmt::Display for RubyTokenType {
348    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
349        let name = match self {
350            Self::Identifier => "Identifier",
351            Self::GlobalVariable => "GlobalVariable",
352            Self::InstanceVariable => "InstanceVariable",
353            Self::ClassVariable => "ClassVariable",
354            Self::Constant => "Constant",
355            Self::IntegerLiteral => "IntegerLiteral",
356            Self::FloatLiteral => "FloatLiteral",
357            Self::StringLiteral => "StringLiteral",
358            Self::Literal => "Literal",
359            Self::Symbol => "Symbol",
360            Self::RegexLiteral => "RegexLiteral",
361
362            Self::If => "If",
363            Self::Unless => "Unless",
364            Self::Elsif => "Elsif",
365            Self::Else => "Else",
366            Self::Case => "Case",
367            Self::When => "When",
368            Self::Then => "Then",
369            Self::For => "For",
370            Self::While => "While",
371            Self::Until => "Until",
372            Self::Break => "Break",
373            Self::Next => "Next",
374            Self::Redo => "Redo",
375            Self::Retry => "Retry",
376            Self::Return => "Return",
377            Self::Yield => "Yield",
378            Self::Def => "Def",
379            Self::Class => "Class",
380            Self::Module => "Module",
381            Self::End => "End",
382            Self::Lambda => "Lambda",
383            Self::Proc => "Proc",
384            Self::Begin => "Begin",
385            Self::Rescue => "Rescue",
386            Self::Ensure => "Ensure",
387            Self::Raise => "Raise",
388            Self::Require => "Require",
389            Self::Load => "Load",
390            Self::Include => "Include",
391            Self::Extend => "Extend",
392            Self::Prepend => "Prepend",
393            Self::And => "And",
394            Self::Or => "Or",
395            Self::Not => "Not",
396            Self::In => "In",
397            Self::True => "True",
398            Self::False => "False",
399            Self::Nil => "Nil",
400            Self::Super => "Super",
401            Self::Self_ => "Self",
402            Self::Alias => "Alias",
403            Self::Undef => "Undef",
404            Self::Defined => "Defined",
405            Self::Do => "Do",
406
407            Self::Plus => "Plus",
408            Self::Minus => "Minus",
409            Self::Multiply => "Multiply",
410            Self::Divide => "Divide",
411            Self::Modulo => "Modulo",
412            Self::Power => "Power",
413            Self::EqualEqual => "EqualEqual",
414            Self::EqualGreater => "EqualGreater",
415            Self::NotEqual => "NotEqual",
416            Self::Less => "Less",
417            Self::Greater => "Greater",
418            Self::LessEqual => "LessEqual",
419            Self::GreaterEqual => "GreaterEqual",
420            Self::EqualEqualEqual => "EqualEqualEqual",
421            Self::Spaceship => "Spaceship",
422            Self::Assign => "Assign",
423            Self::PlusAssign => "PlusAssign",
424            Self::MinusAssign => "MinusAssign",
425            Self::MultiplyAssign => "MultiplyAssign",
426            Self::DivideAssign => "DivideAssign",
427            Self::ModuloAssign => "ModuloAssign",
428            Self::PowerAssign => "PowerAssign",
429            Self::BitAnd => "BitAnd",
430            Self::BitOr => "BitOr",
431            Self::Xor => "Xor",
432            Self::LogicalNot => "LogicalNot",
433            Self::Tilde => "Tilde",
434            Self::LeftShift => "LeftShift",
435            Self::RightShift => "RightShift",
436            Self::AndAssign => "AndAssign",
437            Self::OrAssign => "OrAssign",
438            Self::XorAssign => "XorAssign",
439            Self::LeftShiftAssign => "LeftShiftAssign",
440            Self::RightShiftAssign => "RightShiftAssign",
441            Self::AndAnd => "AndAnd",
442            Self::OrOr => "OrOr",
443            Self::OrOrAssign => "OrOrAssign",
444            Self::AndAndAssign => "AndAndAssign",
445            Self::Question => "Question",
446            Self::DotDot => "DotDot",
447            Self::DotDotDot => "DotDotDot",
448            Self::Match => "Match",
449            Self::NotMatch => "NotMatch",
450
451            Self::LeftParen => "LeftParen",
452            Self::RightParen => "RightParen",
453            Self::LeftBracket => "LeftBracket",
454            Self::RightBracket => "RightBracket",
455            Self::LeftBrace => "LeftBrace",
456            Self::RightBrace => "RightBrace",
457            Self::Comma => "Comma",
458            Self::Colon => "Colon",
459            Self::Semicolon => "Semicolon",
460            Self::Dot => "Dot",
461            Self::DoubleColon => "DoubleColon",
462            Self::At => "At",
463            Self::Dollar => "Dollar",
464
465            Self::Whitespace => "Whitespace",
466            Self::Newline => "Newline",
467            Self::Comment => "Comment",
468            Self::Eof => "Eof",
469            Self::Invalid => "Invalid",
470            Self::Root => "Root",
471            Self::BinaryExpression => "BinaryExpression",
472            Self::UnaryExpression => "UnaryExpression",
473            Self::LiteralExpression => "LiteralExpression",
474            Self::ParenExpression => "ParenExpression",
475            Self::ParenthesizedExpression => "ParenthesizedExpression",
476            Self::MethodDefinition => "MethodDefinition",
477            Self::ClassDefinition => "ClassDefinition",
478            Self::ModuleDefinition => "ModuleDefinition",
479            Self::IfStatement => "IfStatement",
480            Self::WhileStatement => "WhileStatement",
481            Self::ReturnStatement => "ReturnStatement",
482            Self::IfExpression => "IfExpression",
483            Self::CallExpression => "CallExpression",
484            Self::MemberAccess => "MemberAccess",
485            Self::ParameterList => "ParameterList",
486            Self::ArgumentList => "ArgumentList",
487            Self::Error => "Error",
488            Self::Equal => "Equal",
489        };
490        write!(f, "{}", name)
491    }
492}
493
494impl TokenType for RubyTokenType {
495    type Role = UniversalTokenRole;
496    const END_OF_STREAM: Self = Self::Error;
497
498    fn is_ignored(&self) -> bool {
499        false
500    }
501
502    fn role(&self) -> Self::Role {
503        match self {
504            _ => UniversalTokenRole::None,
505        }
506    }
507}
oak_ruby/lexer/token_type.rs

oak_ruby/lexer/
token_type.rs
