oak_ruby/lexer/
token_type.rs

1use oak_core::{Source, Token, TokenType, UniversalElementRole, UniversalTokenRole};
2use std::fmt;
3
4/// Represents a token in a Ruby source file.
5pub type RubyToken = Token<RubyTokenType>;
6
7/// Token types for Ruby.
8#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
9#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
10#[repr(u8)]
11pub enum RubyTokenType {
12    /// An identifier.
13    Identifier,
14    /// A global variable (e.g., `$global`).
15    GlobalVariable,
16    /// An instance variable (e.g., `@instance`).
17    InstanceVariable,
18    /// A class variable (e.g., `@@class`).
19    ClassVariable,
20    /// A constant (e.g., `CONSTANT`).
21    Constant,
22    /// An integer literal.
23    IntegerLiteral,
24    /// A float literal.
25    FloatLiteral,
26    /// A string literal.
27    StringLiteral,
28    /// A literal value.
29    Literal,
30    /// A symbol (e.g., `:symbol`).
31    Symbol,
32    /// A regular expression literal.
33    RegexLiteral,
34
35    /// The `if` keyword.
36    If,
37    /// The `unless` keyword.
38    Unless,
39    /// The `elsif` keyword.
40    Elsif,
41    /// The `else` keyword.
42    Else,
43    /// The `case` keyword.
44    Case,
45    /// The `when` keyword.
46    When,
47    /// The `then` keyword.
48    Then,
49    /// The `for` keyword.
50    For,
51    /// The `while` keyword.
52    While,
53    /// The `until` keyword.
54    Until,
55    /// The `break` keyword.
56    Break,
57    /// The `next` keyword.
58    Next,
59    /// The `redo` keyword.
60    Redo,
61    /// The `retry` keyword.
62    Retry,
63    /// The `return` keyword.
64    Return,
65    /// The `yield` keyword.
66    Yield,
67    /// The `def` keyword.
68    Def,
69    /// The `class` keyword.
70    Class,
71    /// The `module` keyword.
72    Module,
73    /// The `end` keyword.
74    End,
75    /// The `lambda` keyword.
76    Lambda,
77    /// The `proc` keyword.
78    Proc,
79    /// The `begin` keyword.
80    Begin,
81    /// The `rescue` keyword.
82    Rescue,
83    /// The `ensure` keyword.
84    Ensure,
85    /// The `raise` keyword.
86    Raise,
87    /// The `require` keyword.
88    Require,
89    /// The `load` keyword.
90    Load,
91    /// The `include` keyword.
92    Include,
93    /// The `extend` keyword.
94    Extend,
95    /// The `prepend` keyword.
96    Prepend,
97    /// The `and` keyword.
98    And,
99    /// The `or` keyword.
100    Or,
101    /// The `=>` operator.
102    EqualGreater,
103    /// The `not` keyword.
104    Not,
105    /// The `in` keyword.
106    In,
107    /// The `true` keyword.
108    True,
109    /// The `false` keyword.
110    False,
111    /// The `nil` keyword.
112    Nil,
113    /// The `super` keyword.
114    Super,
115    /// The `self` keyword.
116    Self_,
117    /// The `alias` keyword.
118    Alias,
119    /// The `undef` keyword.
120    Undef,
121    /// The `defined?` keyword.
122    Defined,
123    /// The `do` keyword.
124    Do,
125
126    /// Plus operator `+`.
127    Plus,
128    /// Minus operator `-`.
129    Minus,
130    /// Multiply operator `*`.
131    Multiply,
132    /// Divide operator `/`.
133    Divide,
134    /// Modulo operator `%`.
135    Modulo,
136    /// Power operator `**`.
137    Power,
138    /// Equality operator `==`.
139    EqualEqual,
140    /// Inequality operator `!=`.
141    NotEqual,
142    /// Less than operator `<`.
143    Less,
144    /// Greater than operator `>`.
145    Greater,
146    /// Less than or equal operator `<=`.
147    LessEqual,
148    /// Greater than or equal operator `>=`.
149    GreaterEqual,
150    /// Case equality operator `===`.
151    EqualEqualEqual,
152    /// Spaceship operator `<=>`.
153    Spaceship,
154    /// Assignment operator `=`.
155    Assign,
156    /// Addition assignment operator `+=`.
157    PlusAssign,
158    /// Subtraction assignment operator `-=`.
159    MinusAssign,
160    /// Multiplication assignment operator `*=`.
161    MultiplyAssign,
162    /// Division assignment operator `/=`.
163    DivideAssign,
164    /// Modulo assignment operator `%=`.
165    ModuloAssign,
166    /// Power assignment operator `**=`.
167    PowerAssign,
168    /// Bitwise AND operator `&`.
169    BitAnd,
170    /// Bitwise OR operator `|`.
171    BitOr,
172    /// Bitwise XOR operator `^`.
173    Xor,
174    /// Logical NOT operator `!`.
175    LogicalNot,
176    /// Bitwise NOT operator `~`.
177    Tilde,
178    /// Left shift operator `<<`.
179    LeftShift,
180    /// Right shift operator `>>`.
181    RightShift,
182    /// Bitwise AND assignment operator `&=`.
183    AndAssign,
184    /// Bitwise OR assignment operator `|=`.
185    OrAssign,
186    /// Bitwise XOR assignment operator `^=`.
187    XorAssign,
188    /// Left shift assignment operator `<<=`.
189    LeftShiftAssign,
190    /// Right shift assignment operator `>>=`.
191    RightShiftAssign,
192    /// Logical AND operator `&&`.
193    AndAnd,
194    /// Logical OR operator `||`.
195    OrOr,
196    /// Logical OR assignment operator `||=`.
197    OrOrAssign,
198    /// Logical AND assignment operator `&&=`.
199    AndAndAssign,
200    /// Ternary operator `?`.
201    Question,
202    /// Range operator `..`.
203    DotDot,
204    /// Exclusive range operator `...`.
205    DotDotDot,
206    /// Match operator `=~`.
207    Match,
208    /// Not match operator `!~`.
209    NotMatch,
210
211    /// Left parenthesis `(`.
212    LeftParen,
213    /// Right parenthesis `)`.
214    RightParen,
215    /// Left bracket `[`.
216    LeftBracket,
217    /// Right bracket `]`.
218    RightBracket,
219    /// Left brace `{`.
220    LeftBrace,
221    /// Right brace `}`.
222    RightBrace,
223    /// Comma `,`.
224    Comma,
225    /// Colon `:`.
226    Colon,
227    /// Semicolon `;`.
228    Semicolon,
229    /// Dot `.`.
230    Dot,
231    /// Double colon `::`.
232    DoubleColon,
233    /// At symbol `@`.
234    At,
235    /// Dollar symbol `$`.
236    Dollar,
237
238    /// Whitespace.
239    Whitespace,
240    /// Newline.
241    Newline,
242    /// A comment.
243    Comment,
244    /// End of file.
245    Eof,
246    /// An invalid token.
247    Invalid,
248    /// The root of the syntax tree.
249    Root,
250    /// A binary expression.
251    BinaryExpression,
252    /// A unary expression.
253    UnaryExpression,
254    /// A literal expression.
255    LiteralExpression,
256    /// A parenthesized expression.
257    ParenExpression,
258    /// A parenthesized expression (alias).
259    ParenthesizedExpression,
260    /// A method definition.
261    MethodDefinition,
262    /// A class definition.
263    ClassDefinition,
264    /// A module definition.
265    ModuleDefinition,
266    /// An if statement.
267    IfStatement,
268    /// A while statement.
269    WhileStatement,
270    /// A return statement.
271    ReturnStatement,
272    /// An if expression.
273    IfExpression,
274    /// A method call expression.
275    CallExpression,
276    /// A member access expression.
277    MemberAccess,
278    /// A list of parameters.
279    ParameterList,
280    /// A list of arguments.
281    ArgumentList,
282    /// An error element.
283    Error,
284    /// Equality operator (alias).
285    Equal,
286}
287
288impl RubyTokenType {
289    /// Returns true if the token is ignored (whitespace, newline, or comment).
290    pub fn is_ignored(&self) -> bool {
291        matches!(self, Self::Whitespace | Self::Newline | Self::Comment)
292    }
293
294    /// Returns true if the token type is a keyword.
295    pub fn is_keyword(&self) -> bool {
296        matches!(
297            self,
298            Self::If
299                | Self::Unless
300                | Self::Elsif
301                | Self::Else
302                | Self::Case
303                | Self::When
304                | Self::Then
305                | Self::For
306                | Self::While
307                | Self::Until
308                | Self::Break
309                | Self::Next
310                | Self::Redo
311                | Self::Retry
312                | Self::Return
313                | Self::Yield
314                | Self::Def
315                | Self::Class
316                | Self::Module
317                | Self::End
318                | Self::Lambda
319                | Self::Proc
320                | Self::Begin
321                | Self::Rescue
322                | Self::Ensure
323                | Self::Raise
324                | Self::Require
325                | Self::Load
326                | Self::Include
327                | Self::Extend
328                | Self::Prepend
329                | Self::And
330                | Self::Or
331                | Self::Not
332                | Self::In
333                | Self::True
334                | Self::False
335                | Self::Nil
336                | Self::Super
337                | Self::Self_
338                | Self::Alias
339                | Self::Undef
340                | Self::Defined
341                | Self::Do
342        )
343    }
344}
345
346impl fmt::Display for RubyTokenType {
347    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
348        let name = match self {
349            Self::Identifier => "Identifier",
350            Self::GlobalVariable => "GlobalVariable",
351            Self::InstanceVariable => "InstanceVariable",
352            Self::ClassVariable => "ClassVariable",
353            Self::Constant => "Constant",
354            Self::IntegerLiteral => "IntegerLiteral",
355            Self::FloatLiteral => "FloatLiteral",
356            Self::StringLiteral => "StringLiteral",
357            Self::Literal => "Literal",
358            Self::Symbol => "Symbol",
359            Self::RegexLiteral => "RegexLiteral",
360
361            Self::If => "If",
362            Self::Unless => "Unless",
363            Self::Elsif => "Elsif",
364            Self::Else => "Else",
365            Self::Case => "Case",
366            Self::When => "When",
367            Self::Then => "Then",
368            Self::For => "For",
369            Self::While => "While",
370            Self::Until => "Until",
371            Self::Break => "Break",
372            Self::Next => "Next",
373            Self::Redo => "Redo",
374            Self::Retry => "Retry",
375            Self::Return => "Return",
376            Self::Yield => "Yield",
377            Self::Def => "Def",
378            Self::Class => "Class",
379            Self::Module => "Module",
380            Self::End => "End",
381            Self::Lambda => "Lambda",
382            Self::Proc => "Proc",
383            Self::Begin => "Begin",
384            Self::Rescue => "Rescue",
385            Self::Ensure => "Ensure",
386            Self::Raise => "Raise",
387            Self::Require => "Require",
388            Self::Load => "Load",
389            Self::Include => "Include",
390            Self::Extend => "Extend",
391            Self::Prepend => "Prepend",
392            Self::And => "And",
393            Self::Or => "Or",
394            Self::Not => "Not",
395            Self::In => "In",
396            Self::True => "True",
397            Self::False => "False",
398            Self::Nil => "Nil",
399            Self::Super => "Super",
400            Self::Self_ => "Self",
401            Self::Alias => "Alias",
402            Self::Undef => "Undef",
403            Self::Defined => "Defined",
404            Self::Do => "Do",
405
406            Self::Plus => "Plus",
407            Self::Minus => "Minus",
408            Self::Multiply => "Multiply",
409            Self::Divide => "Divide",
410            Self::Modulo => "Modulo",
411            Self::Power => "Power",
412            Self::EqualEqual => "EqualEqual",
413            Self::EqualGreater => "EqualGreater",
414            Self::NotEqual => "NotEqual",
415            Self::Less => "Less",
416            Self::Greater => "Greater",
417            Self::LessEqual => "LessEqual",
418            Self::GreaterEqual => "GreaterEqual",
419            Self::EqualEqualEqual => "EqualEqualEqual",
420            Self::Spaceship => "Spaceship",
421            Self::Assign => "Assign",
422            Self::PlusAssign => "PlusAssign",
423            Self::MinusAssign => "MinusAssign",
424            Self::MultiplyAssign => "MultiplyAssign",
425            Self::DivideAssign => "DivideAssign",
426            Self::ModuloAssign => "ModuloAssign",
427            Self::PowerAssign => "PowerAssign",
428            Self::BitAnd => "BitAnd",
429            Self::BitOr => "BitOr",
430            Self::Xor => "Xor",
431            Self::LogicalNot => "LogicalNot",
432            Self::Tilde => "Tilde",
433            Self::LeftShift => "LeftShift",
434            Self::RightShift => "RightShift",
435            Self::AndAssign => "AndAssign",
436            Self::OrAssign => "OrAssign",
437            Self::XorAssign => "XorAssign",
438            Self::LeftShiftAssign => "LeftShiftAssign",
439            Self::RightShiftAssign => "RightShiftAssign",
440            Self::AndAnd => "AndAnd",
441            Self::OrOr => "OrOr",
442            Self::OrOrAssign => "OrOrAssign",
443            Self::AndAndAssign => "AndAndAssign",
444            Self::Question => "Question",
445            Self::DotDot => "DotDot",
446            Self::DotDotDot => "DotDotDot",
447            Self::Match => "Match",
448            Self::NotMatch => "NotMatch",
449
450            Self::LeftParen => "LeftParen",
451            Self::RightParen => "RightParen",
452            Self::LeftBracket => "LeftBracket",
453            Self::RightBracket => "RightBracket",
454            Self::LeftBrace => "LeftBrace",
455            Self::RightBrace => "RightBrace",
456            Self::Comma => "Comma",
457            Self::Colon => "Colon",
458            Self::Semicolon => "Semicolon",
459            Self::Dot => "Dot",
460            Self::DoubleColon => "DoubleColon",
461            Self::At => "At",
462            Self::Dollar => "Dollar",
463
464            Self::Whitespace => "Whitespace",
465            Self::Newline => "Newline",
466            Self::Comment => "Comment",
467            Self::Eof => "Eof",
468            Self::Invalid => "Invalid",
469            Self::Root => "Root",
470            Self::BinaryExpression => "BinaryExpression",
471            Self::UnaryExpression => "UnaryExpression",
472            Self::LiteralExpression => "LiteralExpression",
473            Self::ParenExpression => "ParenExpression",
474            Self::ParenthesizedExpression => "ParenthesizedExpression",
475            Self::MethodDefinition => "MethodDefinition",
476            Self::ClassDefinition => "ClassDefinition",
477            Self::ModuleDefinition => "ModuleDefinition",
478            Self::IfStatement => "IfStatement",
479            Self::WhileStatement => "WhileStatement",
480            Self::ReturnStatement => "ReturnStatement",
481            Self::IfExpression => "IfExpression",
482            Self::CallExpression => "CallExpression",
483            Self::MemberAccess => "MemberAccess",
484            Self::ParameterList => "ParameterList",
485            Self::ArgumentList => "ArgumentList",
486            Self::Error => "Error",
487            Self::Equal => "Equal",
488        };
489        write!(f, "{}", name)
490    }
491}
492
493impl TokenType for RubyTokenType {
494    type Role = UniversalTokenRole;
495    const END_OF_STREAM: Self = Self::Error;
496
497    fn is_ignored(&self) -> bool {
498        false
499    }
500
501    fn role(&self) -> Self::Role {
502        match self {
503            _ => UniversalTokenRole::None,
504        }
505    }
506}
oak_ruby/lexer/token_type.rs

oak_ruby/lexer/
token_type.rs
