oak_elixir/lexer/
token_type.rs

1/// Token type definitions for Elixir lexer.
2use oak_core::{Token, TokenType, UniversalTokenRole};
3
4/// Token types for Elixir.
5#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
6#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
7pub enum ElixirTokenType {
8    /// Root node.
9    Root,
10    /// Whitespace.
11    Whitespace,
12    /// Newline.
13    Newline,
14    /// Comment.
15    Comment,
16    /// Identifier.
17    Identifier,
18    /// Atom.
19    Atom,
20    /// Variable.
21    Variable,
22    /// Integer number.
23    Number,
24    /// Floating point number.
25    Float,
26    /// String literal.
27    String,
28    /// Character literal.
29    Character,
30    /// Sigil.
31    Sigil,
32
33    /// `after`
34    After,
35    /// `and`
36    And,
37    /// `case`
38    Case,
39    /// `catch`
40    Catch,
41    /// `cond`
42    Cond,
43    /// `def`
44    Def,
45    /// `defp`
46    Defp,
47    /// `defmodule`
48    Defmodule,
49    /// `defstruct`
50    Defstruct,
51    /// `defprotocol`
52    Defprotocol,
53    /// `defimpl`
54    Defimpl,
55    /// `defmacro`
56    Defmacro,
57    /// `defmacrop`
58    Defmacrop,
59    /// `do`
60    Do,
61    /// `else`
62    Else,
63    /// `elsif`
64    Elsif,
65    /// `end`
66    End,
67    /// `false`
68    False,
69    /// `fn`
70    Fn,
71    /// `if`
72    If,
73    /// `in`
74    In,
75    /// `not`
76    Not,
77    /// `or`
78    Or,
79    /// `receive`
80    Receive,
81    /// `rescue`
82    Rescue,
83    /// `true`
84    True,
85    /// `try`
86    Try,
87    /// `unless`
88    Unless,
89    /// `when`
90    When,
91    /// `with`
92    With,
93
94    // Operators
95    /// `+`
96    Plus,
97    /// `-`
98    Minus,
99    /// `*`
100    Mul,
101    /// `/`
102    Div,
103    /// `.`
104    Dot,
105    /// `,`
106    Comma,
107    /// `;`
108    Semicolon,
109    /// `:`
110    Colon,
111    /// `(`
112    LeftParen,
113    /// `)`
114    RightParen,
115    /// `{`
116    LeftBrace,
117    /// `}`
118    RightBrace,
119    /// `[`
120    LeftBracket,
121    /// `]`
122    RightBracket,
123    /// `->`
124    Arrow,
125    /// `|`
126    Pipe,
127
128    // More operators
129    /// `=`
130    Eq,
131    /// `==`
132    EqEq,
133    /// `!=`
134    Ne,
135    /// `<`
136    Lt,
137    /// `<=`
138    Le,
139    /// `>`
140    Gt,
141    /// `>=`
142    Ge,
143    /// `&&`
144    AndAnd,
145    /// `||`
146    OrOr,
147    /// `!`
148    Bang,
149    /// `<>`
150    Concat,
151    /// `++`
152    PlusPlus,
153    /// `--`
154    MinusMinus,
155    /// `|>`
156    Pipeline,
157    /// `<<`
158    LeftDoubleBracket,
159    /// `>>`
160    RightDoubleBracket,
161    /// `@`
162    At,
163    /// `%`
164    Percent,
165
166    /// Error token.
167    Error,
168}
169
170/// Type alias for an Elixir token.
171pub type ElixirToken = Token<ElixirTokenType>;
172
173impl ElixirTokenType {
174    pub fn is_keyword(self) -> bool {
175        matches!(
176            self,
177            Self::After
178                | Self::And
179                | Self::Case
180                | Self::Catch
181                | Self::Cond
182                | Self::Def
183                | Self::Defp
184                | Self::Defmodule
185                | Self::Defstruct
186                | Self::Defprotocol
187                | Self::Defimpl
188                | Self::Defmacro
189                | Self::Defmacrop
190                | Self::Do
191                | Self::Else
192                | Self::Elsif
193                | Self::End
194                | Self::False
195                | Self::Fn
196                | Self::If
197                | Self::In
198                | Self::Not
199                | Self::Or
200                | Self::Receive
201                | Self::Rescue
202                | Self::True
203                | Self::Try
204                | Self::Unless
205                | Self::When
206                | Self::With
207        )
208    }
209}
210
211impl TokenType for ElixirTokenType {
212    type Role = UniversalTokenRole;
213    const END_OF_STREAM: Self = Self::Error;
214
215    fn is_ignored(&self) -> bool {
216        matches!(self, Self::Whitespace | Self::Comment | Self::Newline)
217    }
218
219    fn role(&self) -> Self::Role {
220        match self {
221            Self::Whitespace => UniversalTokenRole::Whitespace,
222            Self::Comment => UniversalTokenRole::Comment,
223            Self::Newline => UniversalTokenRole::Whitespace,
224            Self::Identifier => UniversalTokenRole::Name,
225            Self::Number | Self::Float => UniversalTokenRole::Literal,
226            Self::String | Self::Character => UniversalTokenRole::Literal,
227            Self::Plus
228            | Self::Minus
229            | Self::Mul
230            | Self::Div
231            | Self::Dot
232            | Self::Eq
233            | Self::EqEq
234            | Self::Ne
235            | Self::Lt
236            | Self::Le
237            | Self::Gt
238            | Self::Ge
239            | Self::AndAnd
240            | Self::OrOr
241            | Self::Bang
242            | Self::Concat
243            | Self::PlusPlus
244            | Self::MinusMinus
245            | Self::Pipeline
246            | Self::Arrow
247            | Self::Pipe => UniversalTokenRole::Operator,
248            Self::Comma | Self::Semicolon | Self::Colon | Self::LeftParen | Self::RightParen | Self::LeftBrace | Self::RightBrace | Self::LeftBracket | Self::RightBracket | Self::LeftDoubleBracket | Self::RightDoubleBracket => {
249                UniversalTokenRole::Punctuation
250            }
251            Self::Error => UniversalTokenRole::Error,
252            _ if self.is_keyword() => UniversalTokenRole::Keyword,
253            _ => UniversalTokenRole::None,
254        }
255    }
256}
oak_elixir/lexer/token_type.rs

oak_elixir/lexer/
token_type.rs
