oak_elixir/lexer/
token_type.rs

1use oak_core::{Token, TokenType, UniversalTokenRole};
2#[cfg(feature = "serde")]
3use serde::{Deserialize, Serialize};
4
5/// Token types for Elixir.
6#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
7#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
8pub enum ElixirTokenType {
9    /// Root node.
10    Root,
11    /// Whitespace.
12    Whitespace,
13    /// Newline.
14    Newline,
15    /// Comment.
16    Comment,
17    /// Identifier.
18    Identifier,
19    /// Atom.
20    Atom,
21    /// Variable.
22    Variable,
23    /// Integer number.
24    Number,
25    /// Floating point number.
26    Float,
27    /// String literal.
28    String,
29    /// Character literal.
30    Character,
31    /// Sigil.
32    Sigil,
33
34    /// `after`
35    After,
36    /// `and`
37    And,
38    /// `case`
39    Case,
40    /// `catch`
41    Catch,
42    /// `cond`
43    Cond,
44    /// `def`
45    Def,
46    /// `defp`
47    Defp,
48    /// `defmodule`
49    Defmodule,
50    /// `defstruct`
51    Defstruct,
52    /// `defprotocol`
53    Defprotocol,
54    /// `defimpl`
55    Defimpl,
56    /// `defmacro`
57    Defmacro,
58    /// `defmacrop`
59    Defmacrop,
60    /// `do`
61    Do,
62    /// `else`
63    Else,
64    /// `elsif`
65    Elsif,
66    /// `end`
67    End,
68    /// `false`
69    False,
70    /// `fn`
71    Fn,
72    /// `if`
73    If,
74    /// `in`
75    In,
76    /// `not`
77    Not,
78    /// `or`
79    Or,
80    /// `receive`
81    Receive,
82    /// `rescue`
83    Rescue,
84    /// `true`
85    True,
86    /// `try`
87    Try,
88    /// `unless`
89    Unless,
90    /// `when`
91    When,
92    /// `with`
93    With,
94
95    // Operators
96    /// `+`
97    Plus,
98    /// `-`
99    Minus,
100    /// `*`
101    Mul,
102    /// `/`
103    Div,
104    /// `.`
105    Dot,
106    /// `,`
107    Comma,
108    /// `;`
109    Semicolon,
110    /// `:`
111    Colon,
112    /// `(`
113    LeftParen,
114    /// `)`
115    RightParen,
116    /// `{`
117    LeftBrace,
118    /// `}`
119    RightBrace,
120    /// `[`
121    LeftBracket,
122    /// `]`
123    RightBracket,
124    /// `->`
125    Arrow,
126    /// `|`
127    Pipe,
128
129    // More operators
130    /// `=`
131    Eq,
132    /// `==`
133    EqEq,
134    /// `!=`
135    Ne,
136    /// `<`
137    Lt,
138    /// `<=`
139    Le,
140    /// `>`
141    Gt,
142    /// `>=`
143    Ge,
144    /// `&&`
145    AndAnd,
146    /// `||`
147    OrOr,
148    /// `!`
149    Bang,
150    /// `<>`
151    Concat,
152    /// `++`
153    PlusPlus,
154    /// `--`
155    MinusMinus,
156    /// `|>`
157    Pipeline,
158    /// `<<`
159    LeftDoubleBracket,
160    /// `>>`
161    RightDoubleBracket,
162    /// `@`
163    At,
164    /// `%`
165    Percent,
166
167    /// Error token.
168    Error,
169}
170
171/// Type alias for an Elixir token.
172pub type ElixirToken = Token<ElixirTokenType>;
173
174impl ElixirTokenType {
175    pub fn is_keyword(self) -> bool {
176        matches!(
177            self,
178            Self::After
179                | Self::And
180                | Self::Case
181                | Self::Catch
182                | Self::Cond
183                | Self::Def
184                | Self::Defp
185                | Self::Defmodule
186                | Self::Defstruct
187                | Self::Defprotocol
188                | Self::Defimpl
189                | Self::Defmacro
190                | Self::Defmacrop
191                | Self::Do
192                | Self::Else
193                | Self::Elsif
194                | Self::End
195                | Self::False
196                | Self::Fn
197                | Self::If
198                | Self::In
199                | Self::Not
200                | Self::Or
201                | Self::Receive
202                | Self::Rescue
203                | Self::True
204                | Self::Try
205                | Self::Unless
206                | Self::When
207                | Self::With
208        )
209    }
210}
211
212impl TokenType for ElixirTokenType {
213    type Role = UniversalTokenRole;
214    const END_OF_STREAM: Self = Self::Error;
215
216    fn is_ignored(&self) -> bool {
217        matches!(self, Self::Whitespace | Self::Comment | Self::Newline)
218    }
219
220    fn role(&self) -> Self::Role {
221        match self {
222            Self::Whitespace => UniversalTokenRole::Whitespace,
223            Self::Comment => UniversalTokenRole::Comment,
224            Self::Newline => UniversalTokenRole::Whitespace,
225            Self::Identifier => UniversalTokenRole::Name,
226            Self::Number | Self::Float => UniversalTokenRole::Literal,
227            Self::String | Self::Character => UniversalTokenRole::Literal,
228            Self::Plus
229            | Self::Minus
230            | Self::Mul
231            | Self::Div
232            | Self::Dot
233            | Self::Eq
234            | Self::EqEq
235            | Self::Ne
236            | Self::Lt
237            | Self::Le
238            | Self::Gt
239            | Self::Ge
240            | Self::AndAnd
241            | Self::OrOr
242            | Self::Bang
243            | Self::Concat
244            | Self::PlusPlus
245            | Self::MinusMinus
246            | Self::Pipeline
247            | Self::Arrow
248            | Self::Pipe => UniversalTokenRole::Operator,
249            Self::Comma | Self::Semicolon | Self::Colon | Self::LeftParen | Self::RightParen | Self::LeftBrace | Self::RightBrace | Self::LeftBracket | Self::RightBracket | Self::LeftDoubleBracket | Self::RightDoubleBracket => {
250                UniversalTokenRole::Punctuation
251            }
252            Self::Error => UniversalTokenRole::Error,
253            _ if self.is_keyword() => UniversalTokenRole::Keyword,
254            _ => UniversalTokenRole::None,
255        }
256    }
257}
oak_elixir/lexer/token_type.rs

oak_elixir/lexer/
token_type.rs
