oak_elixir/lexer/
token_type.rs

1use oak_core::{Token, TokenType, UniversalTokenRole};
2#[cfg(feature = "serde")]
3use serde::{Deserialize, Serialize};
4
5#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
6#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
7pub enum ElixirTokenType {
8    Root,
9    Whitespace,
10    Newline,
11    Comment,
12    Identifier,
13    Atom,
14    Variable,
15    Number,
16    Float,
17    String,
18    Character,
19    Sigil,
20
21    After,
22    And,
23    Case,
24    Catch,
25    Cond,
26    Def,
27    Defp,
28    Defmodule,
29    Defstruct,
30    Defprotocol,
31    Defimpl,
32    Defmacro,
33    Defmacrop,
34    Do,
35    Else,
36    Elsif,
37    End,
38    False,
39    Fn,
40    If,
41    In,
42    Not,
43    Or,
44    Receive,
45    Rescue,
46    True,
47    Try,
48    Unless,
49    When,
50    With,
51
52    // Operators
53    Plus,
54    Minus,
55    Mul,
56    Div,
57    Dot,
58    Comma,
59    Semicolon,
60    Colon,
61    LeftParen,
62    RightParen,
63    LeftBrace,
64    RightBrace,
65    LeftBracket,
66    RightBracket,
67    Arrow,
68    Pipe,
69
70    // More operators
71    Eq,
72    EqEq,
73    Ne,
74    Lt,
75    Le,
76    Gt,
77    Ge,
78    AndAnd,
79    OrOr,
80    Bang,
81    Concat,
82    PlusPlus,
83    MinusMinus,
84    Pipeline,
85    LeftDoubleBracket,
86    RightDoubleBracket,
87    At,
88    Percent,
89
90    Error,
91}
92
93pub type ElixirToken = Token<ElixirTokenType>;
94
95impl ElixirTokenType {
96    pub fn is_keyword(self) -> bool {
97        matches!(
98            self,
99            Self::After
100                | Self::And
101                | Self::Case
102                | Self::Catch
103                | Self::Cond
104                | Self::Def
105                | Self::Defp
106                | Self::Defmodule
107                | Self::Defstruct
108                | Self::Defprotocol
109                | Self::Defimpl
110                | Self::Defmacro
111                | Self::Defmacrop
112                | Self::Do
113                | Self::Else
114                | Self::Elsif
115                | Self::End
116                | Self::False
117                | Self::Fn
118                | Self::If
119                | Self::In
120                | Self::Not
121                | Self::Or
122                | Self::Receive
123                | Self::Rescue
124                | Self::True
125                | Self::Try
126                | Self::Unless
127                | Self::When
128                | Self::With
129        )
130    }
131}
132
133impl TokenType for ElixirTokenType {
134    type Role = UniversalTokenRole;
135    const END_OF_STREAM: Self = Self::Error;
136
137    fn is_ignored(&self) -> bool {
138        matches!(self, Self::Whitespace | Self::Comment | Self::Newline)
139    }
140
141    fn role(&self) -> Self::Role {
142        match self {
143            Self::Whitespace => UniversalTokenRole::Whitespace,
144            Self::Comment => UniversalTokenRole::Comment,
145            Self::Newline => UniversalTokenRole::Whitespace,
146            Self::Identifier => UniversalTokenRole::Name,
147            Self::Number | Self::Float => UniversalTokenRole::Literal,
148            Self::String | Self::Character => UniversalTokenRole::Literal,
149            Self::Plus
150            | Self::Minus
151            | Self::Mul
152            | Self::Div
153            | Self::Dot
154            | Self::Eq
155            | Self::EqEq
156            | Self::Ne
157            | Self::Lt
158            | Self::Le
159            | Self::Gt
160            | Self::Ge
161            | Self::AndAnd
162            | Self::OrOr
163            | Self::Bang
164            | Self::Concat
165            | Self::PlusPlus
166            | Self::MinusMinus
167            | Self::Pipeline
168            | Self::Arrow
169            | Self::Pipe => UniversalTokenRole::Operator,
170            Self::Comma | Self::Semicolon | Self::Colon | Self::LeftParen | Self::RightParen | Self::LeftBrace | Self::RightBrace | Self::LeftBracket | Self::RightBracket | Self::LeftDoubleBracket | Self::RightDoubleBracket => {
171                UniversalTokenRole::Punctuation
172            }
173            Self::Error => UniversalTokenRole::Error,
174            _ if self.is_keyword() => UniversalTokenRole::Keyword,
175            _ => UniversalTokenRole::None,
176        }
177    }
178}
oak_elixir/lexer/token_type.rs

oak_elixir/lexer/
token_type.rs
