oak_matlab/lexer/
token_type.rs

1use oak_core::{Token, TokenType, UniversalTokenRole};
2
3/// Token for the Matlab language.
4pub type MatlabToken = Token<MatlabTokenType>;
5
6impl MatlabTokenType {
7    /// Returns true if the token type is a token.
8    pub fn is_token(&self) -> bool {
9        !self.is_element()
10    }
11
12    /// Returns true if the token type is an element.
13    pub fn is_element(&self) -> bool {
14        matches!(self, Self::Script | Self::FunctionDef | Self::ClassDef | Self::Block | Self::Expression | Self::Statement)
15    }
16}
17
18impl TokenType for MatlabTokenType {
19    type Role = UniversalTokenRole;
20    const END_OF_STREAM: Self = Self::Error;
21
22    fn is_ignored(&self) -> bool {
23        false
24    }
25
26    fn role(&self) -> Self::Role {
27        match self {
28            _ => UniversalTokenRole::None,
29        }
30    }
31}
32
33/// Token types for the Matlab language.
34#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
35#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
36#[repr(u8)]
37pub enum MatlabTokenType {
38    // Basic tokens
39    /// Whitespace.
40    Whitespace,
41    /// Newline.
42    Newline,
43    /// Line comment.
44    Comment,
45    /// Block comment.
46    BlockComment,
47
48    // Identifiers and literals
49    /// Identifier.
50    Identifier,
51    /// Number literal.
52    Number,
53    /// String literal.
54    String,
55    /// Character literal.
56    Character,
57
58    // Keywords
59    /// `function` keyword.
60    Function,
61    /// `end` keyword.
62    End,
63    /// `if` keyword.
64    If,
65    /// `else` keyword.
66    Else,
67    /// `elseif` keyword.
68    Elseif,
69    /// `while` keyword.
70    While,
71    /// `for` keyword.
72    For,
73    /// `break` keyword.
74    Break,
75    /// `continue` keyword.
76    Continue,
77    /// `return` keyword.
78    Return,
79    /// `switch` keyword.
80    Switch,
81    /// `case` keyword.
82    Case,
83    /// `otherwise` keyword.
84    Otherwise,
85    /// `try` keyword.
86    Try,
87    /// `catch` keyword.
88    Catch,
89    /// `global` keyword.
90    Global,
91    /// `persistent` keyword.
92    Persistent,
93    /// `classdef` keyword.
94    Classdef,
95    /// `properties` keyword.
96    Properties,
97    /// `methods` keyword.
98    Methods,
99    /// `events` keyword.
100    Events,
101
102    // Operators
103    /// `+` operator.
104    Plus, // +
105    /// `-` operator.
106    Minus, // -
107    /// `*` operator.
108    Times, // *
109    /// `/` operator.
110    Divide, // /
111    /// `^` operator.
112    Power, // ^
113    /// `\` operator.
114    LeftDivide, // \
115    /// `.*` operator.
116    DotTimes, // .*
117    /// `./` operator.
118    DotDivide, // ./
119    /// `.^` operator.
120    DotPower, // .^
121    /// `.\` operator.
122    DotLeftDivide, // .\
123
124    // Comparison operators
125    /// `==` operator.
126    Equal, // ==
127    /// `~=` operator.
128    NotEqual, // ~=
129    /// `<` operator.
130    Less, // <
131    /// `>` operator.
132    Greater, // >
133    /// `<=` operator.
134    LessEqual, // <=
135    /// `>=` operator.
136    GreaterEqual, // >=
137
138    // Logical operators
139    /// `&` operator.
140    And, // &
141    /// `|` operator.
142    Or, // |
143    /// `~` operator.
144    Not, // ~
145    /// `&&` operator.
146    AndAnd, // &&
147    /// `||` operator.
148    OrOr, // ||
149
150    // Assignment operators
151    /// `=` operator.
152    Assign, // =
153
154    // Delimiters
155    /// `(` delimiter.
156    LeftParen, // (
157    /// `)` delimiter.
158    RightParen, // )
159    /// `[` delimiter.
160    LeftBracket, // [
161    /// `]` delimiter.
162    RightBracket, // ]
163    /// `{` delimiter.
164    LeftBrace, // {
165    /// `}` delimiter.
166    RightBrace, // }
167    /// `;` delimiter.
168    Semicolon, // ;
169    /// `,` delimiter.
170    Comma, // ,
171    /// `.` delimiter.
172    Dot, // .
173    /// `:` delimiter.
174    Colon, // :
175    /// `?` delimiter.
176    Question, // ?
177    /// `@` delimiter.
178    At, // @
179
180    // Special operators
181    /// `'` operator.
182    Transpose, // '
183    /// `.'` operator.
184    DotTranspose, // .'
185
186    // Generalized types
187    /// General operator.
188    Operator,
189    /// General delimiter.
190    Delimiter,
191
192    // Error handling
193    /// Error token.
194    Error,
195
196    // Document structure
197    /// Script element.
198    Script,
199    /// Function definition element.
200    FunctionDef,
201    /// Class definition element.
202    ClassDef,
203    /// Block element.
204    Block,
205    /// Expression element.
206    Expression,
207    /// Statement element.
208    Statement,
209
210    // EOF
211    /// End of stream.
212    Eof,
213}
oak_matlab/lexer/token_type.rs

oak_matlab/lexer/
token_type.rs
