oak_delphi/lexer/
token_type.rs

1use oak_core::{Token, TokenType, UniversalTokenRole};
2
3pub type _DelphiToken = Token<DelphiTokenType>;
4
5impl From<crate::parser::element_type::DelphiElementType> for DelphiTokenType {
6    fn from(element: crate::parser::element_type::DelphiElementType) -> Self {
7        use crate::parser::element_type::DelphiElementType as E;
8        match element {
9            E::Root => Self::Root,
10            E::Identifier => Self::Identifier,
11            E::String => Self::String,
12            E::Number => Self::Number,
13            E::Float => Self::Float,
14            E::Whitespace => Self::Whitespace,
15            E::Newline => Self::Newline,
16            E::Program => Self::Program,
17            E::Unit => Self::Unit,
18            E::Interface => Self::Interface,
19            E::Implementation => Self::Implementation,
20            E::Uses => Self::Uses,
21            E::Type => Self::Type,
22            E::Var => Self::Var,
23            E::Const => Self::Const,
24            E::Function => Self::Function,
25            E::Procedure => Self::Procedure,
26            E::Begin => Self::Begin,
27            E::End => Self::End,
28            E::If => Self::If,
29            E::Then => Self::Then,
30            E::Else => Self::Else,
31            E::While => Self::While,
32            E::Do => Self::Do,
33            E::For => Self::For,
34            E::To => Self::To,
35            E::Downto => Self::Downto,
36            E::Repeat => Self::Repeat,
37            E::Until => Self::Until,
38            E::Case => Self::Case,
39            E::Of => Self::Of,
40            E::With => Self::With,
41            E::Try => Self::Try,
42            E::Except => Self::Except,
43            E::Finally => Self::Finally,
44            E::Raise => Self::Raise,
45            E::Class => Self::Class,
46            E::Object => Self::Object,
47            E::Record => Self::Record,
48            E::Array => Self::Array,
49            E::Set => Self::Set,
50            E::File => Self::File,
51            E::Packed => Self::Packed,
52            E::String_ => Self::String_,
53            E::Integer => Self::Integer,
54            E::Real => Self::Real,
55            E::Boolean => Self::Boolean,
56            E::Char => Self::Char,
57            E::Pointer => Self::Pointer,
58            E::Nil => Self::Nil,
59            E::True_ => Self::True_,
60            E::False_ => Self::False_,
61            E::And_ => Self::And_,
62            E::Or_ => Self::Or_,
63            E::Not_ => Self::Not_,
64            E::Div => Self::Div,
65            E::Mod => Self::Mod,
66            E::In_ => Self::In_,
67            E::Is_ => Self::Is_,
68            E::As_ => Self::As_,
69            E::Plus => Self::Plus,
70            E::Minus => Self::Minus,
71            E::Star => Self::Star,
72            E::Slash => Self::Slash,
73            E::Equal => Self::Equal,
74            E::NotEqual => Self::NotEqual,
75            E::Less => Self::Less,
76            E::Greater => Self::Greater,
77            E::LessEqual => Self::LessEqual,
78            E::GreaterEqual => Self::GreaterEqual,
79            E::Assign => Self::Assign,
80            E::Dot => Self::Dot,
81            E::DotDot => Self::DotDot,
82            E::Caret => Self::Caret,
83            E::At => Self::At,
84            E::LeftParen => Self::LeftParen,
85            E::RightParen => Self::RightParen,
86            E::LeftBracket => Self::LeftBracket,
87            E::RightBracket => Self::RightBracket,
88            E::Semicolon => Self::Semicolon,
89            E::Comma => Self::Comma,
90            E::Colon => Self::Colon,
91            E::Comment => Self::Comment,
92            E::LineComment => Self::LineComment,
93            E::BlockComment => Self::BlockComment,
94            E::Error => Self::Error,
95            E::Eof => Self::Eof,
96        }
97    }
98}
99
100impl DelphiTokenType {
101    /// Returns true if this syntax kind is a Delphi keyword
102    pub fn is_keyword(&self) -> bool {
103        matches!(
104            self,
105            Self::Program
106                | Self::Unit
107                | Self::Interface
108                | Self::Implementation
109                | Self::Uses
110                | Self::Type
111                | Self::Var
112                | Self::Const
113                | Self::Function
114                | Self::Procedure
115                | Self::Begin
116                | Self::End
117                | Self::If
118                | Self::Then
119                | Self::Else
120                | Self::While
121                | Self::Do
122                | Self::For
123                | Self::To
124                | Self::Downto
125                | Self::Repeat
126                | Self::Until
127                | Self::Case
128                | Self::Of
129                | Self::With
130                | Self::Try
131                | Self::Except
132                | Self::Finally
133                | Self::Raise
134                | Self::Class
135                | Self::Object
136                | Self::Record
137                | Self::Array
138                | Self::Set
139                | Self::File
140                | Self::Packed
141                | Self::String_
142                | Self::Integer
143                | Self::Real
144                | Self::Boolean
145                | Self::Char
146                | Self::Pointer
147                | Self::Nil
148                | Self::True_
149                | Self::False_
150                | Self::And_
151                | Self::Or_
152                | Self::Not_
153                | Self::Div
154                | Self::Mod
155                | Self::In_
156                | Self::Is_
157                | Self::As_
158        )
159    }
160}
161
162impl TokenType for DelphiTokenType {
163    type Role = UniversalTokenRole;
164    const END_OF_STREAM: Self = Self::Eof;
165
166    fn is_ignored(&self) -> bool {
167        matches!(self, Self::Whitespace | Self::Newline | Self::Comment | Self::LineComment | Self::BlockComment)
168    }
169
170    fn role(&self) -> Self::Role {
171        if self.is_keyword() {
172            return UniversalTokenRole::Keyword;
173        }
174        match self {
175            Self::Identifier => UniversalTokenRole::Name,
176            Self::String | Self::Number | Self::Float | Self::True_ | Self::False_ | Self::Nil => UniversalTokenRole::Literal,
177            Self::Plus | Self::Minus | Self::Star | Self::Slash | Self::Equal | Self::NotEqual | Self::Less | Self::Greater | Self::LessEqual | Self::GreaterEqual | Self::Assign | Self::Caret | Self::At => UniversalTokenRole::Operator,
178            Self::Dot | Self::DotDot | Self::LeftParen | Self::RightParen | Self::LeftBracket | Self::RightBracket | Self::Semicolon | Self::Comma | Self::Colon => UniversalTokenRole::Punctuation,
179            Self::Comment | Self::LineComment | Self::BlockComment => UniversalTokenRole::Comment,
180            Self::Whitespace | Self::Newline => UniversalTokenRole::Whitespace,
181            Self::Error => UniversalTokenRole::Error,
182            Self::Eof => UniversalTokenRole::Eof,
183            _ => UniversalTokenRole::None,
184        }
185    }
186}
187
188/// Represents the different types of tokens in the Delphi language.
189#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
190#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
191pub enum DelphiTokenType {
192    /// Root node.
193    Root,
194    // Basic tokens
195    /// Identifier.
196    Identifier,
197    /// String literal.
198    String,
199    /// Number literal.
200    Number,
201    /// Floating point literal.
202    Float,
203    /// Whitespace.
204    Whitespace,
205    /// Newline.
206    Newline,
207
208    // Delphi keywords
209    /// `program` keyword.
210    Program,
211    /// `unit` keyword.
212    Unit,
213    /// `interface` keyword.
214    Interface,
215    /// `implementation` keyword.
216    Implementation,
217    /// `uses` keyword.
218    Uses,
219    /// `type` keyword.
220    Type,
221    /// `var` keyword.
222    Var,
223    /// `const` keyword.
224    Const,
225    /// `function` keyword.
226    Function,
227    /// `procedure` keyword.
228    Procedure,
229    /// `begin` keyword.
230    Begin,
231    /// `end` keyword.
232    End,
233    /// `if` keyword.
234    If,
235    /// `then` keyword.
236    Then,
237    /// `else` keyword.
238    Else,
239    /// `while` keyword.
240    While,
241    /// `do` keyword.
242    Do,
243    /// `for` keyword.
244    For,
245    /// `to` keyword.
246    To,
247    /// `downto` keyword.
248    Downto,
249    /// `repeat` keyword.
250    Repeat,
251    /// `until` keyword.
252    Until,
253    /// `case` keyword.
254    Case,
255    /// `of` keyword.
256    Of,
257    /// `with` keyword.
258    With,
259    /// `try` keyword.
260    Try,
261    /// `except` keyword.
262    Except,
263    /// `finally` keyword.
264    Finally,
265    /// `raise` keyword.
266    Raise,
267    /// `class` keyword.
268    Class,
269    /// `object` keyword.
270    Object,
271    /// `record` keyword.
272    Record,
273    /// `array` keyword.
274    Array,
275    /// `set` keyword.
276    Set,
277    /// `file` keyword.
278    File,
279    /// `packed` keyword.
280    Packed,
281    /// `string` keyword/type.
282    String_,
283    /// `integer` keyword/type.
284    Integer,
285    /// `real` keyword/type.
286    Real,
287    /// `boolean` keyword/type.
288    Boolean,
289    /// `char` keyword/type.
290    Char,
291    /// `pointer` keyword/type.
292    Pointer,
293    /// `nil` keyword.
294    Nil,
295    /// `true` literal.
296    True_,
297    /// `false` literal.
298    False_,
299    /// `and` keyword/operator.
300    And_,
301    /// `or` keyword/operator.
302    Or_,
303    /// `not` keyword/operator.
304    Not_,
305    /// `div` keyword/operator.
306    Div,
307    /// `mod` keyword/operator.
308    Mod,
309    /// `in` keyword/operator.
310    In_,
311    /// `is` keyword/operator.
312    Is_,
313    /// `as` keyword/operator.
314    As_,
315
316    // Operators
317    /// Plus `+`.
318    Plus,
319    /// Minus `-`.
320    Minus,
321    /// Star `*`.
322    Star,
323    /// Slash `/`.
324    Slash,
325    /// Equal `=`.
326    Equal,
327    /// Not equal `<>`.
328    NotEqual,
329    /// Less than `<`.
330    Less,
331    /// Greater than `>`.
332    Greater,
333    /// Less than or equal `<=`.
334    LessEqual,
335    /// Greater than or equal `>=`.
336    GreaterEqual,
337    /// Assignment `:=`.
338    Assign,
339    /// Dot `.`.
340    Dot,
341    /// Range `..`.
342    DotDot,
343    /// Caret `^`.
344    Caret,
345    /// At symbol `@`.
346    At,
347
348    // Separators
349    /// Left parenthesis `(`.
350    LeftParen,
351    /// Right parenthesis `)`.
352    RightParen,
353    /// Left bracket `[`.
354    LeftBracket,
355    /// Right bracket `]`.
356    RightBracket,
357    /// Semicolon `;`.
358    Semicolon,
359    /// Comma `,`.
360    Comma,
361    /// Colon `:`.
362    Colon,
363
364    // Comments
365    /// Comment.
366    Comment,
367    /// Line comment `//`.
368    LineComment,
369    /// Block comment `{ ... }` or `(* ... *)`.
370    BlockComment,
371
372    // Special
373    /// Error node.
374    Error,
375    /// End of file.
376    Eof,
377}
oak_delphi/lexer/token_type.rs

oak_delphi/lexer/
token_type.rs
