oak_delphi/lexer/
token_type.rs

1use oak_core::{Token, TokenType, UniversalTokenRole};
2#[cfg(feature = "serde")]
3use serde::{Deserialize, Serialize};
4
5pub type _DelphiToken = Token<DelphiTokenType>;
6
7impl From<crate::parser::element_type::DelphiElementType> for DelphiTokenType {
8    fn from(element: crate::parser::element_type::DelphiElementType) -> Self {
9        use crate::parser::element_type::DelphiElementType as E;
10        match element {
11            E::Root => Self::Root,
12            E::Identifier => Self::Identifier,
13            E::String => Self::String,
14            E::Number => Self::Number,
15            E::Float => Self::Float,
16            E::Whitespace => Self::Whitespace,
17            E::Newline => Self::Newline,
18            E::Program => Self::Program,
19            E::Unit => Self::Unit,
20            E::Interface => Self::Interface,
21            E::Implementation => Self::Implementation,
22            E::Uses => Self::Uses,
23            E::Type => Self::Type,
24            E::Var => Self::Var,
25            E::Const => Self::Const,
26            E::Function => Self::Function,
27            E::Procedure => Self::Procedure,
28            E::Begin => Self::Begin,
29            E::End => Self::End,
30            E::If => Self::If,
31            E::Then => Self::Then,
32            E::Else => Self::Else,
33            E::While => Self::While,
34            E::Do => Self::Do,
35            E::For => Self::For,
36            E::To => Self::To,
37            E::Downto => Self::Downto,
38            E::Repeat => Self::Repeat,
39            E::Until => Self::Until,
40            E::Case => Self::Case,
41            E::Of => Self::Of,
42            E::With => Self::With,
43            E::Try => Self::Try,
44            E::Except => Self::Except,
45            E::Finally => Self::Finally,
46            E::Raise => Self::Raise,
47            E::Class => Self::Class,
48            E::Object => Self::Object,
49            E::Record => Self::Record,
50            E::Array => Self::Array,
51            E::Set => Self::Set,
52            E::File => Self::File,
53            E::Packed => Self::Packed,
54            E::String_ => Self::String_,
55            E::Integer => Self::Integer,
56            E::Real => Self::Real,
57            E::Boolean => Self::Boolean,
58            E::Char => Self::Char,
59            E::Pointer => Self::Pointer,
60            E::Nil => Self::Nil,
61            E::True_ => Self::True_,
62            E::False_ => Self::False_,
63            E::And_ => Self::And_,
64            E::Or_ => Self::Or_,
65            E::Not_ => Self::Not_,
66            E::Div => Self::Div,
67            E::Mod => Self::Mod,
68            E::In_ => Self::In_,
69            E::Is_ => Self::Is_,
70            E::As_ => Self::As_,
71            E::Plus => Self::Plus,
72            E::Minus => Self::Minus,
73            E::Star => Self::Star,
74            E::Slash => Self::Slash,
75            E::Equal => Self::Equal,
76            E::NotEqual => Self::NotEqual,
77            E::Less => Self::Less,
78            E::Greater => Self::Greater,
79            E::LessEqual => Self::LessEqual,
80            E::GreaterEqual => Self::GreaterEqual,
81            E::Assign => Self::Assign,
82            E::Dot => Self::Dot,
83            E::DotDot => Self::DotDot,
84            E::Caret => Self::Caret,
85            E::At => Self::At,
86            E::LeftParen => Self::LeftParen,
87            E::RightParen => Self::RightParen,
88            E::LeftBracket => Self::LeftBracket,
89            E::RightBracket => Self::RightBracket,
90            E::Semicolon => Self::Semicolon,
91            E::Comma => Self::Comma,
92            E::Colon => Self::Colon,
93            E::Comment => Self::Comment,
94            E::LineComment => Self::LineComment,
95            E::BlockComment => Self::BlockComment,
96            E::Error => Self::Error,
97            E::Eof => Self::Eof,
98        }
99    }
100}
101
102impl DelphiTokenType {
103    /// Returns true if this syntax kind is a Delphi keyword
104    pub fn is_keyword(&self) -> bool {
105        matches!(
106            self,
107            Self::Program
108                | Self::Unit
109                | Self::Interface
110                | Self::Implementation
111                | Self::Uses
112                | Self::Type
113                | Self::Var
114                | Self::Const
115                | Self::Function
116                | Self::Procedure
117                | Self::Begin
118                | Self::End
119                | Self::If
120                | Self::Then
121                | Self::Else
122                | Self::While
123                | Self::Do
124                | Self::For
125                | Self::To
126                | Self::Downto
127                | Self::Repeat
128                | Self::Until
129                | Self::Case
130                | Self::Of
131                | Self::With
132                | Self::Try
133                | Self::Except
134                | Self::Finally
135                | Self::Raise
136                | Self::Class
137                | Self::Object
138                | Self::Record
139                | Self::Array
140                | Self::Set
141                | Self::File
142                | Self::Packed
143                | Self::String_
144                | Self::Integer
145                | Self::Real
146                | Self::Boolean
147                | Self::Char
148                | Self::Pointer
149                | Self::Nil
150                | Self::True_
151                | Self::False_
152                | Self::And_
153                | Self::Or_
154                | Self::Not_
155                | Self::Div
156                | Self::Mod
157                | Self::In_
158                | Self::Is_
159                | Self::As_
160        )
161    }
162}
163
164impl TokenType for DelphiTokenType {
165    type Role = UniversalTokenRole;
166    const END_OF_STREAM: Self = Self::Eof;
167
168    fn is_ignored(&self) -> bool {
169        matches!(self, Self::Whitespace | Self::Newline | Self::Comment | Self::LineComment | Self::BlockComment)
170    }
171
172    fn role(&self) -> Self::Role {
173        if self.is_keyword() {
174            return UniversalTokenRole::Keyword;
175        }
176        match self {
177            Self::Identifier => UniversalTokenRole::Name,
178            Self::String | Self::Number | Self::Float | Self::True_ | Self::False_ | Self::Nil => UniversalTokenRole::Literal,
179            Self::Plus | Self::Minus | Self::Star | Self::Slash | Self::Equal | Self::NotEqual | Self::Less | Self::Greater | Self::LessEqual | Self::GreaterEqual | Self::Assign | Self::Caret | Self::At => UniversalTokenRole::Operator,
180            Self::Dot | Self::DotDot | Self::LeftParen | Self::RightParen | Self::LeftBracket | Self::RightBracket | Self::Semicolon | Self::Comma | Self::Colon => UniversalTokenRole::Punctuation,
181            Self::Comment | Self::LineComment | Self::BlockComment => UniversalTokenRole::Comment,
182            Self::Whitespace | Self::Newline => UniversalTokenRole::Whitespace,
183            Self::Error => UniversalTokenRole::Error,
184            Self::Eof => UniversalTokenRole::Eof,
185            _ => UniversalTokenRole::None,
186        }
187    }
188}
189
190/// Represents the different types of tokens in the Delphi language.
191#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
192#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
193pub enum DelphiTokenType {
194    /// Root node.
195    Root,
196    // Basic tokens
197    /// Identifier.
198    Identifier,
199    /// String literal.
200    String,
201    /// Number literal.
202    Number,
203    /// Floating point literal.
204    Float,
205    /// Whitespace.
206    Whitespace,
207    /// Newline.
208    Newline,
209
210    // Delphi keywords
211    /// `program` keyword.
212    Program,
213    /// `unit` keyword.
214    Unit,
215    /// `interface` keyword.
216    Interface,
217    /// `implementation` keyword.
218    Implementation,
219    /// `uses` keyword.
220    Uses,
221    /// `type` keyword.
222    Type,
223    /// `var` keyword.
224    Var,
225    /// `const` keyword.
226    Const,
227    /// `function` keyword.
228    Function,
229    /// `procedure` keyword.
230    Procedure,
231    /// `begin` keyword.
232    Begin,
233    /// `end` keyword.
234    End,
235    /// `if` keyword.
236    If,
237    /// `then` keyword.
238    Then,
239    /// `else` keyword.
240    Else,
241    /// `while` keyword.
242    While,
243    /// `do` keyword.
244    Do,
245    /// `for` keyword.
246    For,
247    /// `to` keyword.
248    To,
249    /// `downto` keyword.
250    Downto,
251    /// `repeat` keyword.
252    Repeat,
253    /// `until` keyword.
254    Until,
255    /// `case` keyword.
256    Case,
257    /// `of` keyword.
258    Of,
259    /// `with` keyword.
260    With,
261    /// `try` keyword.
262    Try,
263    /// `except` keyword.
264    Except,
265    /// `finally` keyword.
266    Finally,
267    /// `raise` keyword.
268    Raise,
269    /// `class` keyword.
270    Class,
271    /// `object` keyword.
272    Object,
273    /// `record` keyword.
274    Record,
275    /// `array` keyword.
276    Array,
277    /// `set` keyword.
278    Set,
279    /// `file` keyword.
280    File,
281    /// `packed` keyword.
282    Packed,
283    /// `string` keyword/type.
284    String_,
285    /// `integer` keyword/type.
286    Integer,
287    /// `real` keyword/type.
288    Real,
289    /// `boolean` keyword/type.
290    Boolean,
291    /// `char` keyword/type.
292    Char,
293    /// `pointer` keyword/type.
294    Pointer,
295    /// `nil` keyword.
296    Nil,
297    /// `true` literal.
298    True_,
299    /// `false` literal.
300    False_,
301    /// `and` keyword/operator.
302    And_,
303    /// `or` keyword/operator.
304    Or_,
305    /// `not` keyword/operator.
306    Not_,
307    /// `div` keyword/operator.
308    Div,
309    /// `mod` keyword/operator.
310    Mod,
311    /// `in` keyword/operator.
312    In_,
313    /// `is` keyword/operator.
314    Is_,
315    /// `as` keyword/operator.
316    As_,
317
318    // Operators
319    /// Plus `+`.
320    Plus,
321    /// Minus `-`.
322    Minus,
323    /// Star `*`.
324    Star,
325    /// Slash `/`.
326    Slash,
327    /// Equal `=`.
328    Equal,
329    /// Not equal `<>`.
330    NotEqual,
331    /// Less than `<`.
332    Less,
333    /// Greater than `>`.
334    Greater,
335    /// Less than or equal `<=`.
336    LessEqual,
337    /// Greater than or equal `>=`.
338    GreaterEqual,
339    /// Assignment `:=`.
340    Assign,
341    /// Dot `.`.
342    Dot,
343    /// Range `..`.
344    DotDot,
345    /// Caret `^`.
346    Caret,
347    /// At symbol `@`.
348    At,
349
350    // Separators
351    /// Left parenthesis `(`.
352    LeftParen,
353    /// Right parenthesis `)`.
354    RightParen,
355    /// Left bracket `[`.
356    LeftBracket,
357    /// Right bracket `]`.
358    RightBracket,
359    /// Semicolon `;`.
360    Semicolon,
361    /// Comma `,`.
362    Comma,
363    /// Colon `:`.
364    Colon,
365
366    // Comments
367    /// Comment.
368    Comment,
369    /// Line comment `//`.
370    LineComment,
371    /// Block comment `{ ... }` or `(* ... *)`.
372    BlockComment,
373
374    // Special
375    /// Error node.
376    Error,
377    /// End of file.
378    Eof,
379}
oak_delphi/lexer/token_type.rs

oak_delphi/lexer/
token_type.rs
