oak_delphi/lexer/
token_type.rs

1use oak_core::{Token, TokenType, UniversalTokenRole};
2#[cfg(feature = "serde")]
3use serde::{Deserialize, Serialize};
4
5pub type DelphiToken = Token<DelphiTokenType>;
6
7impl From<crate::parser::element_type::DelphiElementType> for DelphiTokenType {
8    fn from(element: crate::parser::element_type::DelphiElementType) -> Self {
9        use crate::parser::element_type::DelphiElementType as E;
10        match element {
11            E::Root => Self::Root,
12            E::Identifier => Self::Identifier,
13            E::String => Self::String,
14            E::Number => Self::Number,
15            E::Float => Self::Float,
16            E::Whitespace => Self::Whitespace,
17            E::Newline => Self::Newline,
18            E::Program => Self::Program,
19            E::Unit => Self::Unit,
20            E::Interface => Self::Interface,
21            E::Implementation => Self::Implementation,
22            E::Uses => Self::Uses,
23            E::Type => Self::Type,
24            E::Var => Self::Var,
25            E::Const => Self::Const,
26            E::Function => Self::Function,
27            E::Procedure => Self::Procedure,
28            E::Begin => Self::Begin,
29            E::End => Self::End,
30            E::If => Self::If,
31            E::Then => Self::Then,
32            E::Else => Self::Else,
33            E::While => Self::While,
34            E::Do => Self::Do,
35            E::For => Self::For,
36            E::To => Self::To,
37            E::Downto => Self::Downto,
38            E::Repeat => Self::Repeat,
39            E::Until => Self::Until,
40            E::Case => Self::Case,
41            E::Of => Self::Of,
42            E::With => Self::With,
43            E::Try => Self::Try,
44            E::Except => Self::Except,
45            E::Finally => Self::Finally,
46            E::Raise => Self::Raise,
47            E::Class => Self::Class,
48            E::Object => Self::Object,
49            E::Record => Self::Record,
50            E::Array => Self::Array,
51            E::Set => Self::Set,
52            E::File => Self::File,
53            E::Packed => Self::Packed,
54            E::String_ => Self::String_,
55            E::Integer => Self::Integer,
56            E::Real => Self::Real,
57            E::Boolean => Self::Boolean,
58            E::Char => Self::Char,
59            E::Pointer => Self::Pointer,
60            E::Nil => Self::Nil,
61            E::True_ => Self::True_,
62            E::False_ => Self::False_,
63            E::And_ => Self::And_,
64            E::Or_ => Self::Or_,
65            E::Not_ => Self::Not_,
66            E::Div => Self::Div,
67            E::Mod => Self::Mod,
68            E::In_ => Self::In_,
69            E::Is_ => Self::Is_,
70            E::As_ => Self::As_,
71            E::Plus => Self::Plus,
72            E::Minus => Self::Minus,
73            E::Star => Self::Star,
74            E::Slash => Self::Slash,
75            E::Equal => Self::Equal,
76            E::NotEqual => Self::NotEqual,
77            E::Less => Self::Less,
78            E::Greater => Self::Greater,
79            E::LessEqual => Self::LessEqual,
80            E::GreaterEqual => Self::GreaterEqual,
81            E::Assign => Self::Assign,
82            E::Dot => Self::Dot,
83            E::DotDot => Self::DotDot,
84            E::Caret => Self::Caret,
85            E::At => Self::At,
86            E::LeftParen => Self::LeftParen,
87            E::RightParen => Self::RightParen,
88            E::LeftBracket => Self::LeftBracket,
89            E::RightBracket => Self::RightBracket,
90            E::Semicolon => Self::Semicolon,
91            E::Comma => Self::Comma,
92            E::Colon => Self::Colon,
93            E::Comment => Self::Comment,
94            E::LineComment => Self::LineComment,
95            E::BlockComment => Self::BlockComment,
96            E::Error => Self::Error,
97            E::Eof => Self::Eof,
98        }
99    }
100}
101
102impl DelphiTokenType {
103    /// Returns true if this syntax kind is a Delphi keyword
104    pub fn is_keyword(&self) -> bool {
105        matches!(
106            self,
107            Self::Program
108                | Self::Unit
109                | Self::Interface
110                | Self::Implementation
111                | Self::Uses
112                | Self::Type
113                | Self::Var
114                | Self::Const
115                | Self::Function
116                | Self::Procedure
117                | Self::Begin
118                | Self::End
119                | Self::If
120                | Self::Then
121                | Self::Else
122                | Self::While
123                | Self::Do
124                | Self::For
125                | Self::To
126                | Self::Downto
127                | Self::Repeat
128                | Self::Until
129                | Self::Case
130                | Self::Of
131                | Self::With
132                | Self::Try
133                | Self::Except
134                | Self::Finally
135                | Self::Raise
136                | Self::Class
137                | Self::Object
138                | Self::Record
139                | Self::Array
140                | Self::Set
141                | Self::File
142                | Self::Packed
143                | Self::String_
144                | Self::Integer
145                | Self::Real
146                | Self::Boolean
147                | Self::Char
148                | Self::Pointer
149                | Self::Nil
150                | Self::True_
151                | Self::False_
152                | Self::And_
153                | Self::Or_
154                | Self::Not_
155                | Self::Div
156                | Self::Mod
157                | Self::In_
158                | Self::Is_
159                | Self::As_
160        )
161    }
162}
163
164impl TokenType for DelphiTokenType {
165    type Role = UniversalTokenRole;
166    const END_OF_STREAM: Self = Self::Eof;
167
168    fn is_ignored(&self) -> bool {
169        matches!(self, Self::Whitespace | Self::Newline | Self::Comment | Self::LineComment | Self::BlockComment)
170    }
171
172    fn role(&self) -> Self::Role {
173        if self.is_keyword() {
174            return UniversalTokenRole::Keyword;
175        }
176        match self {
177            Self::Identifier => UniversalTokenRole::Name,
178            Self::String | Self::Number | Self::Float | Self::True_ | Self::False_ | Self::Nil => UniversalTokenRole::Literal,
179            Self::Plus | Self::Minus | Self::Star | Self::Slash | Self::Equal | Self::NotEqual | Self::Less | Self::Greater | Self::LessEqual | Self::GreaterEqual | Self::Assign | Self::Caret | Self::At => UniversalTokenRole::Operator,
180            Self::Dot | Self::DotDot | Self::LeftParen | Self::RightParen | Self::LeftBracket | Self::RightBracket | Self::Semicolon | Self::Comma | Self::Colon => UniversalTokenRole::Punctuation,
181            Self::Comment | Self::LineComment | Self::BlockComment => UniversalTokenRole::Comment,
182            Self::Whitespace | Self::Newline => UniversalTokenRole::Whitespace,
183            Self::Error => UniversalTokenRole::Error,
184            Self::Eof => UniversalTokenRole::Eof,
185            _ => UniversalTokenRole::None,
186        }
187    }
188}
189
190#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
191#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
192pub enum DelphiTokenType {
193    Root,
194    // Basic tokens
195    Identifier,
196    String,
197    Number,
198    Float,
199    Whitespace,
200    Newline,
201
202    // Delphi keywords
203    Program,
204    Unit,
205    Interface,
206    Implementation,
207    Uses,
208    Type,
209    Var,
210    Const,
211    Function,
212    Procedure,
213    Begin,
214    End,
215    If,
216    Then,
217    Else,
218    While,
219    Do,
220    For,
221    To,
222    Downto,
223    Repeat,
224    Until,
225    Case,
226    Of,
227    With,
228    Try,
229    Except,
230    Finally,
231    Raise,
232    Class,
233    Object,
234    Record,
235    Array,
236    Set,
237    File,
238    Packed,
239    String_,
240    Integer,
241    Real,
242    Boolean,
243    Char,
244    Pointer,
245    Nil,
246    True_,
247    False_,
248    And_,
249    Or_,
250    Not_,
251    Div,
252    Mod,
253    In_,
254    Is_,
255    As_,
256
257    // Operators
258    Plus,
259    Minus,
260    Star,
261    Slash,
262    Equal,
263    NotEqual,
264    Less,
265    Greater,
266    LessEqual,
267    GreaterEqual,
268    Assign,
269    Dot,
270    DotDot,
271    Caret,
272    At,
273
274    // Separators
275    LeftParen,
276    RightParen,
277    LeftBracket,
278    RightBracket,
279    Semicolon,
280    Comma,
281    Colon,
282
283    // Comments
284    Comment,
285    LineComment,
286    BlockComment,
287
288    // Special
289    Error,
290    Eof,
291}
oak_delphi/lexer/token_type.rs

oak_delphi/lexer/
token_type.rs
