oak_pascal/lexer/
token_type.rs

1use oak_core::{Token, TokenType, UniversalTokenRole};
2
3/// Token types for Pascal.
4#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
5#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
6pub enum PascalTokenType {
7    /// Whitespace characters.
8    Whitespace,
9    /// Newline characters.
10    Newline,
11
12    /// Comments.
13    Comment,
14
15    /// `program` keyword.
16    Program,
17    /// `begin` keyword.
18    Begin,
19    /// `end` keyword.
20    End,
21    /// `var` keyword.
22    Var,
23    /// `const` keyword.
24    Const,
25    /// `type` keyword.
26    Type,
27    /// `function` keyword.
28    Function,
29    /// `procedure` keyword.
30    Procedure,
31    /// `if` keyword.
32    If,
33    /// `then` keyword.
34    Then,
35    /// `else` keyword.
36    Else,
37    /// `while` keyword.
38    While,
39    /// `do` keyword.
40    Do,
41    /// `for` keyword.
42    For,
43    /// `to` keyword.
44    To,
45    /// `downto` keyword.
46    Downto,
47    /// `repeat` keyword.
48    Repeat,
49    /// `until` keyword.
50    Until,
51    /// `case` keyword.
52    Case,
53    /// `of` keyword.
54    Of,
55    /// `with` keyword.
56    With,
57    /// `record` keyword.
58    Record,
59    /// `array` keyword.
60    Array,
61    /// `set` keyword.
62    Set,
63    /// `file` keyword.
64    File,
65    /// `packed` keyword.
66    Packed,
67    /// `nil` keyword.
68    Nil,
69    /// `true` keyword.
70    True,
71    /// `false` keyword.
72    False,
73    /// `and` keyword.
74    And,
75    /// `or` keyword.
76    Or,
77    /// `not` keyword.
78    Not,
79    /// `div` keyword.
80    Div,
81    /// `mod` keyword.
82    Mod,
83    /// `in` keyword.
84    In,
85
86    /// An identifier.
87    Identifier,
88    /// An integer literal.
89    IntegerLiteral,
90    /// A real number literal.
91    RealLiteral,
92    /// A string literal.
93    StringLiteral,
94    /// A character literal.
95    CharLiteral,
96
97    /// Plus operator `+`.
98    Plus,
99    /// Minus operator `-`.
100    Minus,
101    /// Multiply operator `*`.
102    Multiply,
103    /// Divide operator `/`.
104    Divide,
105    /// Assignment operator `:=`.
106    Assign,
107    /// Equality operator `=`.
108    Equal,
109    /// Inequality operator `<>`.
110    NotEqual,
111    /// Less than operator `<`.
112    Less,
113    /// Less than or equal operator `<=`.
114    LessEqual,
115    /// Greater than operator `>`.
116    Greater,
117    /// Greater than or equal operator `>=`.
118    GreaterEqual,
119
120    /// Left parenthesis `(`.
121    LeftParen,
122    /// Right parenthesis `)`.
123    RightParen,
124    /// Left bracket `[`.
125    LeftBracket,
126    /// Right bracket `]`.
127    RightBracket,
128    /// Semicolon `;`.
129    Semicolon,
130    /// Comma `,`.
131    Comma,
132    /// Dot `.`.
133    Dot,
134    /// Colon `:`.
135    Colon,
136    /// Range operator `..`.
137    Range,
138    /// Caret operator `^`.
139    Caret,
140
141    /// Root node (Internal use).
142    Root,
143    /// Program block (Internal use).
144    ProgramBlock,
145    /// Variable section (Internal use).
146    VarSection,
147    /// Constant section (Internal use).
148    ConstSection,
149    /// Type section (Internal use).
150    TypeSection,
151    /// Procedure definition (Internal use).
152    ProcedureDef,
153    /// Function definition (Internal use).
154    FunctionDef,
155    /// Compound statement (Internal use).
156    CompoundStmt,
157    /// Expression (Internal use).
158    Expression,
159
160    /// Error token.
161    Error,
162    /// End of file.
163    Eof,
164}
165
166/// Represents a token in a Pascal source file.
167pub type PascalToken = Token<PascalTokenType>;
168
169impl TokenType for PascalTokenType {
170    type Role = UniversalTokenRole;
171    const END_OF_STREAM: Self = Self::Eof;
172
173    fn is_ignored(&self) -> bool {
174        matches!(self, Self::Whitespace | Self::Newline | Self::Comment)
175    }
176
177    fn role(&self) -> Self::Role {
178        use UniversalTokenRole::*;
179        match self {
180            Self::Whitespace | Self::Newline => Whitespace,
181            Self::Comment => Comment,
182            Self::Identifier => Name,
183            Self::IntegerLiteral | Self::RealLiteral | Self::StringLiteral | Self::CharLiteral => Literal,
184            Self::Program
185            | Self::Begin
186            | Self::End
187            | Self::Var
188            | Self::Const
189            | Self::Type
190            | Self::Function
191            | Self::Procedure
192            | Self::If
193            | Self::Then
194            | Self::Else
195            | Self::While
196            | Self::Do
197            | Self::For
198            | Self::To
199            | Self::Downto
200            | Self::Repeat
201            | Self::Until
202            | Self::Case
203            | Self::Of
204            | Self::With
205            | Self::Record
206            | Self::Array
207            | Self::Set
208            | Self::File
209            | Self::Packed
210            | Self::Nil
211            | Self::True
212            | Self::False
213            | Self::And
214            | Self::Or
215            | Self::Not
216            | Self::Div
217            | Self::Mod
218            | Self::In => UniversalTokenRole::Keyword,
219            Self::Plus | Self::Minus | Self::Multiply | Self::Divide | Self::Assign | Self::Equal | Self::NotEqual | Self::Less | Self::LessEqual | Self::Greater | Self::GreaterEqual | Self::Caret => UniversalTokenRole::Operator,
220            Self::LeftParen | Self::RightParen | Self::LeftBracket | Self::RightBracket | Self::Semicolon | Self::Comma | Self::Dot | Self::Colon | Self::Range => UniversalTokenRole::Punctuation,
221            Self::Error => UniversalTokenRole::Error,
222            Self::Eof => UniversalTokenRole::Eof,
223            _ => UniversalTokenRole::None,
224        }
225    }
226}
oak_pascal/lexer/token_type.rs

oak_pascal/lexer/
token_type.rs
