oak_pascal/lexer/
token_type.rs

1use oak_core::{Source, Token, TokenType, UniversalElementRole, UniversalTokenRole};
2#[cfg(feature = "serde")]
3use serde::{Deserialize, Serialize};
4
5#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
6#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
7pub enum PascalTokenType {
8    // 空白和换行
9    Whitespace,
10    Newline,
11
12    // 注释
13    Comment,
14
15    // 关键字
16    Program,
17    Begin,
18    End,
19    Var,
20    Const,
21    Type,
22    Function,
23    Procedure,
24    If,
25    Then,
26    Else,
27    While,
28    Do,
29    For,
30    To,
31    Downto,
32    Repeat,
33    Until,
34    Case,
35    Of,
36    With,
37    Record,
38    Array,
39    Set,
40    File,
41    Packed,
42    Nil,
43    True,
44    False,
45    And,
46    Or,
47    Not,
48    Div,
49    Mod,
50    In,
51
52    // 标识符和字面量
53    Identifier,
54    IntegerLiteral,
55    RealLiteral,
56    StringLiteral,
57    CharLiteral,
58
59    // 运算符
60    Plus,         // +
61    Minus,        // -
62    Multiply,     // *
63    Divide,       // /
64    Assign,       // :=
65    Equal,        // =
66    NotEqual,     // <>
67    Less,         // <
68    LessEqual,    // <=
69    Greater,      // >
70    GreaterEqual, // >=
71
72    // 分隔符
73    LeftParen,    // (
74    RightParen,   // )
75    LeftBracket,  // [
76    RightBracket, // ]
77    Semicolon,    // ;
78    Comma,        // ,
79    Dot,          // .
80    Colon,        // :
81    Range,        // ..
82    Caret,        // ^
83
84    // Element markers (Internal use)
85    Root,
86    ProgramBlock,
87    VarSection,
88    ConstSection,
89    TypeSection,
90    ProcedureDef,
91    FunctionDef,
92    CompoundStmt,
93    Expression,
94
95    // 其他
96    Error,
97    Eof,
98}
99
100pub type PascalToken = Token<PascalTokenType>;
101
102impl TokenType for PascalTokenType {
103    type Role = UniversalTokenRole;
104    const END_OF_STREAM: Self = Self::Error;
105
106    fn role(&self) -> Self::Role {
107        use UniversalTokenRole::*;
108        match self {
109            Self::Whitespace | Self::Newline => Whitespace,
110            Self::Comment => Comment,
111            Self::Identifier => Name,
112            Self::IntegerLiteral | Self::RealLiteral | Self::StringLiteral | Self::CharLiteral => Literal,
113            Self::Program
114            | Self::Begin
115            | Self::End
116            | Self::Var
117            | Self::Const
118            | Self::Type
119            | Self::Function
120            | Self::Procedure
121            | Self::If
122            | Self::Then
123            | Self::Else
124            | Self::While
125            | Self::Do
126            | Self::For
127            | Self::To
128            | Self::Downto
129            | Self::Repeat
130            | Self::Until
131            | Self::Case
132            | Self::Of
133            | Self::With
134            | Self::Record
135            | Self::Array
136            | Self::Set
137            | Self::File
138            | Self::Packed
139            | Self::Nil
140            | Self::True
141            | Self::False
142            | Self::And
143            | Self::Or
144            | Self::Not
145            | Self::Div
146            | Self::Mod
147            | Self::In => Keyword,
148            Self::Plus | Self::Minus | Self::Multiply | Self::Divide | Self::Assign | Self::Equal | Self::NotEqual | Self::Less | Self::LessEqual | Self::Greater | Self::GreaterEqual | Self::Caret => Operator,
149            Self::LeftParen | Self::RightParen | Self::LeftBracket | Self::RightBracket | Self::Semicolon | Self::Comma | Self::Dot | Self::Colon | Self::Range => Punctuation,
150            Self::Error => Error,
151            Self::Eof => Eof,
152            _ => None,
153        }
154    }
155}
oak_pascal/lexer/token_type.rs

oak_pascal/lexer/
token_type.rs
