oak_python/lexer/
token_type.rs

1//! Python token types.
2
3use oak_core::{Token, TokenType, UniversalTokenRole};
4
5/// Type alias for Python tokens.
6pub type PythonToken = Token<PythonTokenType>;
7
8impl PythonTokenType {
9    /// Returns true if the token type is a keyword.
10    pub fn is_keyword(&self) -> bool {
11        matches!(
12            self,
13            Self::AndKeyword
14                | Self::AsKeyword
15                | Self::AssertKeyword
16                | Self::AsyncKeyword
17                | Self::AwaitKeyword
18                | Self::BreakKeyword
19                | Self::ClassKeyword
20                | Self::ContinueKeyword
21                | Self::DefKeyword
22                | Self::DelKeyword
23                | Self::ElifKeyword
24                | Self::ElseKeyword
25                | Self::ExceptKeyword
26                | Self::FalseKeyword
27                | Self::FinallyKeyword
28                | Self::ForKeyword
29                | Self::FromKeyword
30                | Self::GlobalKeyword
31                | Self::IfKeyword
32                | Self::ImportKeyword
33                | Self::InKeyword
34                | Self::IsKeyword
35                | Self::LambdaKeyword
36                | Self::NoneKeyword
37                | Self::NonlocalKeyword
38                | Self::NotKeyword
39                | Self::OrKeyword
40                | Self::PassKeyword
41                | Self::RaiseKeyword
42                | Self::ReturnKeyword
43                | Self::TrueKeyword
44                | Self::TryKeyword
45                | Self::WhileKeyword
46                | Self::WithKeyword
47                | Self::YieldKeyword
48        )
49    }
50}
51
52impl PythonTokenType {
53    /// Returns true if the token type is a trivia (whitespace or comment).
54    pub fn is_trivia(&self) -> bool {
55        matches!(self, Self::Whitespace | Self::Comment)
56    }
57}
58
59impl TokenType for PythonTokenType {
60    type Role = UniversalTokenRole;
61    const END_OF_STREAM: Self = Self::Error;
62
63    fn is_ignored(&self) -> bool {
64        self.is_trivia()
65    }
66
67    fn role(&self) -> Self::Role {
68        match self {
69            _ => UniversalTokenRole::None,
70        }
71    }
72}
73
74/// Python token types.
75#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
76#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
77#[repr(u16)]
78pub enum PythonTokenType {
79    /// Whitespace
80    Whitespace,
81    /// Comment
82    Comment,
83    /// Identifier
84    Identifier,
85
86    /// Number literal
87    Number,
88    /// String literal
89    String,
90    /// Bytes literal
91    Bytes,
92    /// Formatted string literal
93    FString,
94
95    /// `and`
96    AndKeyword,
97    /// `as`
98    AsKeyword,
99    /// `assert`
100    AssertKeyword,
101    /// `async`
102    AsyncKeyword,
103    /// `await`
104    AwaitKeyword,
105    /// `break`
106    BreakKeyword,
107    /// `class`
108    ClassKeyword,
109    /// `continue`
110    ContinueKeyword,
111    /// `def`
112    DefKeyword,
113    /// `del`
114    DelKeyword,
115    /// `elif`
116    ElifKeyword,
117    /// `else`
118    ElseKeyword,
119    /// `except`
120    ExceptKeyword,
121    /// `False`
122    FalseKeyword,
123    /// `finally`
124    FinallyKeyword,
125    /// `for`
126    ForKeyword,
127    /// `from`
128    FromKeyword,
129    /// `global`
130    GlobalKeyword,
131    /// `if`
132    IfKeyword,
133    /// `import`
134    ImportKeyword,
135    /// `in`
136    InKeyword,
137    /// `is`
138    IsKeyword,
139    /// `lambda`
140    LambdaKeyword,
141    /// `None`
142    NoneKeyword,
143    /// `nonlocal`
144    NonlocalKeyword,
145    /// `not`
146    NotKeyword,
147    /// `or`
148    OrKeyword,
149    /// `pass`
150    PassKeyword,
151    /// `raise`
152    RaiseKeyword,
153    /// `return`
154    ReturnKeyword,
155    /// `True`
156    TrueKeyword,
157    /// `try`
158    TryKeyword,
159    /// `while`
160    WhileKeyword,
161    /// `with`
162    WithKeyword,
163    /// `yield`
164    YieldKeyword,
165
166    /// `+`
167    Plus,
168    /// `-`
169    Minus,
170    /// `*`
171    Star,
172    /// `**`
173    DoubleStar,
174    /// `/`
175    Slash,
176    /// `//`
177    DoubleSlash,
178    /// `%`
179    Percent,
180    /// `@`
181    At,
182    /// `<<`
183    LeftShift,
184    /// `>>`
185    RightShift,
186    /// `&`
187    Ampersand,
188    /// `|`
189    Pipe,
190    /// `^`
191    Caret,
192    /// `~`
193    Tilde,
194    /// `<`
195    Less,
196    /// `>`
197    Greater,
198    /// `<=`
199    LessEqual,
200    /// `>=`
201    GreaterEqual,
202    /// `==`
203    Equal,
204    /// `!=`
205    NotEqual,
206
207    /// `=`
208    Assign,
209    /// `+=`
210    PlusAssign,
211    /// `-=`
212    MinusAssign,
213    /// `*=`
214    StarAssign,
215    /// `**=`
216    DoubleStarAssign,
217    /// `/=`
218    SlashAssign,
219    /// `//=`
220    DoubleSlashAssign,
221    /// `%=`
222    PercentAssign,
223    /// `@=`
224    AtAssign,
225    /// `&=`
226    AmpersandAssign,
227    /// `|=`
228    PipeAssign,
229    /// `^=`
230    CaretAssign,
231    /// `<<=`
232    LeftShiftAssign,
233    /// `>>=`
234    RightShiftAssign,
235
236    /// `(`
237    LeftParen,
238    /// `)`
239    RightParen,
240    /// `[`
241    LeftBracket,
242    /// `]`
243    RightBracket,
244    /// `{`
245    LeftBrace,
246    /// `}`
247    RightBrace,
248    /// `,`
249    Comma,
250    /// `:`
251    Colon,
252    /// `;`
253    Semicolon,
254    /// `.`
255    Dot,
256    /// `->`
257    Arrow,
258    /// `...`
259    Ellipsis,
260
261    /// Newline
262    Newline,
263    /// Indent
264    Indent,
265    /// Dedent
266    Dedent,
267    /// End of stream
268    Eof,
269    /// Error token
270    Error,
271}
272
273impl From<PythonTokenType> for u16 {
274    fn from(k: PythonTokenType) -> u16 {
275        k as u16
276    }
277}
278
279impl From<u16> for PythonTokenType {
280    fn from(d: u16) -> PythonTokenType {
281        unsafe { core::mem::transmute::<u16, PythonTokenType>(d) }
282    }
283}
oak_python/lexer/token_type.rs

oak_python/lexer/
token_type.rs
