oak_python/lexer/
token_type.rs

1//! Python token types.
2
3use oak_core::{Token, TokenType, UniversalTokenRole};
4#[cfg(feature = "serde")]
5use serde::{Deserialize, Serialize};
6
7/// Type alias for Python tokens.
8pub type PythonToken = Token<PythonTokenType>;
9
10impl PythonTokenType {
11    /// Returns true if the token type is a keyword.
12    pub fn is_keyword(&self) -> bool {
13        matches!(
14            self,
15            Self::AndKeyword
16                | Self::AsKeyword
17                | Self::AssertKeyword
18                | Self::AsyncKeyword
19                | Self::AwaitKeyword
20                | Self::BreakKeyword
21                | Self::ClassKeyword
22                | Self::ContinueKeyword
23                | Self::DefKeyword
24                | Self::DelKeyword
25                | Self::ElifKeyword
26                | Self::ElseKeyword
27                | Self::ExceptKeyword
28                | Self::FalseKeyword
29                | Self::FinallyKeyword
30                | Self::ForKeyword
31                | Self::FromKeyword
32                | Self::GlobalKeyword
33                | Self::IfKeyword
34                | Self::ImportKeyword
35                | Self::InKeyword
36                | Self::IsKeyword
37                | Self::LambdaKeyword
38                | Self::NoneKeyword
39                | Self::NonlocalKeyword
40                | Self::NotKeyword
41                | Self::OrKeyword
42                | Self::PassKeyword
43                | Self::RaiseKeyword
44                | Self::ReturnKeyword
45                | Self::TrueKeyword
46                | Self::TryKeyword
47                | Self::WhileKeyword
48                | Self::WithKeyword
49                | Self::YieldKeyword
50        )
51    }
52}
53
54impl PythonTokenType {
55    /// Returns true if the token type is a trivia (whitespace or comment).
56    pub fn is_trivia(&self) -> bool {
57        matches!(self, Self::Whitespace | Self::Comment)
58    }
59}
60
61impl TokenType for PythonTokenType {
62    type Role = UniversalTokenRole;
63    const END_OF_STREAM: Self = Self::Error;
64
65    fn is_ignored(&self) -> bool {
66        false
67    }
68
69    fn role(&self) -> Self::Role {
70        match self {
71            _ => UniversalTokenRole::None,
72        }
73    }
74}
75
76/// Python token types.
77#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
78#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
79#[repr(u16)]
80pub enum PythonTokenType {
81    /// Whitespace
82    Whitespace,
83    /// Comment
84    Comment,
85    /// Identifier
86    Identifier,
87
88    /// Number literal
89    Number,
90    /// String literal
91    String,
92    /// Bytes literal
93    Bytes,
94    /// Formatted string literal
95    FString,
96
97    /// `and`
98    AndKeyword,
99    /// `as`
100    AsKeyword,
101    /// `assert`
102    AssertKeyword,
103    /// `async`
104    AsyncKeyword,
105    /// `await`
106    AwaitKeyword,
107    /// `break`
108    BreakKeyword,
109    /// `class`
110    ClassKeyword,
111    /// `continue`
112    ContinueKeyword,
113    /// `def`
114    DefKeyword,
115    /// `del`
116    DelKeyword,
117    /// `elif`
118    ElifKeyword,
119    /// `else`
120    ElseKeyword,
121    /// `except`
122    ExceptKeyword,
123    /// `False`
124    FalseKeyword,
125    /// `finally`
126    FinallyKeyword,
127    /// `for`
128    ForKeyword,
129    /// `from`
130    FromKeyword,
131    /// `global`
132    GlobalKeyword,
133    /// `if`
134    IfKeyword,
135    /// `import`
136    ImportKeyword,
137    /// `in`
138    InKeyword,
139    /// `is`
140    IsKeyword,
141    /// `lambda`
142    LambdaKeyword,
143    /// `None`
144    NoneKeyword,
145    /// `nonlocal`
146    NonlocalKeyword,
147    /// `not`
148    NotKeyword,
149    /// `or`
150    OrKeyword,
151    /// `pass`
152    PassKeyword,
153    /// `raise`
154    RaiseKeyword,
155    /// `return`
156    ReturnKeyword,
157    /// `True`
158    TrueKeyword,
159    /// `try`
160    TryKeyword,
161    /// `while`
162    WhileKeyword,
163    /// `with`
164    WithKeyword,
165    /// `yield`
166    YieldKeyword,
167
168    /// `+`
169    Plus,
170    /// `-`
171    Minus,
172    /// `*`
173    Star,
174    /// `**`
175    DoubleStar,
176    /// `/`
177    Slash,
178    /// `//`
179    DoubleSlash,
180    /// `%`
181    Percent,
182    /// `@`
183    At,
184    /// `<<`
185    LeftShift,
186    /// `>>`
187    RightShift,
188    /// `&`
189    Ampersand,
190    /// `|`
191    Pipe,
192    /// `^`
193    Caret,
194    /// `~`
195    Tilde,
196    /// `<`
197    Less,
198    /// `>`
199    Greater,
200    /// `<=`
201    LessEqual,
202    /// `>=`
203    GreaterEqual,
204    /// `==`
205    Equal,
206    /// `!=`
207    NotEqual,
208
209    /// `=`
210    Assign,
211    /// `+=`
212    PlusAssign,
213    /// `-=`
214    MinusAssign,
215    /// `*=`
216    StarAssign,
217    /// `**=`
218    DoubleStarAssign,
219    /// `/=`
220    SlashAssign,
221    /// `//=`
222    DoubleSlashAssign,
223    /// `%=`
224    PercentAssign,
225    /// `@=`
226    AtAssign,
227    /// `&=`
228    AmpersandAssign,
229    /// `|=`
230    PipeAssign,
231    /// `^=`
232    CaretAssign,
233    /// `<<=`
234    LeftShiftAssign,
235    /// `>>=`
236    RightShiftAssign,
237
238    /// `(`
239    LeftParen,
240    /// `)`
241    RightParen,
242    /// `[`
243    LeftBracket,
244    /// `]`
245    RightBracket,
246    /// `{`
247    LeftBrace,
248    /// `}`
249    RightBrace,
250    /// `,`
251    Comma,
252    /// `:`
253    Colon,
254    /// `;`
255    Semicolon,
256    /// `.`
257    Dot,
258    /// `->`
259    Arrow,
260    /// `...`
261    Ellipsis,
262
263    /// Newline
264    Newline,
265    /// Indent
266    Indent,
267    /// Dedent
268    Dedent,
269    /// End of stream
270    Eof,
271    /// Error token
272    Error,
273}
274
275impl From<PythonTokenType> for u16 {
276    fn from(k: PythonTokenType) -> u16 {
277        k as u16
278    }
279}
280
281impl From<u16> for PythonTokenType {
282    fn from(d: u16) -> PythonTokenType {
283        unsafe { core::mem::transmute::<u16, PythonTokenType>(d) }
284    }
285}
oak_python/lexer/token_type.rs

oak_python/lexer/
token_type.rs
