oak_typst/lexer/
token_type.rs

1use oak_core::{Source, Token, TokenType, UniversalElementRole, UniversalTokenRole};
2
3/// A token in a Typst document.
4pub type TypstToken = Token<TypstTokenType>;
5
6impl TokenType for TypstTokenType {
7    type Role = UniversalTokenRole;
8    const END_OF_STREAM: Self = Self::Eof;
9
10    fn is_ignored(&self) -> bool {
11        matches!(self, Self::Whitespace | Self::Newline | Self::LineComment | Self::BlockComment)
12    }
13
14    fn role(&self) -> Self::Role {
15        match self {
16            Self::Whitespace | Self::Newline => UniversalTokenRole::Whitespace,
17            Self::LineComment | Self::BlockComment => UniversalTokenRole::Comment,
18            Self::Let | Self::If | Self::Else | Self::For | Self::While | Self::Break | Self::Continue | Self::Return | Self::Import | Self::Include | Self::Set | Self::Show => UniversalTokenRole::Keyword,
19            Self::True | Self::False | Self::StringLiteral | Self::NumericLiteral => UniversalTokenRole::Literal,
20            Self::Identifier => UniversalTokenRole::Name,
21            Self::Plus | Self::Minus | Self::Star | Self::Slash | Self::Percent | Self::Equal | Self::EqualEqual | Self::NotEqual | Self::Less | Self::Greater | Self::LessEqual | Self::GreaterEqual | Self::And | Self::Or | Self::Not => {
22                UniversalTokenRole::Operator
23            }
24            Self::LeftParen
25            | Self::RightParen
26            | Self::LeftBrace
27            | Self::RightBrace
28            | Self::LeftBracket
29            | Self::RightBracket
30            | Self::Semicolon
31            | Self::Comma
32            | Self::Dot
33            | Self::Colon
34            | Self::Hash
35            | Self::At
36            | Self::Dollar
37            | Self::Underscore
38            | Self::Backtick => UniversalTokenRole::Punctuation,
39            Self::Eof => UniversalTokenRole::Eof,
40            Self::Error => UniversalTokenRole::Error,
41            _ => UniversalTokenRole::None,
42        }
43    }
44}
45
46/// Represents the type of a Typst token.
47#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
48#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
49pub enum TypstTokenType {
50    /// Root node.
51    Root,
52    /// A document.
53    Document,
54    /// A block of content.
55    Block,
56
57    /// A heading (= Heading).
58    Heading,
59    /// A paragraph of text.
60    Paragraph,
61    /// A list (+ Item).
62    List,
63    /// A list item.
64    ListItem,
65    /// An enumeration item (1. Item).
66    EnumItem,
67    /// A table.
68    Table,
69    /// A table row.
70    TableRow,
71    /// A table cell.
72    TableCell,
73    /// A figure.
74    Figure,
75    /// An image.
76    Image,
77    /// A link.
78    Link,
79
80    /// Plain text.
81    Text,
82    /// Strong text (*strong*).
83    Strong,
84    /// Emphasized text (_emphasis_).
85    Emphasis,
86    /// Code content (`code`).
87    Code,
88    /// Math content ($...$).
89    Math,
90    /// Inline math content.
91    InlineMath,
92    /// Display math content.
93    DisplayMath,
94    /// Raw content.
95    Raw,
96    /// A quote.
97    Quote,
98
99    /// A script.
100    Script,
101    /// An expression.
102    Expression,
103    /// A function call.
104    FunctionCall,
105    /// A variable.
106    Variable,
107    /// An assignment.
108    Assignment,
109    /// A conditional statement.
110    Conditional,
111    /// A loop statement.
112    Loop,
113    /// An import statement.
114    Import,
115    /// An include statement.
116    Include,
117
118    /// A set rule.
119    Set,
120    /// A show rule.
121    Show,
122    /// A style rule.
123    Style,
124    /// A color.
125    Color,
126    /// A font.
127    Font,
128    /// A size.
129    Size,
130
131    /// 'let' keyword.
132    Let,
133    /// 'if' keyword.
134    If,
135    /// 'else' keyword.
136    Else,
137    /// 'for' keyword.
138    For,
139    /// 'while' keyword.
140    While,
141    /// 'break' keyword.
142    Break,
143    /// 'continue' keyword.
144    Continue,
145    /// 'return' keyword.
146    Return,
147    /// 'true' keyword.
148    True,
149    /// 'false' keyword.
150    False,
151
152    /// Plus operator (+).
153    Plus,
154    /// Minus operator (-).
155    Minus,
156    /// Multiplication operator (*).
157    Star,
158    /// Division operator (/).
159    Slash,
160    /// Modulo operator (%).
161    Percent,
162    /// Assignment operator (=).
163    Equal,
164    /// Equality operator (==).
165    EqualEqual,
166    /// Inequality operator (!=).
167    NotEqual,
168    /// Less than operator (<).
169    Less,
170    /// Greater than operator (>).
171    Greater,
172    /// Less than or equal to operator (<=).
173    LessEqual,
174    /// Greater than or equal to operator (>=).
175    GreaterEqual,
176    /// Logical AND operator (and).
177    And,
178    /// Logical OR operator (or).
179    Or,
180    /// Logical NOT operator (not).
181    Not,
182
183    /// Left parenthesis (().
184    LeftParen,
185    /// Right parenthesis ()).
186    RightParen,
187    /// Left brace ({).
188    LeftBrace,
189    /// Right brace (}).
190    RightBrace,
191    /// Left bracket ([).
192    LeftBracket,
193    /// Right bracket (]).
194    RightBracket,
195    /// Semicolon separator (;).
196    Semicolon,
197    /// Comma separator (,).
198    Comma,
199    /// Dot separator (.).
200    Dot,
201    /// Colon separator (:).
202    Colon,
203    /// Hash symbol (#).
204    Hash,
205    /// At symbol (@).
206    At,
207    /// Dollar symbol ($).
208    Dollar,
209    /// Underscore symbol (_).
210    Underscore,
211    /// Backtick symbol (`).
212    Backtick,
213
214    /// A string literal ("...").
215    StringLiteral,
216    /// A numeric literal.
217    NumericLiteral,
218    /// An identifier.
219    Identifier,
220
221    /// A line comment (// ...).
222    LineComment,
223    /// A block comment (/* ... */).
224    BlockComment,
225    /// Whitespace characters.
226    Whitespace,
227    /// A newline character.
228    Newline,
229
230    /// End of file marker.
231    Eof,
232    /// Error token.
233    Error,
234}
oak_typst/lexer/token_type.rs

oak_typst/lexer/
token_type.rs
