microcad_lang_parse/
token.rs

1// Copyright © 2026 The µcad authors <info@microcad.xyz>
2// SPDX-License-Identifier: AGPL-3.0-or-later
3
4use std::borrow::Cow;
5
6pub use crate::lex::LexerError;
7
8/// Source token for µcad files
9#[derive(Debug, PartialEq, Clone)]
10pub enum Token<'a> {
11    /// Whitespace
12    Whitespace(Cow<'a, str>),
13
14    /// A single-line comment, starting with `//`
15    SingleLineComment(Cow<'a, str>),
16    /// A multi-line comment, starting with `/*` and ending with `*/`
17    MultiLineComment(Cow<'a, str>),
18    /// A doc-comment, starting with `///`
19    DocComment(Cow<'a, str>),
20    /// An inner doc-comment, starting with `//!`
21    InnerDocComment(Cow<'a, str>),
22
23    /// The `mod` keyword
24    KeywordMod,
25    /// The `part` keyword
26    KeywordPart,
27    /// The `sketch` keyword
28    KeywordSketch,
29    /// The `op` keyword
30    KeywordOp,
31    /// The `fn` keyword
32    KeywordFn,
33    /// The `if` keyword
34    KeywordIf,
35    /// The `else` keyword
36    KeywordElse,
37    /// The `use` keyword
38    KeywordUse,
39    /// The `as` keyword
40    KeywordAs,
41    /// The `return` keyword
42    KeywordReturn,
43    /// The `pub` keyword
44    KeywordPub,
45    /// The `const` keyword
46    KeywordConst,
47    /// The `prop` keyword
48    KeywordProp,
49    /// The `init` keyword
50    KeywordInit,
51    /// The `__plugin` keyword
52    KeywordPlugin,
53    /// The `assembly` keyword
54    KeywordAssembly,
55    /// The `material` keyword
56    KeywordMaterial,
57    /// The `unit` keyword
58    KeywordUnit,
59    /// The `enum` keyword
60    KeywordEnum,
61    /// The `struct` keyword
62    KeywordStruct,
63    /// The `match` keyword
64    KeywordMatch,
65    /// The `type` keyword
66    KeywordType,
67    /// The `extern` keyword
68    KeywordExtern,
69
70    /// An identifier, alphanumeric, starting with either an alpha character or a underscore
71    Identifier(Cow<'a, str>),
72    /// A unit identifier
73    Unit(Cow<'a, str>),
74
75    /// An integer literal
76    LiteralInt(Cow<'a, str>),
77    /// A float literal
78    LiteralFloat(Cow<'a, str>),
79    /// A boolean literal
80    LiteralBool(bool),
81    /// A string literal
82    LiteralString(Cow<'a, str>),
83
84    /// Double-quote indicating the start of a format string
85    FormatStringStart,
86    /// Double-quote indicating the end of a format string
87    FormatStringEnd,
88    /// Literal string content of a format string
89    StringContent(Cow<'a, str>),
90    /// Escaped character inside a format string
91    Character(char),
92    /// The start of the format expression inside a format string
93    StringFormatOpen,
94    /// The end of the format expression inside a format string
95    StringFormatClose,
96    /// The precision specification of the format expression inside a format string
97    StringFormatPrecision(Cow<'a, str>),
98    /// The width specification of the format expression inside a format string
99    StringFormatWidth(Cow<'a, str>),
100
101    /// The `:` symbol
102    SigilColon,
103    /// The `;` symbol
104    SigilSemiColon,
105    /// The `::` symbol
106    SigilDoubleColon,
107    /// The `(` symbol
108    SigilOpenBracket,
109    /// The `)` symbol
110    SigilCloseBracket,
111    /// The `[` symbol
112    SigilOpenSquareBracket,
113    /// The `]` symbol
114    SigilCloseSquareBracket,
115    /// The `{` symbol
116    SigilOpenCurlyBracket,
117    /// The `}` symbol
118    SigilCloseCurlyBracket,
119    /// The `#` symbol
120    SigilHash,
121    /// The `.` symbol
122    SigilDot,
123    /// The `,` symbol
124    SigilComma,
125    /// The `..` symbol
126    SigilDoubleDot,
127    /// The `@` symbol
128    SigilAt,
129    /// The `->` symbol
130    SigilSingleArrow,
131    /// The `"` symbol
132    SigilQuote,
133
134    /// Add operator: `+`
135    OperatorAdd,
136    /// Subtract operator: `-`
137    OperatorSubtract,
138    /// Multiply operator: `-`
139    OperatorMultiply,
140    /// Divide operator: `/`
141    OperatorDivide,
142    /// Union operator: `|`
143    OperatorUnion,
144    /// Intersect operator: `&`
145    OperatorIntersect,
146    /// xor operator: `^`
147    OperatorPowerXor,
148    /// Greater-than operator: `>`
149    OperatorGreaterThan,
150    /// Less-than operator: `<`
151    OperatorLessThan,
152    /// Greater-or-equal operator: `>=`
153    OperatorGreaterEqual,
154    /// Less-or-equal operator: `<=`
155    OperatorLessEqual,
156    /// Near operator: `~`
157    OperatorNear,
158    /// Equal operator: `==`
159    OperatorEqual,
160    /// Not-equal operator: `!=`
161    OperatorNotEqual,
162    /// And operator: `and`
163    OperatorAnd,
164    /// Or operator: `or`
165    OperatorOr,
166    /// Xor operator: `xor'
167    OperatorXor,
168    /// Not operator: `!`
169    OperatorNot,
170    /// Assignment operator: `=`
171    OperatorAssignment,
172
173    /// A lexer failure
174    ///
175    /// When encountering an error, the lexer attempts to recover and continue emitting further tokens
176    Error(LexerError),
177}
178
179impl std::fmt::Display for Token<'_> {
180    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
181        write!(f, "{}", self.kind())
182    }
183}
184
185impl Token<'_> {
186    /// Create an owned version of the token
187    pub fn into_owned(self) -> Token<'static> {
188        match self {
189            Token::Whitespace(c) => Token::Whitespace(c.into_owned().into()),
190            Token::SingleLineComment(c) => Token::SingleLineComment(c.into_owned().into()),
191            Token::MultiLineComment(c) => Token::MultiLineComment(c.into_owned().into()),
192            Token::DocComment(c) => Token::DocComment(c.into_owned().into()),
193            Token::InnerDocComment(c) => Token::InnerDocComment(c.into_owned().into()),
194            Token::Identifier(s) => Token::Identifier(s.into_owned().into()),
195            Token::Unit(s) => Token::Unit(s.into_owned().into()),
196            Token::LiteralInt(s) => Token::LiteralInt(s.into_owned().into()),
197            Token::LiteralFloat(s) => Token::LiteralFloat(s.into_owned().into()),
198            Token::LiteralString(s) => Token::LiteralString(s.into_owned().into()),
199
200            Token::KeywordMod => Token::KeywordMod,
201            Token::KeywordPart => Token::KeywordPart,
202            Token::KeywordSketch => Token::KeywordSketch,
203            Token::KeywordOp => Token::KeywordOp,
204            Token::KeywordFn => Token::KeywordFn,
205            Token::KeywordIf => Token::KeywordIf,
206            Token::KeywordElse => Token::KeywordElse,
207            Token::KeywordUse => Token::KeywordUse,
208            Token::KeywordAs => Token::KeywordAs,
209            Token::KeywordReturn => Token::KeywordReturn,
210            Token::KeywordPub => Token::KeywordPub,
211            Token::KeywordConst => Token::KeywordConst,
212            Token::KeywordProp => Token::KeywordProp,
213            Token::KeywordInit => Token::KeywordInit,
214            Token::KeywordAssembly => Token::KeywordAssembly,
215            Token::KeywordPlugin => Token::KeywordPlugin,
216            Token::KeywordMaterial => Token::KeywordMaterial,
217            Token::KeywordUnit => Token::KeywordUnit,
218            Token::KeywordEnum => Token::KeywordEnum,
219            Token::KeywordStruct => Token::KeywordStruct,
220            Token::KeywordMatch => Token::KeywordMatch,
221            Token::KeywordType => Token::KeywordType,
222            Token::KeywordExtern => Token::KeywordExtern,
223            Token::LiteralBool(l) => Token::LiteralBool(l),
224            Token::SigilColon => Token::SigilColon,
225            Token::SigilSemiColon => Token::SigilSemiColon,
226            Token::SigilDoubleColon => Token::SigilDoubleColon,
227            Token::SigilOpenBracket => Token::SigilOpenBracket,
228            Token::SigilCloseBracket => Token::SigilCloseBracket,
229            Token::SigilOpenSquareBracket => Token::SigilOpenSquareBracket,
230            Token::SigilCloseSquareBracket => Token::SigilCloseSquareBracket,
231            Token::SigilOpenCurlyBracket => Token::SigilOpenCurlyBracket,
232            Token::SigilCloseCurlyBracket => Token::SigilCloseCurlyBracket,
233            Token::SigilHash => Token::SigilHash,
234            Token::SigilDot => Token::SigilDot,
235            Token::SigilComma => Token::SigilComma,
236            Token::SigilDoubleDot => Token::SigilDoubleDot,
237            Token::SigilAt => Token::SigilAt,
238            Token::SigilSingleArrow => Token::SigilSingleArrow,
239            Token::OperatorAdd => Token::OperatorAdd,
240            Token::OperatorSubtract => Token::OperatorSubtract,
241            Token::OperatorMultiply => Token::OperatorMultiply,
242            Token::OperatorDivide => Token::OperatorDivide,
243            Token::OperatorUnion => Token::OperatorUnion,
244            Token::OperatorIntersect => Token::OperatorIntersect,
245            Token::OperatorPowerXor => Token::OperatorPowerXor,
246            Token::OperatorGreaterThan => Token::OperatorGreaterThan,
247            Token::OperatorLessThan => Token::OperatorLessThan,
248            Token::OperatorGreaterEqual => Token::OperatorGreaterEqual,
249            Token::OperatorLessEqual => Token::OperatorLessEqual,
250            Token::OperatorNear => Token::OperatorNear,
251            Token::OperatorEqual => Token::OperatorEqual,
252            Token::OperatorNotEqual => Token::OperatorNotEqual,
253            Token::OperatorAnd => Token::OperatorAnd,
254            Token::OperatorOr => Token::OperatorOr,
255            Token::OperatorXor => Token::OperatorXor,
256            Token::OperatorNot => Token::OperatorNot,
257            Token::OperatorAssignment => Token::OperatorAssignment,
258            Token::FormatStringStart => Token::FormatStringStart,
259            Token::FormatStringEnd => Token::FormatStringEnd,
260            Token::StringContent(s) => Token::StringContent(s.into_owned().into()),
261            Token::Character(c) => Token::Character(c),
262            Token::StringFormatOpen => Token::StringFormatOpen,
263            Token::StringFormatClose => Token::StringFormatClose,
264            Token::SigilQuote => Token::SigilQuote,
265            Token::Error(e) => Token::Error(e),
266            Token::StringFormatPrecision(c) => Token::StringFormatPrecision(c.into_owned().into()),
267            Token::StringFormatWidth(c) => Token::StringFormatWidth(c.into_owned().into()),
268        }
269    }
270
271    /// Get a descriptive name or symbol for the token type
272    pub fn kind(&self) -> &'static str {
273        match self {
274            Token::Whitespace(_) => "whitespace",
275            Token::SingleLineComment(_) => "single-line comment",
276            Token::MultiLineComment(_) => "multi-line comment",
277            Token::DocComment(_) => "doc comment",
278            Token::InnerDocComment(_) => "inner doc comment",
279            Token::KeywordMod => "mod",
280            Token::KeywordPart => "part",
281            Token::KeywordSketch => "sketch",
282            Token::KeywordOp => "op",
283            Token::KeywordFn => "fn",
284            Token::KeywordIf => "if",
285            Token::KeywordElse => "else",
286            Token::KeywordUse => "use",
287            Token::KeywordAs => "as",
288            Token::KeywordReturn => "return",
289            Token::KeywordPub => "pub",
290            Token::KeywordConst => "const",
291            Token::KeywordProp => "prop",
292            Token::KeywordInit => "init",
293            Token::KeywordPlugin => "__plugin",
294            Token::KeywordAssembly => "assembly",
295            Token::KeywordMaterial => "material",
296            Token::KeywordUnit => "unit",
297            Token::KeywordEnum => "enum",
298            Token::KeywordStruct => "struct",
299            Token::KeywordMatch => "match",
300            Token::KeywordType => "type",
301            Token::KeywordExtern => "extern",
302            Token::Identifier(_) => "identifier",
303            Token::Unit(_) => "unit",
304            Token::LiteralInt(_) => "integer literal",
305            Token::LiteralFloat(_) => "float literal",
306            Token::LiteralBool(_) => "boolean literal",
307            Token::LiteralString(_) => "string literal",
308            Token::FormatStringStart => "start of string",
309            Token::FormatStringEnd => "end of string",
310            Token::StringContent(_) => "string content",
311            Token::Character(_) => "escaped character",
312            Token::StringFormatOpen => "string format start",
313            Token::StringFormatClose => "string format end",
314            Token::SigilColon => ":",
315            Token::SigilSemiColon => ";",
316            Token::SigilDoubleColon => "::",
317            Token::SigilOpenBracket => "(",
318            Token::SigilCloseBracket => ")",
319            Token::SigilOpenSquareBracket => "[",
320            Token::SigilCloseSquareBracket => "]",
321            Token::SigilOpenCurlyBracket => "{",
322            Token::SigilCloseCurlyBracket => "}",
323            Token::SigilHash => "#",
324            Token::SigilDot => ".",
325            Token::SigilComma => ",",
326            Token::SigilDoubleDot => "..",
327            Token::SigilAt => "@",
328            Token::SigilSingleArrow => "->",
329            Token::SigilQuote => "\"",
330            Token::OperatorAdd => "+",
331            Token::OperatorSubtract => "-",
332            Token::OperatorMultiply => "*",
333            Token::OperatorDivide => "/",
334            Token::OperatorUnion => "|",
335            Token::OperatorIntersect => "&",
336            Token::OperatorPowerXor => "^",
337            Token::OperatorGreaterThan => ">",
338            Token::OperatorLessThan => "<",
339            Token::OperatorGreaterEqual => ">=",
340            Token::OperatorLessEqual => "<=",
341            Token::OperatorNear => "!",
342            Token::OperatorEqual => "==",
343            Token::OperatorNotEqual => "!=",
344            Token::OperatorAnd => "and",
345            Token::OperatorOr => "or",
346            Token::OperatorXor => "xor",
347            Token::OperatorNot => "not",
348            Token::OperatorAssignment => "=",
349            Token::StringFormatPrecision(_) => "format precision",
350            Token::StringFormatWidth(_) => "format width",
351            Token::Error(e) => e.kind(),
352        }
353    }
354
355    /// Check if the token is an error
356    pub fn is_error(&self) -> bool {
357        matches!(self, Token::Error(_))
358    }
359}
microcad_lang_parse/token.rs

microcad_lang_parse/
token.rs
