mortar_compiler/
token.rs

1//! # token.rs
2//!
3//! # token.rs 文件
4//!
5//! ## Module Overview
6//!
7//! ## 模块概述
8//!
9//! Defines the `Token` enum and lexical analysis logic for the Mortar language.
10//!
11//! 定义 Mortar 语言的 `Token` 枚举和词法分析逻辑。
12//!
13//! Uses the `logos` crate to generate a high-performance lexer.
14//!
15//! 使用 `logos` crate 生成高性能的词法分析器。
16//!
17//! ## Source File Overview
18//!
19//! ## 源文件概述
20//!
21//! Contains the `Token` enum definition, token display formatting, and the `tokenize` function used by the parser and LSP.
22//!
23//! 包含 `Token` 枚举定义、token 显示格式化以及解析器和 LSP 使用的 `tokenize` 函数。
24
25use logos::Logos;
26use owo_colors::OwoColorize;
27use std::fmt;
28
29#[derive(Logos, Debug, PartialEq, Clone)]
30// Ignore whitespace
31#[logos(skip r"[ \t\r\n]+")]
32pub enum Token<'a> {
33    #[allow(dead_code)]
34    Error,
35
36    // region Comments
37    #[regex(r"//[^\n]*", |lex| lex.slice())]
38    SingleLineComment(&'a str),
39
40    #[regex(r"/\*([^*]|\*[^/])*\*/", |lex| lex.slice())]
41    MultiLineComment(&'a str),
42    // endregion
43
44    // region Keywords
45    #[token("node")]
46    #[token("nd")]
47    Node,
48    #[token("text")]
49    Text,
50    #[token("line")]
51    Line,
52    #[token("events")]
53    Events,
54    #[token("choice")]
55    Choice,
56    #[token("fn")]
57    #[token("function")]
58    Fn,
59    #[token("return")]
60    Return,
61    #[token("break")]
62    Break,
63    #[token("when")]
64    When,
65
66    // Variable and constant keywords
67    #[token("let")]
68    Let,
69    #[token("const")]
70    Const,
71    #[token("pub")]
72    #[token("public")]
73    Pub,
74    #[token("enum")]
75    Enum,
76
77    // Branch interpolation keyword
78    #[token("branch")]
79    Branch,
80
81    // Control flow keywords
82    #[token("if")]
83    If,
84    #[token("else")]
85    Else,
86
87    // Performance system keywords
88    #[token("event")]
89    Event,
90    #[token("run")]
91    Run,
92    #[token("with")]
93    With,
94    #[token("now")]
95    Now,
96    #[token("timeline")]
97    #[token("tl")]
98    Timeline,
99    #[token("wait")]
100    Wait,
101    #[token("index")]
102    Index,
103    #[token("action")]
104    Action,
105    #[token("duration")]
106    Duration,
107
108    // Type keywords
109    #[token("String")]
110    StringType,
111    #[token("Number")]
112    NumberType,
113    #[token("Boolean")]
114    #[token("Bool")]
115    BooleanType,
116
117    // Boolean literals
118    #[token("true")]
119    True,
120    #[token("false")]
121    False,
122    // endregion
123
124    // region Operators & Punctuation
125    #[token("->")]
126    Arrow,
127    #[token(":")]
128    Colon,
129    #[token(",")]
130    Comma,
131    #[token(";")]
132    Semicolon,
133    #[token(".")]
134    Dot,
135    #[token("{")]
136    LeftBrace,
137    #[token("}")]
138    RightBrace,
139    #[token("[")]
140    LeftBracket,
141    #[token("]")]
142    RightBracket,
143    #[token("(")]
144    LeftParen,
145    #[token(")")]
146    RightParen,
147    #[token("=")]
148    Equals,
149    #[token("<")]
150    Less,
151    #[token(">")]
152    Greater,
153    #[token("<=")]
154    LessEqual,
155    #[token(">=")]
156    GreaterEqual,
157    #[token("==")]
158    EqualEqual,
159    #[token("!=")]
160    NotEqual,
161    #[token("&&")]
162    And,
163    #[token("||")]
164    Or,
165    #[token("!")]
166    Not,
167    // endregion
168
169    // region Literals
170    // Triple-quoted multiline string: """..."""
171    #[token("\"\"\"", lex_triple_quoted_string)]
172    TripleQuotedString(&'a str),
173
174    #[regex(r#""([^"\\]|\\.)*""#, |lex| {
175        let s = lex.slice();
176        &s[1..s.len()-1]
177    })]
178    #[regex(r#"'([^'\\]|\\.)*'"#, |lex| {
179        let s = lex.slice();
180        &s[1..s.len()-1]
181    })]
182    String(&'a str),
183
184    // Interpolated string: $"text {expression} more text"
185    // Using callback to properly handle nested quotes
186    #[token("$\"", lex_interpolated_string)]
187    InterpolatedString(&'a str),
188
189    #[regex(r"[0-9]+(\.[0-9]+)?")]
190    Number(&'a str),
191
192    #[regex(r"[A-Za-z_][A-Za-z0-9_]*")]
193    Identifier(&'a str),
194    // endregion
195}
196
197/// Lexical analysis result containing token information and position
198#[derive(Debug, Clone)]
199pub struct TokenInfo<'a> {
200    pub token: Token<'a>,
201    pub start: usize,
202    pub end: usize,
203    pub text: &'a str,
204}
205
206/// Public lexical analysis interface for LSP and other external components
207pub fn tokenize(input: &str) -> Vec<TokenInfo<'_>> {
208    use logos::Logos;
209
210    let mut lexer = Token::lexer(input);
211    let mut tokens = Vec::new();
212
213    while let Some(token_result) = lexer.next() {
214        match token_result {
215            Ok(token) => {
216                let span = lexer.span();
217                tokens.push(TokenInfo {
218                    token,
219                    start: span.start,
220                    end: span.end,
221                    text: &input[span.start..span.end],
222                });
223            }
224            Err(_) => {
225                let span = lexer.span();
226                tokens.push(TokenInfo {
227                    token: Token::Error,
228                    start: span.start,
229                    end: span.end,
230                    text: &input[span.start..span.end],
231                });
232            }
233        }
234    }
235
236    tokens
237}
238
239impl fmt::Display for Token<'_> {
240    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
241        use Token::*;
242        match self {
243            Error => write!(f, "Error"),
244
245            SingleLineComment(s) => write!(f, "{}", s),
246            MultiLineComment(s) => write!(f, "{}", s),
247
248            Node => write!(f, "node"),
249            Text => write!(f, "text"),
250            Line => write!(f, "line"),
251            Events => write!(f, "events"),
252            Choice => write!(f, "choice"),
253            Fn => write!(f, "fn"),
254            Return => write!(f, "return"),
255            Break => write!(f, "break"),
256            When => write!(f, "when"),
257            Let => write!(f, "let"),
258            Const => write!(f, "const"),
259            Pub => write!(f, "pub"),
260            Enum => write!(f, "enum"),
261            Branch => write!(f, "branch"),
262            If => write!(f, "if"),
263            Else => write!(f, "else"),
264            Event => write!(f, "event"),
265            Run => write!(f, "run"),
266            With => write!(f, "with"),
267            Now => write!(f, "now"),
268            Timeline => write!(f, "timeline"),
269            Wait => write!(f, "wait"),
270            Index => write!(f, "index"),
271            Action => write!(f, "action"),
272            Duration => write!(f, "duration"),
273
274            StringType => write!(f, "String"),
275            NumberType => write!(f, "Number"),
276            BooleanType => write!(f, "Boolean"),
277            True => write!(f, "true"),
278            False => write!(f, "false"),
279
280            Arrow => write!(f, "->"),
281            Colon => write!(f, ":"),
282            Comma => write!(f, ","),
283            Semicolon => write!(f, ";"),
284            Dot => write!(f, "."),
285            LeftBrace => write!(f, "{{"),
286            RightBrace => write!(f, "}}"),
287            LeftBracket => write!(f, "["),
288            RightBracket => write!(f, "]"),
289            LeftParen => write!(f, "("),
290            RightParen => write!(f, ")"),
291            Equals => write!(f, "="),
292            Less => write!(f, "<"),
293            Greater => write!(f, ">"),
294            LessEqual => write!(f, "<="),
295            GreaterEqual => write!(f, ">="),
296            EqualEqual => write!(f, "=="),
297            NotEqual => write!(f, "!="),
298            And => write!(f, "&&"),
299            Or => write!(f, "||"),
300            Not => write!(f, "!"),
301
302            TripleQuotedString(s) => write!(f, "\"\"\"{}\"\"\"", s),
303            String(s) => write!(f, "\"{}\"", s),
304            InterpolatedString(s) => write!(f, "$\"{}\"", s),
305            Number(s) => write!(f, "{}", s),
306            Identifier(s) => write!(f, "{}", s),
307        }
308    }
309}
310
311fn lex_interpolated_string<'a>(lex: &mut logos::Lexer<'a, Token<'a>>) -> Option<&'a str> {
312    let start = lex.span().end;
313    let source = lex.source();
314    let bytes = source.as_bytes();
315
316    let mut pos = start;
317    let mut depth = 0;
318    let mut in_string_literal = false;
319    let mut escape_next = false;
320
321    while pos < bytes.len() {
322        let ch = bytes[pos] as char;
323        pos += 1;
324
325        if escape_next {
326            escape_next = false;
327            continue;
328        }
329
330        if ch == '\\' {
331            escape_next = true;
332            continue;
333        }
334
335        if ch == '"' && depth == 0 && !in_string_literal {
336            // Found the closing quote
337            lex.bump(pos - start);
338            let content_start = start;
339            let content_end = pos - 1;
340            return Some(&source[content_start..content_end]);
341        }
342
343        if ch == '"' {
344            in_string_literal = !in_string_literal;
345            continue;
346        }
347
348        if !in_string_literal {
349            if ch == '{' {
350                depth += 1;
351            } else if ch == '}' && depth > 0 {
352                depth -= 1;
353            }
354        }
355    }
356
357    None
358}
359
360/// Lexes a triple-quoted string: """..."""
361/// Returns the content between the opening and closing triple quotes.
362fn lex_triple_quoted_string<'a>(lex: &mut logos::Lexer<'a, Token<'a>>) -> Option<&'a str> {
363    let start = lex.span().end; // Position after opening """
364    let source = lex.source();
365    let bytes = source.as_bytes();
366
367    let mut pos = start;
368
369    // Look for closing """
370    while pos + 2 < bytes.len() {
371        if bytes[pos] == b'"' && bytes[pos + 1] == b'"' && bytes[pos + 2] == b'"' {
372            // Found closing """
373            lex.bump(pos - start + 3); // Include the closing """
374            return Some(&source[start..pos]);
375        }
376        pos += 1;
377    }
378
379    // Check edge case: """ at the very end
380    if pos + 2 == bytes.len()
381        && bytes[pos] == b'"'
382        && bytes[pos + 1] == b'"'
383        && bytes.get(pos + 2) == Some(&b'"')
384    {
385        lex.bump(pos - start + 3);
386        return Some(&source[start..pos]);
387    }
388
389    // No closing """ found
390    None
391}
392
393pub(crate) fn lex_with_output(input: &str) -> Vec<Token<'_>> {
394    let lex = Token::lexer(input);
395    let mut tokens = Vec::new();
396
397    println!();
398    println!("{}", "(Mortar) Lexer output:".green());
399
400    for result in lex {
401        match result {
402            Ok(token) => {
403                print!("{:?} ", token);
404                tokens.push(token);
405            }
406            Err(_) => {
407                println!("{}", "Lexer error encountered!".red());
408                break;
409            }
410        }
411    }
412
413    println!("\n");
414    tokens
415}
416
417#[allow(dead_code)]
418pub(crate) fn lex_silent(input: &str) -> Vec<Token<'_>> {
419    let lex = Token::lexer(input);
420    let mut tokens = Vec::new();
421
422    for result in lex {
423        match result {
424            Ok(token) => {
425                tokens.push(token);
426            }
427            Err(_) => {
428                break;
429            }
430        }
431    }
432
433    tokens
434}
mortar_compiler/token.rs

mortar_compiler/
token.rs
