bubbles/compiler/
lexer.rs

1//! Logos-based lexer that tokenises `.bub` source and expression strings.
2
3use logos::Logos;
4
5/// Resolves the `\"`, `\\`, and `\n` escapes in a string-literal body in a
6/// single pass. Unknown escapes are kept verbatim.
7fn unescape(body: &str) -> String {
8    let mut out = String::with_capacity(body.len());
9    let mut chars = body.chars();
10    while let Some(c) = chars.next() {
11        if c != '\\' {
12            out.push(c);
13            continue;
14        }
15        match chars.next() {
16            Some('"') => out.push('"'),
17            Some('n') => out.push('\n'),
18            Some('\\') | None => out.push('\\'),
19            Some(other) => {
20                out.push('\\');
21                out.push(other);
22            }
23        }
24    }
25    out
26}
27
28/// A lexical token produced by the lexer.
29#[derive(Logos, Debug, Clone, PartialEq)]
30#[logos(skip r"[ \t\r\f]+")] // skip horizontal whitespace; newlines are significant in the parser
31pub enum Token {
32    // ── literals ──────────────────────────────────────────────────────────────
33    /// Floating-point or integer literal.
34    #[regex(r"[0-9]+(\.[0-9]+)?", |lex| lex.slice().parse::<f64>().ok())]
35    Number(f64),
36
37    /// Double-quoted string literal.
38    #[regex(r#""([^"\\]|\\.)*""#, |lex| {
39        let s = lex.slice();
40        Some(unescape(&s[1..s.len() - 1]))
41    })]
42    Str(String),
43
44    // ── identifiers / keywords ─────────────────────────────────────────────────
45    /// Variable beginning with `$`.
46    #[regex(r"\$[A-Za-z_][A-Za-z0-9_]*", |lex| lex.slice().to_owned())]
47    Var(String),
48
49    /// Plain identifier or keyword.
50    #[regex(r"[A-Za-z_][A-Za-z0-9_]*", |lex| lex.slice().to_owned())]
51    Ident(String),
52
53    // ── delimiters ─────────────────────────────────────────────────────────────
54    /// `(` – opens a parenthesised sub-expression or argument list.
55    #[token("(")]
56    LParen,
57    /// `)` – closes a parenthesised sub-expression or argument list.
58    #[token(")")]
59    RParen,
60    /// `,` – argument separator.
61    #[token(",")]
62    Comma,
63    /// `<<` – opens a command/statement block.
64    #[token("<<")]
65    CmdOpen,
66    /// `>>` – closes a command/statement block.
67    #[token(">>")]
68    CmdClose,
69    /// `{` – opens an inline expression.
70    #[token("{")]
71    BraceOpen,
72    /// `}` – closes an inline expression.
73    #[token("}")]
74    BraceClose,
75
76    // ── arithmetic ─────────────────────────────────────────────────────────────
77    /// `+`
78    #[token("+")]
79    Plus,
80    /// `-`
81    #[token("-")]
82    Minus,
83    /// `*`
84    #[token("*")]
85    Star,
86    /// `/`
87    #[token("/")]
88    Slash,
89    /// `%`
90    #[token("%")]
91    Percent,
92
93    // ── comparison (order matters: `>=` before `>`) ───────────────────────────
94    /// `>=`
95    #[token(">=")]
96    Gte,
97    /// `<=`
98    #[token("<=")]
99    Lte,
100    /// `>`
101    #[token(">")]
102    Gt,
103    /// `<`
104    #[token("<")]
105    Lt,
106    /// `==`
107    #[token("==")]
108    EqEq,
109    /// `!=`
110    #[token("!=")]
111    Neq,
112
113    // ── logical ────────────────────────────────────────────────────────────────
114    /// `&&`
115    #[token("&&")]
116    AndAnd,
117    /// `||`
118    #[token("||")]
119    OrOr,
120    /// `!`
121    #[token("!")]
122    Bang,
123
124    // ── assignment / misc ──────────────────────────────────────────────────────
125    /// `=` (used in `<<set $x = …>>`)
126    #[token("=")]
127    Eq,
128    /// `:`
129    #[token(":")]
130    Colon,
131    /// `->`
132    #[token("->")]
133    Arrow,
134    /// `=>`
135    #[token("=>")]
136    FatArrow,
137    /// `---` body-start delimiter.
138    #[token("---")]
139    BodyStart,
140    /// `===` node-end delimiter.
141    #[token("===")]
142    NodeEnd,
143    /// `#` tag prefix.
144    #[token("#")]
145    Hash,
146    /// Newline.
147    #[token("\n")]
148    Newline,
149}
150
151/// A spanned token pair.
152pub type Spanned = (Token, std::ops::Range<usize>);
153
154/// Lexes `input` into a [`Vec`] of spanned tokens, returning an error on
155/// any character that does not match a known token.
156///
157/// # Errors
158///
159/// Returns [`crate::error::DialogueError::Parse`] with `file` / `line` context
160/// when an unrecognised character is encountered, so the caller receives a
161/// precise pointer into the source rather than a confusing downstream failure.
162pub fn tokenise(input: &str, file: &str, line: usize) -> crate::error::Result<Vec<Spanned>> {
163    let mut tokens = Vec::new();
164    for (result, span) in Token::lexer(input).spanned() {
165        if let Ok(tok) = result {
166            tokens.push((tok, span));
167        } else {
168            let ch = input[span].chars().next().unwrap_or('?');
169            return Err(crate::error::DialogueError::Parse {
170                file: file.to_owned(),
171                line,
172                message: format!(
173                    "unexpected character `{ch}` in expression; \
174                     did you mean `$` for a variable?"
175                ),
176            });
177        }
178    }
179    Ok(tokens)
180}
181
182#[cfg(test)]
183#[path = "lexer_tests.rs"]
184mod tests;
bubbles/compiler/lexer.rs

bubbles/compiler/
lexer.rs
