brink_syntax/lexer/
mod.rs

1mod ident;
2mod punctuation;
3#[cfg(test)]
4mod tests;
5
6use crate::SyntaxKind;
7pub use ident::{classify_keyword, is_ident_char, is_ink_ident_codepoint, scan_ident};
8pub use punctuation::lex_punctuation;
9
10/// Lex `source` into a sequence of `(SyntaxKind, slice)` pairs.
11///
12/// Every byte of `source` appears in exactly one token (lossless).
13/// The only mutable state is a `string_depth: u32` tracking nested string
14/// interpolation. When `string_depth > 0`, we are inside a string and lex
15/// `STRING_TEXT`/`STRING_ESCAPE`/`QUOTE`/`L_BRACE`. `{` inside a string
16/// increments depth and exits string mode; `}` outside a string with
17/// `depth > 0` decrements depth and re-enters string mode.
18pub fn lex(source: &str) -> Vec<(SyntaxKind, &str)> {
19    Lexer::new(source).run()
20}
21
22struct Lexer<'src> {
23    source: &'src str,
24    bytes: &'src [u8],
25    pos: usize,
26    /// Nesting depth of string interpolations.
27    /// 0 = outside any string. 1 = inside a string. 2 = inside an interpolation
28    /// inside a string, etc. Odd = in string mode, even = in code mode (but
29    /// with pending string closings to track).
30    string_depth: u32,
31    tokens: Vec<(SyntaxKind, &'src str)>,
32}
33
34impl<'src> Lexer<'src> {
35    fn new(source: &'src str) -> Self {
36        Self {
37            source,
38            bytes: source.as_bytes(),
39            pos: 0,
40            string_depth: 0,
41            tokens: Vec::new(),
42        }
43    }
44
45    fn in_string(&self) -> bool {
46        self.string_depth % 2 == 1
47    }
48
49    fn run(mut self) -> Vec<(SyntaxKind, &'src str)> {
50        while self.pos < self.bytes.len() {
51            if self.in_string() {
52                self.lex_string_token();
53            } else {
54                self.lex_code_token();
55            }
56        }
57        self.tokens
58    }
59
60    fn emit(&mut self, kind: SyntaxKind, start: usize) {
61        self.tokens.push((kind, &self.source[start..self.pos]));
62    }
63
64    // ── String-mode lexing ──────────────────────────────────────
65
66    fn lex_string_token(&mut self) {
67        let start = self.pos;
68        let b = self.bytes[self.pos];
69
70        // Closing quote — pop one level of string nesting
71        if b == b'"' {
72            self.pos += 1;
73            self.string_depth -= 1;
74            self.emit(SyntaxKind::QUOTE, start);
75            return;
76        }
77
78        // Escape sequence
79        if b == b'\\' && self.pos + 1 < self.bytes.len() {
80            let next = self.bytes[self.pos + 1];
81            if matches!(next, b'n' | b't' | b'\\' | b'"') {
82                self.pos += 2;
83                self.emit(SyntaxKind::STRING_ESCAPE, start);
84                return;
85            }
86        }
87
88        // Opening brace — enter interpolation (push depth)
89        if b == b'{' {
90            self.pos += 1;
91            self.string_depth += 1;
92            self.emit(SyntaxKind::L_BRACE, start);
93            return;
94        }
95
96        // Brackets — emit as L_BRACKET/R_BRACKET even in string mode so the
97        // parser can find choice bracket boundaries regardless of context.
98        // In ink, `[` and `]` in choice content are always bracket delimiters
99        // even when they appear inside quoted text like `"tired[."]`.
100        if b == b'[' {
101            self.pos += 1;
102            self.emit(SyntaxKind::L_BRACKET, start);
103            return;
104        }
105        if b == b']' {
106            self.pos += 1;
107            self.emit(SyntaxKind::R_BRACKET, start);
108            return;
109        }
110
111        // Glue `<>` — recognized even in string mode because in ink `"` in
112        // content text is a literal character (dialogue quotes), not a string
113        // delimiter. Glue must break out of STRING_TEXT so the parser sees it.
114        if b == b'<' && self.pos + 1 < self.bytes.len() && self.bytes[self.pos + 1] == b'>' {
115            self.pos += 2;
116            self.emit(SyntaxKind::GLUE, start);
117            return;
118        }
119
120        // Newline terminates an unterminated string
121        if b == b'\n' || b == b'\r' {
122            self.pos += 1;
123            if b == b'\r' && self.pos < self.bytes.len() && self.bytes[self.pos] == b'\n' {
124                self.pos += 1;
125            }
126            self.string_depth -= 1;
127            self.emit(SyntaxKind::NEWLINE, start);
128            return;
129        }
130
131        // `STRING_TEXT`: run of non-special chars
132        self.pos += 1;
133        while self.pos < self.bytes.len() {
134            match self.bytes[self.pos] {
135                b'"' | b'\\' | b'{' | b'\n' | b'\r' | b'[' | b']' => break,
136                // Break on `<>` (glue)
137                b'<' if self.pos + 1 < self.bytes.len() && self.bytes[self.pos + 1] == b'>' => {
138                    break;
139                }
140                _ => self.pos += 1,
141            }
142        }
143        self.emit(SyntaxKind::STRING_TEXT, start);
144    }
145
146    // ── Code-mode lexing ────────────────────────────────────────
147
148    fn lex_code_token(&mut self) {
149        let start = self.pos;
150        let b = self.bytes[self.pos];
151
152        // Newlines
153        if b == b'\n' {
154            self.pos += 1;
155            self.emit(SyntaxKind::NEWLINE, start);
156            return;
157        }
158        if b == b'\r' {
159            self.pos += 1;
160            if self.pos < self.bytes.len() && self.bytes[self.pos] == b'\n' {
161                self.pos += 1;
162            }
163            self.emit(SyntaxKind::NEWLINE, start);
164            return;
165        }
166
167        // UTF-8 BOM (U+FEFF) — treat as whitespace trivia for lossless roundtrip.
168        if b == 0xEF
169            && self.pos + 2 < self.bytes.len()
170            && self.bytes[self.pos + 1] == 0xBB
171            && self.bytes[self.pos + 2] == 0xBF
172        {
173            self.pos += 3;
174            self.emit(SyntaxKind::WHITESPACE, start);
175            return;
176        }
177
178        // Whitespace (spaces + tabs only)
179        if b == b' ' || b == b'\t' {
180            self.pos += 1;
181            while self.pos < self.bytes.len()
182                && (self.bytes[self.pos] == b' ' || self.bytes[self.pos] == b'\t')
183            {
184                self.pos += 1;
185            }
186            self.emit(SyntaxKind::WHITESPACE, start);
187            return;
188        }
189
190        // Comments (before punctuation, since `/` is also SLASH)
191        if b == b'/'
192            && let Some(kind) = self.try_lex_comment()
193        {
194            self.emit(kind, start);
195            return;
196        }
197
198        // Closing brace — if `string_depth > 0`, re-enter string mode
199        if b == b'}' && self.string_depth > 0 {
200            self.pos += 1;
201            self.string_depth -= 1;
202            self.emit(SyntaxKind::R_BRACE, start);
203            return;
204        }
205
206        // Multi-char punctuation (greedy, longest-first)
207        if let Some((kind, advance)) = lex_punctuation(self.bytes, self.pos) {
208            self.pos += advance;
209            if kind == SyntaxKind::QUOTE {
210                self.string_depth += 1;
211            }
212            self.emit(kind, start);
213            return;
214        }
215
216        // Digits — could be INTEGER, FLOAT, or digit-start IDENT
217        if b.is_ascii_digit() {
218            self.lex_number_or_ident();
219            return;
220        }
221
222        // Identifiers (and keywords)
223        if is_ident_char(self.bytes, self.pos) {
224            let end = scan_ident(self.bytes, self.pos + char_len_utf8(self.bytes, self.pos));
225            let text = &self.source[start..end];
226            let kind = classify_keyword(text);
227            self.pos = end;
228            self.tokens.push((kind, text));
229            return;
230        }
231
232        // Anything else is an error token (one char at a time)
233        self.pos += char_len_utf8(self.bytes, self.pos);
234        self.emit(SyntaxKind::ERROR_TOKEN, start);
235    }
236
237    /// Try to lex a comment starting at current position (which is `/`).
238    /// Returns `Some(kind)` and advances `self.pos` if successful, `None` otherwise.
239    fn try_lex_comment(&mut self) -> Option<SyntaxKind> {
240        if self.pos + 1 >= self.bytes.len() {
241            return None;
242        }
243        match self.bytes[self.pos + 1] {
244            b'/' => {
245                // Line comment — consume through end of line (not including newline)
246                self.pos += 2;
247                while self.pos < self.bytes.len()
248                    && self.bytes[self.pos] != b'\n'
249                    && self.bytes[self.pos] != b'\r'
250                {
251                    self.pos += 1;
252                }
253                Some(SyntaxKind::LINE_COMMENT)
254            }
255            b'*' => {
256                // Block comment — consume through `*/`
257                self.pos += 2;
258                loop {
259                    if self.pos + 1 < self.bytes.len()
260                        && self.bytes[self.pos] == b'*'
261                        && self.bytes[self.pos + 1] == b'/'
262                    {
263                        self.pos += 2;
264                        break;
265                    }
266                    if self.pos >= self.bytes.len() {
267                        break; // unterminated
268                    }
269                    self.pos += 1;
270                }
271                Some(SyntaxKind::BLOCK_COMMENT)
272            }
273            _ => None,
274        }
275    }
276
277    /// Lex a sequence starting with a digit. Could be:
278    /// - `INTEGER` (digits, NOT followed by an identifier character)
279    /// - `FLOAT` (digits.digits, NOT followed by an identifier character)
280    /// - digit-start `IDENT` (digits followed by an identifier character)
281    fn lex_number_or_ident(&mut self) {
282        let start = self.pos;
283
284        // Consume leading digits
285        while self.pos < self.bytes.len() && self.bytes[self.pos].is_ascii_digit() {
286            self.pos += 1;
287        }
288
289        // Check for digit-start identifier: digits followed by an identifier character
290        if self.pos < self.bytes.len() && is_ident_char(self.bytes, self.pos) {
291            self.pos = scan_ident(self.bytes, self.pos + char_len_utf8(self.bytes, self.pos));
292            self.emit(SyntaxKind::IDENT, start);
293            return;
294        }
295
296        // Check for float: digits.digits (NOT followed by an identifier character)
297        if self.pos < self.bytes.len()
298            && self.bytes[self.pos] == b'.'
299            && self.pos + 1 < self.bytes.len()
300            && self.bytes[self.pos + 1].is_ascii_digit()
301        {
302            self.pos += 1; // skip the dot
303            while self.pos < self.bytes.len() && self.bytes[self.pos].is_ascii_digit() {
304                self.pos += 1;
305            }
306            // If followed by an identifier character, the whole thing is an ident
307            if self.pos < self.bytes.len() && is_ident_char(self.bytes, self.pos) {
308                self.pos = scan_ident(self.bytes, self.pos + char_len_utf8(self.bytes, self.pos));
309                self.emit(SyntaxKind::IDENT, start);
310                return;
311            }
312            self.emit(SyntaxKind::FLOAT, start);
313            return;
314        }
315
316        // Plain integer
317        self.emit(SyntaxKind::INTEGER, start);
318    }
319}
320
321/// Length of the UTF-8 character starting at `pos` (1–4 bytes).
322pub(crate) fn char_len_utf8(bytes: &[u8], pos: usize) -> usize {
323    let b = bytes[pos];
324    if b < 0x80 {
325        1
326    } else if b < 0xE0 {
327        2
328    } else if b < 0xF0 {
329        3
330    } else {
331        4
332    }
333}
brink_syntax/lexer/mod.rs

brink_syntax/lexer/
mod.rs
