caixa_ast/
lexer.rs

1//! Lisp lexer — scans source into tokens with byte spans.
2//!
3//! Implementation: thin wrapper over [`logos`](https://docs.rs/logos)
4//! 0.14. The hand-rolled byte-level lexer that lived here previously
5//! shipped two latent bugs (UTF-8 mishandling, unterminated-string
6//! detection) and was not maintainable as the syntax grew. logos
7//! delegates regex/UTF-8 to its DFA engine and exposes byte spans
8//! directly, so this file shrinks to atoms + a few callbacks while
9//! getting strictly better correctness.
10//!
11//! Token alphabet (unchanged — parser.rs needs no edits):
12//!   - `(` `)` — list delimiters
13//!   - `'` `` ` `` `,` `,@` — reader macros
14//!   - `"…"` — strings, with `\"` `\\` `\n` `\t` `\r` escapes
15//!   - `#t` / `#f` — booleans
16//!   - `nil` — the nil atom
17//!   - integers / floats with optional sign
18//!   - `:name-like` — keywords
19//!   - `; …` — line comments
20//!   - `\n+` (with surrounding spaces/`\r`/`\t`) — newline runs (carries
21//!     the line count so the parser can decide blank-line trivia)
22//!   - ` `/`\t` — whitespace (no count needed)
23//!   - everything else is a symbol
24
25use std::num::{ParseFloatError, ParseIntError};
26
27use logos::{Lexer, Logos};
28use thiserror::Error;
29
30use crate::span::Span;
31
32#[derive(Debug, Clone, PartialEq)]
33pub enum TokenKind {
34    /// A verbatim `#!…` first line. See [`crate::trivia::TriviaKind::Shebang`].
35    Shebang(String),
36    LParen,
37    RParen,
38    LBrace,
39    RBrace,
40    LBracket,
41    RBracket,
42    Quote,
43    Quasiquote,
44    Unquote,
45    UnquoteSplice,
46    Str(String),
47    Int(i64),
48    Float(f64),
49    Bool(bool),
50    Nil,
51    Symbol(String),
52    Keyword(String),
53    LineComment(String),
54    Newlines(u32),
55    Whitespace,
56}
57
58#[derive(Debug, Clone, PartialEq)]
59pub struct Token {
60    pub kind: TokenKind,
61    pub span: Span,
62}
63
64#[derive(Debug, Default, Error, PartialEq, Eq, Clone)]
65pub enum LexError {
66    #[default]
67    #[error("unrecognized token")]
68    Unrecognized,
69    #[error("unterminated string at offset {0}")]
70    UnterminatedString(u32),
71    #[error("invalid escape sequence \\{1} at offset {0}")]
72    BadEscape(u32, char),
73    #[error("invalid number literal at offset {0}: {1}")]
74    BadInt(u32, String),
75    #[error("invalid float literal at offset {0}: {1}")]
76    BadFloat(u32, String),
77    #[error("unexpected character {1:?} at offset {0}")]
78    UnexpectedChar(u32, char),
79}
80
81impl From<(u32, ParseIntError)> for LexError {
82    fn from(v: (u32, ParseIntError)) -> Self {
83        Self::BadInt(v.0, v.1.to_string())
84    }
85}
86
87impl From<(u32, ParseFloatError)> for LexError {
88    fn from(v: (u32, ParseFloatError)) -> Self {
89        Self::BadFloat(v.0, v.1.to_string())
90    }
91}
92
93// ── logos token enum ──────────────────────────────────────────────
94//
95// Internal to the module. We translate to the public `TokenKind` /
96// `Token` types in `tokenize` so the parser keeps its existing API.
97
98#[derive(Logos, Debug, PartialEq)]
99#[logos(error = LexError)]
100enum LogosKind {
101    #[token("(")]
102    LParen,
103
104    #[token(")")]
105    RParen,
106
107    // The brace/vector dialect. `{ :k v }` and `[ a b ]` are REAL
108    // SYNTAX, not sugar — theory/TATARA-LISP-CONSOLIDATION.md D4, on the
109    // evidence of 62 live caixa.lisp manifests that author nested maps
110    // (`:package { :name "…" :version "…" }`) and are consumed today.
111    //
112    // Until now these four bytes had no token here at all: they fell
113    // through to the Symbol regex below, so a map lexed as a flat run of
114    // atoms with `{` and `}` as ordinary symbols. That made every real
115    // manifest an odd-length list to the printer, which is why `feira
116    // fmt` abandoned the key/value shape and exploded them one atom per
117    // line. caixa-ts/grammar.js has had `map` and `vector` rules from the
118    // start and its header says the two grammars are kept in lockstep —
119    // this closes the gap on the Rust side.
120    #[token("{")]
121    LBrace,
122
123    #[token("}")]
124    RBrace,
125
126    #[token("[")]
127    LBracket,
128
129    #[token("]")]
130    RBracket,
131
132    #[token("'")]
133    Quote,
134
135    #[token("`")]
136    Quasiquote,
137
138    // `,@` MUST come before `,` so it wins on the longest-match.
139    #[token(",@")]
140    UnquoteSplice,
141
142    #[token(",")]
143    Unquote,
144
145    #[token("#t", |_| true)]
146    #[token("#f", |_| false)]
147    Bool(bool),
148
149    // Strings: opening `"`, then repeated non-`\`/non-`"` chars OR
150    // backslash-something escapes, then closing `"`. The callback
151    // unescapes the body. UTF-8 is delegated to logos / regex.
152    #[regex(r#""(?:[^"\\]|\\.)*""#, lex_string_body)]
153    Str(String),
154
155    // Numbers: integer first (priority 3 so it doesn't lose to symbol).
156    // Float separately — has a `.` or `e/E`.
157    #[regex(r"[+-]?[0-9]+", priority = 3, callback = parse_int)]
158    Int(i64),
159
160    #[regex(
161        r"[+-]?(?:[0-9]+\.[0-9]*|\.[0-9]+|[0-9]+[eE][+-]?[0-9]+|[0-9]+\.[0-9]*[eE][+-]?[0-9]+|\.[0-9]+[eE][+-]?[0-9]+)",
162        priority = 3,
163        callback = parse_float
164    )]
165    Float(f64),
166
167    // Keyword: `:` followed by atom chars. `{}[]` terminate it, or
168    // `:version "0.3.0"}` would lex the closing brace into the keyword.
169    #[regex(":[^\\s()'`,\";\\{\\}\\[\\]]+", |lex| lex.slice()[1..].to_string())]
170    Keyword(String),
171
172    // Line comment: `;` to end of line. The leading `;` is NOT
173    // included in the captured body, matching the prior behavior.
174    #[regex(r";[^\n]*", |lex| {
175        let s = lex.slice();
176        // strip the leading ';'
177        s[1..].to_string()
178    })]
179    LineComment(String),
180
181    // Newline runs: any \n followed by whitespace including more \n's.
182    // The callback counts \n bytes so blank-line detection works
183    // exactly as before (count >= 2 means a blank line).
184    #[regex(r"[\n][ \t\r\n]*", count_newlines)]
185    Newlines(u32),
186
187    // Pure-space whitespace (no newline). Intentional and separate
188    // from Newlines so the parser can skip both without losing
189    // line-count info.
190    #[regex(r"[ \t\r]+")]
191    Whitespace,
192
193    // Anything else is a symbol or `nil`. The atom-terminator set
194    // matches the prior is_atom_terminator (space/tab/cr/lf/parens/
195    // single-quote/backtick/comma/double-quote/semicolon) PLUS `#`,
196    // which is the boolean / reader-macro dispatch prefix and never
197    // appears inside a tatara-lisp symbol. Excluding `#` here lets
198    // adjacent forms like `#t#f` tokenize as two booleans rather
199    // than a single `#t#f` symbol.
200    // `{}[]` join the terminator set for the same reason `()` are in it:
201    // they are structural delimiters now, so `{:name` must lex as LBrace
202    // + Keyword rather than as one symbol `{:name`. caixa-ts states the
203    // same set as an ALLOW-list (`[A-Za-z_+\-*/=<>?!%&~.]…`), which
204    // already excluded braces — this is the Rust side catching up.
205    #[regex(
206        "[^\\s()'`,\";#\\{\\}\\[\\]][^\\s()'`,\";#\\{\\}\\[\\]]*",
207        |lex| lex.slice().to_string()
208    )]
209    Symbol(String),
210}
211
212// ── callbacks ─────────────────────────────────────────────────────
213
214fn lex_string_body(lex: &mut Lexer<LogosKind>) -> Result<String, LexError> {
215    let raw = lex.slice();
216    debug_assert!(raw.starts_with('"') && raw.ends_with('"'));
217    let inner = &raw[1..raw.len() - 1];
218    let span_start = u32::try_from(lex.span().start).unwrap_or(u32::MAX);
219
220    let mut out = String::with_capacity(inner.len());
221    let mut chars = inner.char_indices();
222    while let Some((i, c)) = chars.next() {
223        if c == '\\' {
224            match chars.next() {
225                Some((_, 'n')) => out.push('\n'),
226                Some((_, 't')) => out.push('\t'),
227                Some((_, 'r')) => out.push('\r'),
228                Some((_, '"')) => out.push('"'),
229                Some((_, '\\')) => out.push('\\'),
230                // An UNKNOWN escape yields the character itself, dropping
231                // the backslash — matching the canonical reader exactly
232                // (`tatara-lisp/src/reader.rs`: `other => other`).
233                //
234                // Rejecting these was a real divergence, not strictness:
235                // `actions/db-migrate/run.tlisp` carries a grep pattern
236                // written `'Applied\|migration\|up to date'`, which the
237                // canonical reader accepts and this lexer refused, so the
238                // formatter could not read a file the runtime runs. Two
239                // readers disagreeing about what the language IS is the
240                // concrete cost of the fleet's 13 independent
241                // S-expression readers; here the canonical one is the
242                // oracle and this one conforms.
243                Some((_, other)) => out.push(other),
244                None => {
245                    return Err(LexError::BadEscape(
246                        span_start + 1 + u32::try_from(i).unwrap_or(0),
247                        '\\',
248                    ));
249                }
250            }
251        } else {
252            out.push(c);
253        }
254    }
255    Ok(out)
256}
257
258fn parse_int(lex: &mut Lexer<LogosKind>) -> Result<i64, LexError> {
259    let span_start = u32::try_from(lex.span().start).unwrap_or(u32::MAX);
260    lex.slice()
261        .parse::<i64>()
262        .map_err(|e| LexError::BadInt(span_start, e.to_string()))
263}
264
265fn parse_float(lex: &mut Lexer<LogosKind>) -> Result<f64, LexError> {
266    let span_start = u32::try_from(lex.span().start).unwrap_or(u32::MAX);
267    lex.slice()
268        .parse::<f64>()
269        .map_err(|e| LexError::BadFloat(span_start, e.to_string()))
270}
271
272fn count_newlines(lex: &mut Lexer<LogosKind>) -> u32 {
273    let s = lex.slice();
274    let n = s.bytes().filter(|&b| b == b'\n').count();
275    u32::try_from(n).unwrap_or(u32::MAX)
276}
277
278// ── public entry point ────────────────────────────────────────────
279
280/// Scan a source string into tokens. Trivia (whitespace, comments) is
281/// preserved — the parser filters what it doesn't need.
282pub fn tokenize(src: &str) -> Result<Vec<Token>, LexError> {
283    let mut out = Vec::new();
284
285    // A leading `#!` line is a shebang, not source. Emitted as its own
286    // token so it survives formatting verbatim; logos never sees it, since
287    // `#` is not otherwise part of the grammar. Only at offset 0 — a `#!`
288    // anywhere else is genuinely invalid and must still be an error.
289    let body_start = if src.starts_with("#!") {
290        let end = src.find('\n').unwrap_or(src.len());
291        out.push(Token {
292            kind: TokenKind::Shebang(src[..end].to_string()),
293            span: Span::new(0, u32::try_from(end).unwrap_or(u32::MAX)),
294        });
295        end
296    } else {
297        0
298    };
299
300    let mut lex = LogosKind::lexer(&src[body_start..]);
301
302    while let Some(result) = lex.next() {
303        let span = lex.span();
304        let span_start = u32::try_from(span.start + body_start).unwrap_or(u32::MAX);
305        let span_end = u32::try_from(span.end + body_start).unwrap_or(u32::MAX);
306        let span = Span::new(span_start, span_end);
307
308        match result {
309            Ok(kind) => {
310                let public = match kind {
311                    LogosKind::LParen => TokenKind::LParen,
312                    LogosKind::RParen => TokenKind::RParen,
313                    LogosKind::LBrace => TokenKind::LBrace,
314                    LogosKind::RBrace => TokenKind::RBrace,
315                    LogosKind::LBracket => TokenKind::LBracket,
316                    LogosKind::RBracket => TokenKind::RBracket,
317                    LogosKind::Quote => TokenKind::Quote,
318                    LogosKind::Quasiquote => TokenKind::Quasiquote,
319                    LogosKind::Unquote => TokenKind::Unquote,
320                    LogosKind::UnquoteSplice => TokenKind::UnquoteSplice,
321                    LogosKind::Bool(b) => TokenKind::Bool(b),
322                    LogosKind::Str(s) => TokenKind::Str(s),
323                    LogosKind::Int(i) => TokenKind::Int(i),
324                    LogosKind::Float(f) => TokenKind::Float(f),
325                    LogosKind::Keyword(s) => TokenKind::Keyword(s),
326                    LogosKind::LineComment(s) => TokenKind::LineComment(s),
327                    LogosKind::Newlines(n) => TokenKind::Newlines(n),
328                    LogosKind::Whitespace => TokenKind::Whitespace,
329                    LogosKind::Symbol(s) => {
330                        if s == "nil" {
331                            TokenKind::Nil
332                        } else {
333                            TokenKind::Symbol(s)
334                        }
335                    }
336                };
337                out.push(Token { kind: public, span });
338            }
339            Err(_) => {
340                // Unrecognized byte — most likely an unterminated
341                // string (since strings are the only multi-byte form
342                // that can fail to close). Distinguish them by source
343                // shape so the LexError carries the right variant.
344                let slice = lex.slice();
345                if slice.starts_with('"') {
346                    return Err(LexError::UnterminatedString(span_start));
347                }
348                let ch = slice.chars().next().unwrap_or(' ');
349                return Err(LexError::UnexpectedChar(span_start, ch));
350            }
351        }
352    }
353
354    Ok(out)
355}
356
357#[cfg(test)]
358mod tests {
359    use super::*;
360
361    fn kinds(src: &str) -> Vec<TokenKind> {
362        tokenize(src)
363            .unwrap()
364            .into_iter()
365            .map(|t| t.kind)
366            .filter(|k| !matches!(k, TokenKind::Whitespace | TokenKind::Newlines(_)))
367            .collect()
368    }
369
370    // `3.14` below is the *expected lex output* for the input string
371    // `"3.14"` — a float-literal round-trip fixture, not an approximation
372    // of `f64::consts::PI` used in a computation. `clippy::approx_constant`
373    // is deny-by-default (correctness group), so without this scoped allow
374    // `cargo clippy` aborts this crate with a hard error and never reports
375    // the rest of the workspace at all. Substituting `PI` here would break
376    // the round-trip the assertion exists to prove.
377    #[allow(
378        clippy::approx_constant,
379        reason = "float-literal lex fixture, not a PI approximation"
380    )]
381    #[test]
382    fn basic_atoms() {
383        assert_eq!(kinds("42"), vec![TokenKind::Int(42)]);
384        assert_eq!(kinds("3.14"), vec![TokenKind::Float(3.14)]);
385        assert_eq!(kinds("-7"), vec![TokenKind::Int(-7)]);
386        assert_eq!(kinds("#t"), vec![TokenKind::Bool(true)]);
387        assert_eq!(kinds("#f"), vec![TokenKind::Bool(false)]);
388        assert_eq!(kinds("nil"), vec![TokenKind::Nil]);
389        assert_eq!(kinds("\"hi\\n\""), vec![TokenKind::Str("hi\n".into())]);
390        assert_eq!(
391            kinds(":key-word"),
392            vec![TokenKind::Keyword("key-word".into())]
393        );
394        assert_eq!(kinds("my-sym"), vec![TokenKind::Symbol("my-sym".into())]);
395    }
396
397    #[test]
398    fn lists_and_readers() {
399        assert_eq!(
400            kinds("(a b)"),
401            vec![
402                TokenKind::LParen,
403                TokenKind::Symbol("a".into()),
404                TokenKind::Symbol("b".into()),
405                TokenKind::RParen,
406            ]
407        );
408        assert_eq!(
409            kinds("'x"),
410            vec![TokenKind::Quote, TokenKind::Symbol("x".into())]
411        );
412        assert_eq!(
413            kinds(",@xs"),
414            vec![TokenKind::UnquoteSplice, TokenKind::Symbol("xs".into())]
415        );
416    }
417
418    #[test]
419    fn line_comment() {
420        let toks = tokenize("; hello\nworld").unwrap();
421        assert!(matches!(toks[0].kind, TokenKind::LineComment(ref s) if s == " hello"));
422        assert!(matches!(toks[1].kind, TokenKind::Newlines(_)));
423        assert!(matches!(toks[2].kind, TokenKind::Symbol(ref s) if s == "world"));
424    }
425
426    #[test]
427    fn unterminated_string_errors() {
428        assert!(matches!(
429            tokenize(r#""oops"#),
430            Err(LexError::UnterminatedString(_))
431        ));
432    }
433
434    #[test]
435    fn utf8_in_string_round_trip() {
436        // Multi-byte chars (Greek, emoji, accented) must come back
437        // exactly — the previous byte-as-Latin-1 lexer mangled these.
438        let src = r#""π — émoji 🎉""#;
439        let toks = tokenize(src).unwrap();
440        match &toks[0].kind {
441            TokenKind::Str(s) => assert_eq!(s, "π — émoji 🎉"),
442            other => panic!("{other:?}"),
443        }
444    }
445
446    #[test]
447    fn newline_run_preserves_count() {
448        let toks = tokenize("a\n\n\nb").unwrap();
449        // a, newlines(3), b
450        assert!(matches!(toks[0].kind, TokenKind::Symbol(ref s) if s == "a"));
451        match toks[1].kind {
452            TokenKind::Newlines(n) => assert_eq!(n, 3),
453            ref other => panic!("{other:?}"),
454        }
455        assert!(matches!(toks[2].kind, TokenKind::Symbol(ref s) if s == "b"));
456    }
457
458    #[test]
459    fn float_with_exponent() {
460        assert_eq!(kinds("1.5e10"), vec![TokenKind::Float(1.5e10)]);
461        assert_eq!(kinds("1e-3"), vec![TokenKind::Float(1e-3)]);
462        assert_eq!(kinds("-2.5E2"), vec![TokenKind::Float(-2.5e2)]);
463    }
464
465    #[test]
466    fn bool_keyword_clash_handled() {
467        // `#t#f` should tokenize as two booleans (no separator
468        // required). Logos' longest-match handles this for free.
469        assert_eq!(
470            kinds("#t#f"),
471            vec![TokenKind::Bool(true), TokenKind::Bool(false)]
472        );
473    }
474}
caixa_ast/lexer.rs

caixa_ast/
lexer.rs
