cljrs_reader/
lexer.rs

1// CljxError embeds NamedSource<String> for miette diagnostics, which is
2// unavoidably large. Suppress the false-positive for every returning function.
3#![allow(clippy::result_large_err)]
4
5use std::sync::Arc;
6
7use miette::NamedSource;
8
9use cljrs_types::error::{CljxError, CljxResult};
10use cljrs_types::span::Span;
11
12use crate::chars::{is_symbol_char, is_symbol_start};
13use crate::token::Token;
14
15// ─── Lexer ───────────────────────────────────────────────────────────────────
16
17pub struct Lexer {
18    source: Arc<String>,
19    file: Arc<String>,
20    pos: usize, // byte offset, always on a char boundary
21    line: u32,  // 1-based
22    col: u32,   // 1-based byte offset from line start
23}
24
25impl Lexer {
26    pub fn new(source: String, file: String) -> Self {
27        Self {
28            source: Arc::new(source),
29            file: Arc::new(file),
30            pos: 0,
31            line: 1,
32            col: 1,
33        }
34    }
35
36    // ── Public getters ────────────────────────────────────────────────────
37
38    pub fn source(&self) -> &Arc<String> {
39        &self.source
40    }
41
42    pub fn file(&self) -> &Arc<String> {
43        &self.file
44    }
45
46    // ── Low-level helpers ─────────────────────────────────────────────────
47
48    fn peek(&self) -> Option<char> {
49        self.source[self.pos..].chars().next()
50    }
51
52    fn peek_next(&self) -> Option<char> {
53        let mut chars = self.source[self.pos..].chars();
54        chars.next(); // skip current
55        chars.next()
56    }
57
58    fn advance(&mut self) -> Option<char> {
59        let ch = self.peek()?;
60        self.pos += ch.len_utf8();
61        if ch == '\n' {
62            self.line += 1;
63            self.col = 1;
64        } else {
65            self.col += ch.len_utf8() as u32;
66        }
67        Some(ch)
68    }
69
70    fn span_from(&self, start_pos: usize, start_line: u32, start_col: u32) -> Span {
71        Span::new(
72            Arc::clone(&self.file),
73            start_pos,
74            self.pos,
75            start_line,
76            start_col,
77        )
78    }
79
80    fn make_error(&self, msg: impl Into<String>, span: Span) -> CljxError {
81        CljxError::ReadError {
82            message: msg.into(),
83            span: Some(miette::SourceSpan::from(span)),
84            src: NamedSource::new((*self.file).clone(), (*self.source).clone()),
85        }
86    }
87
88    /// Consume characters while `is_symbol_char` holds, returning the collected
89    /// string.
90    fn read_symbol_chars(&mut self) -> String {
91        let mut buf = String::new();
92        while let Some(ch) = self.peek() {
93            if is_symbol_char(ch) {
94                buf.push(ch);
95                self.advance();
96            } else {
97                break;
98            }
99        }
100        buf
101    }
102
103    // ── Whitespace / comment skipping ─────────────────────────────────────
104
105    fn skip_whitespace_and_comments(&mut self) {
106        loop {
107            match self.peek() {
108                // Shebang: only recognised at the very start of the file.
109                Some('#') if self.pos == 0 => {
110                    if self.peek_next() == Some('!') {
111                        // skip to end of line
112                        while let Some(ch) = self.advance() {
113                            if ch == '\n' {
114                                break;
115                            }
116                        }
117                    } else {
118                        break; // '#' is meaningful, stop skipping
119                    }
120                }
121                Some(' ') | Some('\t') | Some('\r') | Some('\n') | Some(',') => {
122                    self.advance();
123                }
124                Some(';') => {
125                    while let Some(ch) = self.advance() {
126                        if ch == '\n' {
127                            break;
128                        }
129                    }
130                }
131                _ => break,
132            }
133        }
134    }
135
136    // ── `~` (unquote / unquote-splicing) ─────────────────────────────────
137
138    fn lex_unquote(
139        &mut self,
140        start_pos: usize,
141        start_line: u32,
142        start_col: u32,
143    ) -> CljxResult<(Token, Span)> {
144        self.advance(); // consume '~'
145        if self.peek() == Some('@') {
146            self.advance();
147            Ok((
148                Token::UnquoteSplice,
149                self.span_from(start_pos, start_line, start_col),
150            ))
151        } else {
152            Ok((
153                Token::Unquote,
154                self.span_from(start_pos, start_line, start_col),
155            ))
156        }
157    }
158
159    // ── `#` dispatch ──────────────────────────────────────────────────────
160
161    fn lex_hash(
162        &mut self,
163        start_pos: usize,
164        start_line: u32,
165        start_col: u32,
166    ) -> CljxResult<(Token, Span)> {
167        self.advance(); // consume '#'
168        match self.peek() {
169            Some('(') => {
170                self.advance();
171                Ok((
172                    Token::HashFn,
173                    self.span_from(start_pos, start_line, start_col),
174                ))
175            }
176            Some('{') => {
177                self.advance();
178                Ok((
179                    Token::HashSet,
180                    self.span_from(start_pos, start_line, start_col),
181                ))
182            }
183            Some('\'') => {
184                self.advance();
185                Ok((
186                    Token::HashVar,
187                    self.span_from(start_pos, start_line, start_col),
188                ))
189            }
190            Some('_') => {
191                self.advance();
192                Ok((
193                    Token::HashDiscard,
194                    self.span_from(start_pos, start_line, start_col),
195                ))
196            }
197            Some('"') => self.lex_regex(start_pos, start_line, start_col),
198            Some('?') => {
199                self.advance(); // consume '?'
200                if self.peek() == Some('@') {
201                    self.advance();
202                    Ok((
203                        Token::ReaderCondSplice,
204                        self.span_from(start_pos, start_line, start_col),
205                    ))
206                } else {
207                    Ok((
208                        Token::ReaderCond,
209                        self.span_from(start_pos, start_line, start_col),
210                    ))
211                }
212            }
213            Some('#') => self.lex_symbolic(start_pos, start_line, start_col),
214            Some(':') => self.lex_namespaced_map(start_pos, start_line, start_col),
215            Some(c) if is_symbol_start(c) => {
216                let name = self.read_symbol_chars();
217                Ok((
218                    Token::TaggedLiteral(name),
219                    self.span_from(start_pos, start_line, start_col),
220                ))
221            }
222            other => {
223                let span = self.span_from(start_pos, start_line, start_col);
224                Err(self.make_error(format!("unknown # dispatch character: {:?}", other), span))
225            }
226        }
227    }
228
229    fn lex_regex(
230        &mut self,
231        start_pos: usize,
232        start_line: u32,
233        start_col: u32,
234    ) -> CljxResult<(Token, Span)> {
235        self.advance(); // consume opening '"'
236        let mut buf = String::new();
237        loop {
238            match self.advance() {
239                None => {
240                    let span = self.span_from(start_pos, start_line, start_col);
241                    return Err(self.make_error("unterminated regex literal", span));
242                }
243                Some('"') => break,
244                Some('\\') => {
245                    // Store escape verbatim (two chars) — no processing.
246                    buf.push('\\');
247                    match self.advance() {
248                        Some(c) => buf.push(c),
249                        None => {
250                            let span = self.span_from(start_pos, start_line, start_col);
251                            return Err(self.make_error("unterminated regex literal", span));
252                        }
253                    }
254                }
255                Some(c) => buf.push(c),
256            }
257        }
258        Ok((
259            Token::Regex(buf),
260            self.span_from(start_pos, start_line, start_col),
261        ))
262    }
263
264    fn lex_symbolic(
265        &mut self,
266        start_pos: usize,
267        start_line: u32,
268        start_col: u32,
269    ) -> CljxResult<(Token, Span)> {
270        self.advance(); // consume second '#'
271        let name = self.read_symbol_chars();
272        match name.as_str() {
273            "Inf" | "-Inf" | "NaN" => Ok((
274                Token::Symbolic(name),
275                self.span_from(start_pos, start_line, start_col),
276            )),
277            _ => {
278                let span = self.span_from(start_pos, start_line, start_col);
279                Err(self.make_error(format!("unknown symbolic value: ##{name}"), span))
280            }
281        }
282    }
283
284    // ── String literal ────────────────────────────────────────────────────
285
286    fn lex_string(
287        &mut self,
288        start_pos: usize,
289        start_line: u32,
290        start_col: u32,
291    ) -> CljxResult<(Token, Span)> {
292        self.advance(); // consume opening '"'
293        let mut buf = String::new();
294        loop {
295            match self.advance() {
296                None => {
297                    let span = self.span_from(start_pos, start_line, start_col);
298                    return Err(self.make_error("unterminated string literal", span));
299                }
300                Some('"') => break,
301                Some('\\') => match self.advance() {
302                    Some('n') => buf.push('\n'),
303                    Some('t') => buf.push('\t'),
304                    Some('r') => buf.push('\r'),
305                    Some('b') => buf.push('\x08'),
306                    Some('f') => buf.push('\x0C'),
307                    Some('\\') => buf.push('\\'),
308                    Some('"') => buf.push('"'),
309                    Some('u') => {
310                        let ch = self.read_unicode_escape(start_pos, start_line, start_col)?;
311                        buf.push(ch);
312                    }
313                    Some(c) => {
314                        let span = self.span_from(start_pos, start_line, start_col);
315                        return Err(self.make_error(format!("unknown string escape: \\{c}"), span));
316                    }
317                    None => {
318                        let span = self.span_from(start_pos, start_line, start_col);
319                        return Err(self.make_error("unterminated string literal", span));
320                    }
321                },
322                Some(c) => buf.push(c),
323            }
324        }
325        Ok((
326            Token::Str(buf),
327            self.span_from(start_pos, start_line, start_col),
328        ))
329    }
330
331    /// Read exactly 4 hex digits after `\u` and return the corresponding char.
332    fn read_unicode_escape(
333        &mut self,
334        start_pos: usize,
335        start_line: u32,
336        start_col: u32,
337    ) -> CljxResult<char> {
338        let mut hex = String::with_capacity(4);
339        for _ in 0..4 {
340            match self.advance() {
341                Some(c) if c.is_ascii_hexdigit() => hex.push(c),
342                Some(c) => {
343                    let span = self.span_from(start_pos, start_line, start_col);
344                    return Err(self.make_error(
345                        format!("invalid \\u escape: expected hex digit, got {c:?}"),
346                        span,
347                    ));
348                }
349                None => {
350                    let span = self.span_from(start_pos, start_line, start_col);
351                    return Err(self.make_error("unterminated \\u escape", span));
352                }
353            }
354        }
355        let code = u32::from_str_radix(&hex, 16).unwrap();
356        char::from_u32(code).ok_or_else(|| {
357            let span = self.span_from(start_pos, start_line, start_col);
358            self.make_error(format!("invalid unicode code point: \\u{hex}"), span)
359        })
360    }
361
362    // ── Character literal `\X` ────────────────────────────────────────────
363
364    fn lex_char_literal(
365        &mut self,
366        start_pos: usize,
367        start_line: u32,
368        start_col: u32,
369    ) -> CljxResult<(Token, Span)> {
370        self.advance(); // consume '\'
371
372        // Peek ahead at all symbol-constituent chars to figure out the name.
373        let rest_start = self.pos;
374        let rest: String = self.source[rest_start..]
375            .chars()
376            .take_while(|&c| c.is_alphanumeric() || c == '-')
377            .collect();
378
379        let ch = match rest.as_str() {
380            "newline" => {
381                self.pos += "newline".len();
382                self.col += "newline".len() as u32;
383                '\n'
384            }
385            "space" => {
386                self.pos += "space".len();
387                self.col += "space".len() as u32;
388                ' '
389            }
390            "tab" => {
391                self.pos += "tab".len();
392                self.col += "tab".len() as u32;
393                '\t'
394            }
395            "backspace" => {
396                self.pos += "backspace".len();
397                self.col += "backspace".len() as u32;
398                '\x08'
399            }
400            "formfeed" => {
401                self.pos += "formfeed".len();
402                self.col += "formfeed".len() as u32;
403                '\x0C'
404            }
405            "return" => {
406                self.pos += "return".len();
407                self.col += "return".len() as u32;
408                '\r'
409            }
410            _ if rest.starts_with('u') && rest.len() >= 5 => {
411                // Try \uXXXX
412                let hex_part = &rest[1..5];
413                if hex_part.chars().all(|c| c.is_ascii_hexdigit()) {
414                    let code = u32::from_str_radix(hex_part, 16).unwrap();
415                    let c = char::from_u32(code).ok_or_else(|| {
416                        let span = self.span_from(start_pos, start_line, start_col);
417                        self.make_error(
418                            format!("invalid unicode code point in char literal: \\u{hex_part}"),
419                            span,
420                        )
421                    })?;
422                    // advance 5 bytes: 'u' + 4 hex digits
423                    self.pos += 5;
424                    self.col += 5;
425                    c
426                } else {
427                    let span = self.span_from(start_pos, start_line, start_col);
428                    return Err(self.make_error(format!("unknown character name: {rest}"), span));
429                }
430            }
431            _ if rest.len() == 1 => {
432                // Single ASCII or first char
433                let c = self.source[rest_start..].chars().next().unwrap();
434                self.pos += c.len_utf8();
435                self.col += c.len_utf8() as u32;
436                c
437            }
438            _ if rest.is_empty() => {
439                // Nothing after backslash — try a single non-alphanumeric char
440                match self.source[rest_start..].chars().next() {
441                    Some(c) => {
442                        self.pos += c.len_utf8();
443                        self.col += c.len_utf8() as u32;
444                        c
445                    }
446                    None => {
447                        let span = self.span_from(start_pos, start_line, start_col);
448                        return Err(self.make_error("unexpected end of file after \\", span));
449                    }
450                }
451            }
452            _ => {
453                let span = self.span_from(start_pos, start_line, start_col);
454                return Err(self.make_error(format!("unknown character name: {rest}"), span));
455            }
456        };
457
458        Ok((
459            Token::Char(ch),
460            self.span_from(start_pos, start_line, start_col),
461        ))
462    }
463
464    /// Lex the `#:ns` / `#::` / `#::alias` prefix of a namespaced map. `#` is
465    /// already consumed and the next char is `:`. The `{` is left in the stream
466    /// so the parser reads the body through the ordinary map path.
467    fn lex_namespaced_map(
468        &mut self,
469        start_pos: usize,
470        start_line: u32,
471        start_col: u32,
472    ) -> CljxResult<(Token, Span)> {
473        self.advance(); // consume first ':'
474        let auto = self.peek() == Some(':');
475        if auto {
476            self.advance(); // consume second ':'
477        }
478        let text = self.read_symbol_chars();
479        let ns = crate::namespaced_map::MapNs::parse(&text, auto).map_err(|msg| {
480            let span = self.span_from(start_pos, start_line, start_col);
481            self.make_error(msg, span)
482        })?;
483        // The JVM reader skips whitespace between the prefix and the map, but
484        // not comments: `#:foo {…}` and `#:foo, {…}` are the same literal.
485        self.skip_whitespace_only();
486        if self.peek() != Some('{') {
487            let span = self.span_from(start_pos, start_line, start_col);
488            return Err(self.make_error("namespaced map literal must be followed by a map", span));
489        }
490        Ok((
491            Token::NamespacedMap(ns),
492            self.span_from(start_pos, start_line, start_col),
493        ))
494    }
495
496    /// Skip spaces, tabs, newlines and commas - and nothing else.
497    fn skip_whitespace_only(&mut self) {
498        while let Some(ch) = self.peek() {
499            if ch.is_whitespace() || ch == ',' {
500                self.advance();
501            } else {
502                break;
503            }
504        }
505    }
506
507    // ── Keyword ───────────────────────────────────────────────────────────
508
509    fn lex_keyword(
510        &mut self,
511        start_pos: usize,
512        start_line: u32,
513        start_col: u32,
514    ) -> CljxResult<(Token, Span)> {
515        self.advance(); // consume first ':'
516        if self.peek() == Some(':') {
517            self.advance(); // consume second ':'
518            let name = self.read_symbol_chars();
519            if name.is_empty() {
520                let span = self.span_from(start_pos, start_line, start_col);
521                return Err(self.make_error("empty auto-resolved keyword", span));
522            }
523            Ok((
524                Token::AutoKeyword(name),
525                self.span_from(start_pos, start_line, start_col),
526            ))
527        } else {
528            let name = self.read_symbol_chars();
529            if name.is_empty() {
530                let span = self.span_from(start_pos, start_line, start_col);
531                return Err(self.make_error("empty keyword", span));
532            }
533            Ok((
534                Token::Keyword(name),
535                self.span_from(start_pos, start_line, start_col),
536            ))
537        }
538    }
539
540    // ── Symbol (and nil/true/false) ────────────────────────────────────────
541
542    fn lex_symbol(
543        &mut self,
544        start_pos: usize,
545        start_line: u32,
546        start_col: u32,
547    ) -> CljxResult<(Token, Span)> {
548        let mut name = self.read_symbol_chars();
549
550        // Peek for a version suffix: `@<commit-hash>`.  We only consume the `@`
551        // when it is immediately followed by 7–40 hex characters so that a
552        // standalone `@expr` (deref reader macro) is never affected — deref
553        // always starts a *new* form where `@` is the first character, not a
554        // mid-symbol suffix.
555        if self.peek() == Some('@') {
556            let version_candidate = self.peek_version_hash();
557            if let Some(hash) = version_candidate {
558                self.advance(); // consume '@'
559                for _ in 0..hash.len() {
560                    self.advance();
561                }
562                name.push('@');
563                name.push_str(&hash);
564            }
565        }
566
567        let tok = match name.as_str() {
568            "nil" => Token::Nil,
569            "true" => Token::Bool(true),
570            "false" => Token::Bool(false),
571            _ => Token::Symbol(name),
572        };
573        Ok((tok, self.span_from(start_pos, start_line, start_col)))
574    }
575
576    /// Look ahead past the `@` that `peek()` just returned and collect
577    /// characters as long as they are ASCII hex digits, up to 40.  Returns
578    /// `Some(hash)` if the candidate is 7–40 hex chars followed by a
579    /// non-hex-digit (or EOF), `None` otherwise.  Does **not** advance the
580    /// cursor.
581    fn peek_version_hash(&self) -> Option<String> {
582        // Start one byte past the current `@`.
583        let at_byte = self.pos + 1; // '@' is single-byte ASCII
584        let rest = &self.source[at_byte..];
585        let hash: String = rest
586            .chars()
587            .take(40)
588            .take_while(|c| c.is_ascii_hexdigit())
589            .collect();
590        if hash.len() >= 7 {
591            // Make sure the character after the hash is a delimiter (or EOF).
592            let after = rest[hash.len()..].chars().next();
593            let is_delimited = after.is_none_or(|c| !c.is_ascii_hexdigit());
594            if is_delimited {
595                return Some(hash);
596            }
597        }
598        None
599    }
600
601    // ── Number ────────────────────────────────────────────────────────────
602
603    fn lex_number(
604        &mut self,
605        start_pos: usize,
606        start_line: u32,
607        start_col: u32,
608    ) -> CljxResult<(Token, Span)> {
609        // Optional sign
610        let negative = match self.peek() {
611            Some('-') => {
612                self.advance();
613                true
614            }
615            Some('+') => {
616                self.advance();
617                false
618            }
619            _ => false,
620        };
621        let sign_str = if negative { "-" } else { "" };
622
623        // Integer part (decimal digits)
624        let mut int_part = String::new();
625        while let Some(c) = self.peek() {
626            if c.is_ascii_digit() {
627                int_part.push(c);
628                self.advance();
629            } else {
630                break;
631            }
632        }
633
634        // Hex literal: 0x / 0X  (also -0x…)
635        if int_part == "0" && matches!(self.peek(), Some('x') | Some('X')) {
636            self.advance(); // consume 'x'/'X'
637            let mut hex = String::new();
638            while let Some(c) = self.peek() {
639                if c.is_ascii_hexdigit() {
640                    hex.push(c);
641                    self.advance();
642                } else {
643                    break;
644                }
645            }
646            if hex.is_empty() {
647                let span = self.span_from(start_pos, start_line, start_col);
648                return Err(self.make_error("expected hex digits after 0x", span));
649            }
650            let value = u128::from_str_radix(&hex, 16).unwrap_or(u128::MAX);
651            let span = self.span_from(start_pos, start_line, start_col);
652            return if negative {
653                // -0x8000000000000000 == i64::MIN is valid; anything larger overflows.
654                if value <= (i64::MAX as u128) + 1 {
655                    Ok((Token::Int(0i64.wrapping_sub(value as i64)), span))
656                } else {
657                    // Store as signed decimal string for BigInt.
658                    Ok((Token::BigInt(format!("-{value}")), span))
659                }
660            } else if value <= i64::MAX as u128 {
661                Ok((Token::Int(value as i64), span))
662            } else {
663                Ok((Token::BigInt(value.to_string()), span))
664            };
665        }
666
667        // Radix literal: NNrDIGITS
668        if matches!(self.peek(), Some('r') | Some('R')) {
669            let radix: u32 = int_part.parse().unwrap_or(0);
670            self.advance(); // consume 'r'/'R'
671            let mut digits = String::new();
672            while let Some(c) = self.peek() {
673                if c.is_ascii_alphanumeric() {
674                    digits.push(c);
675                    self.advance();
676                } else {
677                    break;
678                }
679            }
680            let mut value: u128 = 0;
681            for c in digits.chars() {
682                let d = c.to_digit(radix).ok_or_else(|| {
683                    let span = self.span_from(start_pos, start_line, start_col);
684                    self.make_error(format!("invalid digit {c:?} for radix {radix}"), span)
685                })?;
686                value = value.wrapping_mul(radix as u128).wrapping_add(d as u128);
687            }
688            if negative {
689                // Check if it fits as negative i64
690                if value <= (i64::MAX as u128) + 1 {
691                    let signed = -(value as i64);
692                    return Ok((
693                        Token::Int(signed),
694                        self.span_from(start_pos, start_line, start_col),
695                    ));
696                } else {
697                    // Store as decimal string with sign
698                    return Ok((
699                        Token::BigInt(format!("-{value}")),
700                        self.span_from(start_pos, start_line, start_col),
701                    ));
702                }
703            } else if value <= i64::MAX as u128 {
704                return Ok((
705                    Token::Int(value as i64),
706                    self.span_from(start_pos, start_line, start_col),
707                ));
708            } else {
709                return Ok((
710                    Token::BigInt(value.to_string()),
711                    self.span_from(start_pos, start_line, start_col),
712                ));
713            }
714        }
715
716        // BigInt suffix 'N'
717        if self.peek() == Some('N') {
718            self.advance();
719            return Ok((
720                Token::BigInt(format!("{sign_str}{int_part}")),
721                self.span_from(start_pos, start_line, start_col),
722            ));
723        }
724
725        // BigDecimal suffix 'M' on integer literal (e.g. 4M)
726        if self.peek() == Some('M') {
727            self.advance();
728            return Ok((
729                Token::BigDecimal(format!("{sign_str}{int_part}")),
730                self.span_from(start_pos, start_line, start_col),
731            ));
732        }
733
734        // Float: decimal point or exponent
735        if matches!(self.peek(), Some('.') | Some('e') | Some('E')) {
736            let mut raw = format!("{sign_str}{int_part}");
737            if self.peek() == Some('.') {
738                raw.push('.');
739                self.advance();
740                while let Some(c) = self.peek() {
741                    if c.is_ascii_digit() {
742                        raw.push(c);
743                        self.advance();
744                    } else {
745                        break;
746                    }
747                }
748            }
749            if matches!(self.peek(), Some('e') | Some('E')) {
750                raw.push('e');
751                self.advance();
752                if matches!(self.peek(), Some('+') | Some('-')) {
753                    raw.push(self.peek().unwrap());
754                    self.advance();
755                }
756                while let Some(c) = self.peek() {
757                    if c.is_ascii_digit() {
758                        raw.push(c);
759                        self.advance();
760                    } else {
761                        break;
762                    }
763                }
764            }
765            // BigDecimal suffix 'M'
766            if self.peek() == Some('M') {
767                self.advance();
768                return Ok((
769                    Token::BigDecimal(raw),
770                    self.span_from(start_pos, start_line, start_col),
771                ));
772            }
773            let val: f64 = raw.parse().map_err(|_| {
774                let span = self.span_from(start_pos, start_line, start_col);
775                self.make_error(format!("invalid float: {raw}"), span)
776            })?;
777            return Ok((
778                Token::Float(val),
779                self.span_from(start_pos, start_line, start_col),
780            ));
781        }
782
783        // Ratio: INT/DIGITS — only if next char after '/' is a digit
784        if self.peek() == Some('/') && matches!(self.peek_next(), Some(c) if c.is_ascii_digit()) {
785            self.advance(); // consume '/'
786            let mut denom = String::new();
787            while let Some(c) = self.peek() {
788                if c.is_ascii_digit() {
789                    denom.push(c);
790                    self.advance();
791                } else {
792                    break;
793                }
794            }
795            return Ok((
796                Token::Ratio(format!("{sign_str}{int_part}/{denom}")),
797                self.span_from(start_pos, start_line, start_col),
798            ));
799        }
800
801        // Plain integer
802        let full = format!("{sign_str}{int_part}");
803        match full.parse::<i64>() {
804            Ok(n) => Ok((
805                Token::Int(n),
806                self.span_from(start_pos, start_line, start_col),
807            )),
808            Err(_) => {
809                // Overflow: store decimal string
810                Ok((
811                    Token::BigInt(full),
812                    self.span_from(start_pos, start_line, start_col),
813                ))
814            }
815        }
816    }
817
818    // ── Top-level token dispatch ──────────────────────────────────────────
819
820    pub fn next_token(&mut self) -> CljxResult<(Token, Span)> {
821        self.skip_whitespace_and_comments();
822
823        let start_pos = self.pos;
824        let start_line = self.line;
825        let start_col = self.col;
826
827        let ch = match self.peek() {
828            None => {
829                return Ok((Token::Eof, self.span_from(start_pos, start_line, start_col)));
830            }
831            Some(c) => c,
832        };
833
834        match ch {
835            '(' => {
836                self.advance();
837                Ok((
838                    Token::LParen,
839                    self.span_from(start_pos, start_line, start_col),
840                ))
841            }
842            ')' => {
843                self.advance();
844                Ok((
845                    Token::RParen,
846                    self.span_from(start_pos, start_line, start_col),
847                ))
848            }
849            '[' => {
850                self.advance();
851                Ok((
852                    Token::LBracket,
853                    self.span_from(start_pos, start_line, start_col),
854                ))
855            }
856            ']' => {
857                self.advance();
858                Ok((
859                    Token::RBracket,
860                    self.span_from(start_pos, start_line, start_col),
861                ))
862            }
863            '{' => {
864                self.advance();
865                Ok((
866                    Token::LBrace,
867                    self.span_from(start_pos, start_line, start_col),
868                ))
869            }
870            '}' => {
871                self.advance();
872                Ok((
873                    Token::RBrace,
874                    self.span_from(start_pos, start_line, start_col),
875                ))
876            }
877            '\'' => {
878                self.advance();
879                Ok((
880                    Token::Quote,
881                    self.span_from(start_pos, start_line, start_col),
882                ))
883            }
884            '`' => {
885                self.advance();
886                Ok((
887                    Token::SyntaxQuote,
888                    self.span_from(start_pos, start_line, start_col),
889                ))
890            }
891            '@' => {
892                self.advance();
893                Ok((
894                    Token::Deref,
895                    self.span_from(start_pos, start_line, start_col),
896                ))
897            }
898            '^' => {
899                self.advance();
900                Ok((
901                    Token::Meta,
902                    self.span_from(start_pos, start_line, start_col),
903                ))
904            }
905            '~' => self.lex_unquote(start_pos, start_line, start_col),
906            '#' => self.lex_hash(start_pos, start_line, start_col),
907            '"' => self.lex_string(start_pos, start_line, start_col),
908            '\\' => self.lex_char_literal(start_pos, start_line, start_col),
909            ':' => self.lex_keyword(start_pos, start_line, start_col),
910            c if c.is_ascii_digit() => self.lex_number(start_pos, start_line, start_col),
911            '+' | '-' if matches!(self.peek_next(), Some(d) if d.is_ascii_digit()) => {
912                self.lex_number(start_pos, start_line, start_col)
913            }
914            c if is_symbol_start(c) => self.lex_symbol(start_pos, start_line, start_col),
915            // '+' and '-' alone (or before non-digit) are symbols
916            '+' | '-' => self.lex_symbol(start_pos, start_line, start_col),
917            c => {
918                self.advance();
919                let span = self.span_from(start_pos, start_line, start_col);
920                Err(self.make_error(format!("unexpected character: {c:?}"), span))
921            }
922        }
923    }
924}
925
926impl Iterator for Lexer {
927    type Item = CljxResult<(Token, Span)>;
928
929    fn next(&mut self) -> Option<Self::Item> {
930        match self.next_token() {
931            Ok((Token::Eof, _)) => None,
932            result => Some(result),
933        }
934    }
935}
936
937// ─── Tests ────────────────────────────────────────────────────────────────────
938
939#[cfg(test)]
940mod tests {
941    use super::*;
942
943    fn lex_all(src: &str) -> Vec<Token> {
944        Lexer::new(src.to_string(), "<test>".to_string())
945            .map(|r: CljxResult<(Token, Span)>| r.expect("lex error").0)
946            .collect()
947    }
948
949    fn lex_one(src: &str) -> Token {
950        let mut l = Lexer::new(src.to_string(), "<test>".to_string());
951        l.next_token().expect("lex error").0
952    }
953
954    fn lex_err(src: &str) -> String {
955        let mut l = Lexer::new(src.to_string(), "<test>".to_string());
956        loop {
957            match l.next_token() {
958                Err(CljxError::ReadError { message, .. }) => return message,
959                Err(e) => panic!("unexpected error type: {e}"),
960                Ok((Token::Eof, _)) => panic!("expected an error but got Eof"),
961                Ok(_) => {}
962            }
963        }
964    }
965
966    // ── nil / bool ────────────────────────────────────────────────────────
967
968    #[test]
969    fn test_nil() {
970        assert_eq!(lex_one("nil"), Token::Nil);
971    }
972
973    #[test]
974    fn test_bool() {
975        assert_eq!(lex_one("true"), Token::Bool(true));
976        assert_eq!(lex_one("false"), Token::Bool(false));
977    }
978
979    // ── Integers ──────────────────────────────────────────────────────────
980
981    #[test]
982    fn test_int_plain() {
983        assert_eq!(lex_one("42"), Token::Int(42));
984        assert_eq!(lex_one("-42"), Token::Int(-42));
985        assert_eq!(lex_one("+42"), Token::Int(42));
986        assert_eq!(lex_one("0"), Token::Int(0));
987    }
988
989    #[test]
990    fn test_bigint_suffix() {
991        assert_eq!(lex_one("42N"), Token::BigInt("42".to_string()));
992        assert_eq!(lex_one("-42N"), Token::BigInt("-42".to_string()));
993    }
994
995    #[test]
996    fn test_hex_literal() {
997        assert_eq!(lex_one("0xff"), Token::Int(255));
998        assert_eq!(lex_one("0xFF"), Token::Int(255));
999        assert_eq!(lex_one("0x0"), Token::Int(0));
1000        assert_eq!(lex_one("0x7FFFFFFFFFFFFFFF"), Token::Int(i64::MAX));
1001        assert_eq!(lex_one("-0x8000000000000000"), Token::Int(i64::MIN));
1002        assert_eq!(lex_one("-0xff"), Token::Int(-255));
1003        // Overflow → BigInt
1004        match lex_one("0xFFFFFFFFFFFFFFFF") {
1005            Token::BigInt(_) => {}
1006            other => panic!("expected BigInt for 0xFFFF…, got {other:?}"),
1007        }
1008    }
1009
1010    #[test]
1011    fn test_radix() {
1012        assert_eq!(lex_one("2r1010"), Token::Int(10));
1013        assert_eq!(lex_one("8r77"), Token::Int(63));
1014        assert_eq!(lex_one("16rFF"), Token::Int(255));
1015        assert_eq!(lex_one("16rff"), Token::Int(255));
1016        assert_eq!(lex_one("36rZ"), Token::Int(35));
1017    }
1018
1019    #[test]
1020    fn test_radix_overflow() {
1021        // 2^64 fits in u128 but not i64
1022        let tok = lex_one("10r18446744073709551616");
1023        match tok {
1024            Token::BigInt(_) => {}
1025            other => panic!("expected BigInt, got {other:?}"),
1026        }
1027    }
1028
1029    // ── Floats ────────────────────────────────────────────────────────────
1030
1031    #[test]
1032    #[allow(clippy::approx_constant)]
1033    fn test_floats() {
1034        assert_eq!(lex_one("3.14"), Token::Float(3.14));
1035        assert_eq!(lex_one("1e10"), Token::Float(1e10));
1036        assert_eq!(lex_one("1.5e-3"), Token::Float(1.5e-3));
1037        assert_eq!(lex_one("-0.5"), Token::Float(-0.5));
1038    }
1039
1040    #[test]
1041    fn test_bigdecimal() {
1042        assert_eq!(lex_one("3.14M"), Token::BigDecimal("3.14".to_string()));
1043        assert_eq!(lex_one("1e5M"), Token::BigDecimal("1e5".to_string()));
1044    }
1045
1046    // ── Ratio ────────────────────────────────────────────────────────────
1047
1048    #[test]
1049    fn test_ratio() {
1050        assert_eq!(lex_one("3/4"), Token::Ratio("3/4".to_string()));
1051        assert_eq!(lex_one("-1/2"), Token::Ratio("-1/2".to_string()));
1052    }
1053
1054    #[test]
1055    fn test_ratio_vs_symbol() {
1056        // "3/foo" should lex as Int(3) then Symbol("/foo") — not a ratio
1057        let toks = lex_all("3/foo");
1058        assert_eq!(toks[0], Token::Int(3));
1059        assert_eq!(toks[1], Token::Symbol("/foo".to_string()));
1060    }
1061
1062    // ── Char literals ────────────────────────────────────────────────────
1063
1064    #[test]
1065    fn test_char_simple() {
1066        assert_eq!(lex_one("\\a"), Token::Char('a'));
1067    }
1068
1069    #[test]
1070    fn test_char_named() {
1071        assert_eq!(lex_one("\\newline"), Token::Char('\n'));
1072        assert_eq!(lex_one("\\space"), Token::Char(' '));
1073        assert_eq!(lex_one("\\tab"), Token::Char('\t'));
1074        assert_eq!(lex_one("\\backspace"), Token::Char('\x08'));
1075        assert_eq!(lex_one("\\formfeed"), Token::Char('\x0C'));
1076        assert_eq!(lex_one("\\return"), Token::Char('\r'));
1077    }
1078
1079    #[test]
1080    fn test_char_unicode() {
1081        assert_eq!(lex_one("\\u0041"), Token::Char('A'));
1082        assert_eq!(lex_one("\\u00e9"), Token::Char('é'));
1083    }
1084
1085    // ── Strings ──────────────────────────────────────────────────────────
1086
1087    #[test]
1088    fn test_string_basic() {
1089        assert_eq!(lex_one("\"hello\""), Token::Str("hello".to_string()));
1090    }
1091
1092    #[test]
1093    fn test_string_escapes() {
1094        assert_eq!(
1095            lex_one(r#""\n\t\r\b\f\\\"" "#),
1096            Token::Str("\n\t\r\x08\x0C\\\"".to_string())
1097        );
1098    }
1099
1100    #[test]
1101    fn test_string_unicode_escape() {
1102        assert_eq!(lex_one("\"\\u0041\""), Token::Str("A".to_string()));
1103    }
1104
1105    // ── Symbols ──────────────────────────────────────────────────────────
1106
1107    #[test]
1108    fn test_symbols() {
1109        assert_eq!(lex_one("foo"), Token::Symbol("foo".to_string()));
1110        assert_eq!(lex_one("ns/name"), Token::Symbol("ns/name".to_string()));
1111        assert_eq!(lex_one("/"), Token::Symbol("/".to_string()));
1112        assert_eq!(lex_one(".."), Token::Symbol("..".to_string()));
1113        assert_eq!(lex_one(".method"), Token::Symbol(".method".to_string()));
1114        assert_eq!(lex_one("+"), Token::Symbol("+".to_string()));
1115        assert_eq!(lex_one("-"), Token::Symbol("-".to_string()));
1116        assert_eq!(lex_one("+foo"), Token::Symbol("+foo".to_string()));
1117    }
1118
1119    #[test]
1120    fn test_auto_gensym_symbol() {
1121        // `x#` is a single symbol token (trailing `#` is auto-gensym syntax,
1122        // not a `#`-dispatch macro — `#` is non-terminating mid-token).
1123        assert_eq!(lex_one("x#"), Token::Symbol("x#".to_string()));
1124        let toks = lex_all("`(let [x# 1] x#)");
1125        assert_eq!(
1126            toks,
1127            vec![
1128                Token::SyntaxQuote,
1129                Token::LParen,
1130                Token::Symbol("let".to_string()),
1131                Token::LBracket,
1132                Token::Symbol("x#".to_string()),
1133                Token::Int(1),
1134                Token::RBracket,
1135                Token::Symbol("x#".to_string()),
1136                Token::RParen,
1137            ]
1138        );
1139    }
1140
1141    // ── Keywords ─────────────────────────────────────────────────────────
1142
1143    #[test]
1144    fn test_keyword() {
1145        assert_eq!(lex_one(":foo"), Token::Keyword("foo".to_string()));
1146        assert_eq!(lex_one(":ns/name"), Token::Keyword("ns/name".to_string()));
1147    }
1148
1149    #[test]
1150    fn test_keyword_with_embedded_colon() {
1151        // An embedded `:` is a literal name character, not a sub-token
1152        // boundary — `:xlink:href` is one keyword, not two.
1153        assert_eq!(
1154            lex_one(":xlink:href"),
1155            Token::Keyword("xlink:href".to_string())
1156        );
1157        let toks = lex_all("[:xlink:href]");
1158        assert_eq!(
1159            toks,
1160            vec![
1161                Token::LBracket,
1162                Token::Keyword("xlink:href".to_string()),
1163                Token::RBracket,
1164            ]
1165        );
1166    }
1167
1168    #[test]
1169    fn test_auto_keyword() {
1170        assert_eq!(lex_one("::foo"), Token::AutoKeyword("foo".to_string()));
1171        assert_eq!(
1172            lex_one("::ns/alias"),
1173            Token::AutoKeyword("ns/alias".to_string())
1174        );
1175    }
1176
1177    #[test]
1178    fn test_namespaced_map_prefix() {
1179        use crate::namespaced_map::MapNs;
1180        // The `{` is deliberately NOT consumed - the parser reads the body
1181        // through the ordinary map path.
1182        assert_eq!(
1183            lex_one("#:adt{:a 1}"),
1184            Token::NamespacedMap(MapNs::Literal("adt".to_string()))
1185        );
1186        assert_eq!(lex_one("#::{:a 1}"), Token::NamespacedMap(MapNs::CurrentNs));
1187        assert_eq!(
1188            lex_one("#::al{:a 1}"),
1189            Token::NamespacedMap(MapNs::Alias("al".to_string()))
1190        );
1191    }
1192
1193    #[test]
1194    fn test_namespaced_map_prefix_skips_whitespace_before_the_brace() {
1195        use crate::namespaced_map::MapNs;
1196        // The JVM accepts whitespace and commas between the prefix and `{`.
1197        for src in ["#:adt {:a 1}", "#:adt, {:a 1}", "#:adt\n{:a 1}"] {
1198            assert_eq!(
1199                lex_one(src),
1200                Token::NamespacedMap(MapNs::Literal("adt".to_string())),
1201                "{src}"
1202            );
1203        }
1204        assert_eq!(
1205            lex_one("#:: {:a 1}"),
1206            Token::NamespacedMap(MapNs::CurrentNs)
1207        );
1208        assert_eq!(
1209            lex_one("#::al {:a 1}"),
1210            Token::NamespacedMap(MapNs::Alias("al".to_string()))
1211        );
1212    }
1213
1214    #[test]
1215    fn test_namespaced_map_prefix_does_not_skip_comments() {
1216        // Whitespace only - a comment between the prefix and `{` is not a map.
1217        assert!(lex_err("#:adt ;; here\n{:a 1}").contains("must be followed by a map"));
1218    }
1219
1220    #[test]
1221    fn test_namespaced_map_prefix_errors() {
1222        assert!(lex_err("#:{:a 1}").contains("requires a namespace"));
1223        assert!(lex_err("#:adt [1]").contains("must be followed by a map"));
1224    }
1225
1226    #[test]
1227    fn test_namespaced_map_prefix_must_be_an_unqualified_symbol() {
1228        for src in [
1229            "#:foo/bar{:a 1}",
1230            "#:1{:a 1}",
1231            "#:nil{:a 1}",
1232            "#::foo/bar{:a 1}",
1233        ] {
1234            let err = lex_err(src);
1235            assert!(err.contains("unqualified symbol"), "{src}: {err}");
1236        }
1237    }
1238
1239    // ── Delimiters ───────────────────────────────────────────────────────
1240
1241    #[test]
1242    fn test_delimiters() {
1243        assert_eq!(
1244            lex_all("([{}])"),
1245            vec![
1246                Token::LParen,
1247                Token::LBracket,
1248                Token::LBrace,
1249                Token::RBrace,
1250                Token::RBracket,
1251                Token::RParen,
1252            ]
1253        );
1254    }
1255
1256    // ── Reader macros ────────────────────────────────────────────────────
1257
1258    #[test]
1259    fn test_reader_macros() {
1260        assert_eq!(lex_one("'x"), Token::Quote);
1261        assert_eq!(lex_one("`x"), Token::SyntaxQuote);
1262        assert_eq!(lex_one("~x"), Token::Unquote);
1263        assert_eq!(lex_one("~@x"), Token::UnquoteSplice);
1264        assert_eq!(lex_one("@x"), Token::Deref);
1265        assert_eq!(lex_one("^x"), Token::Meta);
1266    }
1267
1268    // ── `#` dispatch ─────────────────────────────────────────────────────
1269
1270    #[test]
1271    fn test_hash_dispatch() {
1272        assert_eq!(lex_one("#("), Token::HashFn);
1273        assert_eq!(lex_one("#{"), Token::HashSet);
1274        assert_eq!(lex_one("#'"), Token::HashVar);
1275        assert_eq!(lex_one("#_"), Token::HashDiscard);
1276        assert_eq!(lex_one("#?"), Token::ReaderCond);
1277        assert_eq!(lex_one("#?@"), Token::ReaderCondSplice);
1278    }
1279
1280    #[test]
1281    fn test_regex() {
1282        assert_eq!(lex_one("#\"[a-z]+\""), Token::Regex("[a-z]+".to_string()));
1283    }
1284
1285    #[test]
1286    fn test_symbolic() {
1287        assert_eq!(lex_one("##Inf"), Token::Symbolic("Inf".to_string()));
1288        assert_eq!(lex_one("##-Inf"), Token::Symbolic("-Inf".to_string()));
1289        assert_eq!(lex_one("##NaN"), Token::Symbolic("NaN".to_string()));
1290    }
1291
1292    #[test]
1293    fn test_tagged_literal() {
1294        assert_eq!(lex_one("#mytag"), Token::TaggedLiteral("mytag".to_string()));
1295    }
1296
1297    // ── Multi-token ──────────────────────────────────────────────────────
1298
1299    #[test]
1300    fn test_multi_token() {
1301        let toks = lex_all("(+ 1 2)");
1302        assert_eq!(
1303            toks,
1304            vec![
1305                Token::LParen,
1306                Token::Symbol("+".to_string()),
1307                Token::Int(1),
1308                Token::Int(2),
1309                Token::RParen,
1310            ]
1311        );
1312    }
1313
1314    // ── Whitespace / comments ────────────────────────────────────────────
1315
1316    #[test]
1317    fn test_comma_skipped() {
1318        assert_eq!(lex_all("{,,,}"), vec![Token::LBrace, Token::RBrace]);
1319    }
1320
1321    #[test]
1322    fn test_comment_skipped() {
1323        assert_eq!(lex_all("; this is a comment\n42"), vec![Token::Int(42)]);
1324    }
1325
1326    #[test]
1327    fn test_shebang_skipped() {
1328        assert_eq!(lex_all("#!/usr/bin/env cljx\n42"), vec![Token::Int(42)]);
1329    }
1330
1331    // ── Span tracking ────────────────────────────────────────────────────
1332
1333    #[test]
1334    fn test_span_col() {
1335        let mut l = Lexer::new("  foo".to_string(), "<test>".to_string());
1336        let (_tok, span) = l.next_token().unwrap();
1337        assert_eq!(span.start, 2);
1338        assert_eq!(span.col, 3);
1339    }
1340
1341    #[test]
1342    fn test_span_newline() {
1343        let mut l = Lexer::new("a\nb".to_string(), "<test>".to_string());
1344        l.next_token().unwrap(); // consume 'a'
1345        let (_tok, span) = l.next_token().unwrap(); // 'b'
1346        assert_eq!(span.line, 2);
1347        assert_eq!(span.col, 1);
1348    }
1349
1350    // ── Errors ───────────────────────────────────────────────────────────
1351
1352    #[test]
1353    fn test_error_unterminated_string() {
1354        let msg = lex_err("\"unterminated");
1355        assert!(msg.contains("unterminated string"));
1356    }
1357
1358    #[test]
1359    fn test_error_bad_hash_dispatch() {
1360        // '#1' is invalid: '1' is not a symbol start and not a special dispatch char
1361        let msg = lex_err("#1");
1362        assert!(msg.contains("unknown # dispatch"));
1363    }
1364
1365    #[test]
1366    fn test_error_bad_unicode_escape_in_string() {
1367        let msg = lex_err("\"\\uGHIJ\"");
1368        assert!(msg.contains("invalid") || msg.contains("hex"));
1369    }
1370
1371    #[test]
1372    fn test_error_unknown_char_name() {
1373        let msg = lex_err("\\bogus");
1374        assert!(msg.contains("unknown character name"));
1375    }
1376
1377    #[test]
1378    fn test_error_unknown_symbolic() {
1379        let msg = lex_err("##Bogus");
1380        assert!(msg.contains("unknown symbolic value"));
1381    }
1382
1383    #[test]
1384    fn test_error_bad_string_escape() {
1385        let msg = lex_err("\"\\q\"");
1386        assert!(msg.contains("unknown string escape"));
1387    }
1388}
cljrs_reader/lexer.rs

cljrs_reader/
lexer.rs
