cljrs_reader/
lexer.rs

1// CljxError embeds NamedSource<String> for miette diagnostics, which is
2// unavoidably large. Suppress the false-positive for every returning function.
3#![allow(clippy::result_large_err)]
4
5use std::sync::Arc;
6
7use miette::NamedSource;
8
9use cljrs_types::error::{CljxError, CljxResult};
10use cljrs_types::span::Span;
11
12use crate::chars::{is_symbol_char, is_symbol_start};
13use crate::token::Token;
14
15// ─── Lexer ───────────────────────────────────────────────────────────────────
16
17pub struct Lexer {
18    source: Arc<String>,
19    file: Arc<String>,
20    pos: usize, // byte offset, always on a char boundary
21    line: u32,  // 1-based
22    col: u32,   // 1-based byte offset from line start
23}
24
25impl Lexer {
26    pub fn new(source: String, file: String) -> Self {
27        Self {
28            source: Arc::new(source),
29            file: Arc::new(file),
30            pos: 0,
31            line: 1,
32            col: 1,
33        }
34    }
35
36    // ── Public getters ────────────────────────────────────────────────────
37
38    pub fn source(&self) -> &Arc<String> {
39        &self.source
40    }
41
42    pub fn file(&self) -> &Arc<String> {
43        &self.file
44    }
45
46    // ── Low-level helpers ─────────────────────────────────────────────────
47
48    fn peek(&self) -> Option<char> {
49        self.source[self.pos..].chars().next()
50    }
51
52    fn peek_next(&self) -> Option<char> {
53        let mut chars = self.source[self.pos..].chars();
54        chars.next(); // skip current
55        chars.next()
56    }
57
58    fn advance(&mut self) -> Option<char> {
59        let ch = self.peek()?;
60        self.pos += ch.len_utf8();
61        if ch == '\n' {
62            self.line += 1;
63            self.col = 1;
64        } else {
65            self.col += ch.len_utf8() as u32;
66        }
67        Some(ch)
68    }
69
70    fn span_from(&self, start_pos: usize, start_line: u32, start_col: u32) -> Span {
71        Span::new(
72            Arc::clone(&self.file),
73            start_pos,
74            self.pos,
75            start_line,
76            start_col,
77        )
78    }
79
80    fn make_error(&self, msg: impl Into<String>, span: Span) -> CljxError {
81        CljxError::ReadError {
82            message: msg.into(),
83            span: Some(miette::SourceSpan::from(span)),
84            src: NamedSource::new((*self.file).clone(), (*self.source).clone()),
85        }
86    }
87
88    /// Consume characters while `is_symbol_char` holds, returning the collected
89    /// string.
90    fn read_symbol_chars(&mut self) -> String {
91        let mut buf = String::new();
92        while let Some(ch) = self.peek() {
93            if is_symbol_char(ch) {
94                buf.push(ch);
95                self.advance();
96            } else {
97                break;
98            }
99        }
100        buf
101    }
102
103    // ── Whitespace / comment skipping ─────────────────────────────────────
104
105    fn skip_whitespace_and_comments(&mut self) {
106        loop {
107            match self.peek() {
108                // Shebang: only recognised at the very start of the file.
109                Some('#') if self.pos == 0 => {
110                    if self.peek_next() == Some('!') {
111                        // skip to end of line
112                        while let Some(ch) = self.advance() {
113                            if ch == '\n' {
114                                break;
115                            }
116                        }
117                    } else {
118                        break; // '#' is meaningful, stop skipping
119                    }
120                }
121                Some(' ') | Some('\t') | Some('\r') | Some('\n') | Some(',') => {
122                    self.advance();
123                }
124                Some(';') => {
125                    while let Some(ch) = self.advance() {
126                        if ch == '\n' || ch == '\r' {
127                            break;
128                        }
129                    }
130                }
131                _ => break,
132            }
133        }
134    }
135
136    // ── `~` (unquote / unquote-splicing) ─────────────────────────────────
137
138    fn lex_unquote(
139        &mut self,
140        start_pos: usize,
141        start_line: u32,
142        start_col: u32,
143    ) -> CljxResult<(Token, Span)> {
144        self.advance(); // consume '~'
145        if self.peek() == Some('@') {
146            self.advance();
147            Ok((
148                Token::UnquoteSplice,
149                self.span_from(start_pos, start_line, start_col),
150            ))
151        } else {
152            Ok((
153                Token::Unquote,
154                self.span_from(start_pos, start_line, start_col),
155            ))
156        }
157    }
158
159    // ── `#` dispatch ──────────────────────────────────────────────────────
160
161    fn lex_hash(
162        &mut self,
163        start_pos: usize,
164        start_line: u32,
165        start_col: u32,
166    ) -> CljxResult<(Token, Span)> {
167        self.advance(); // consume '#'
168        match self.peek() {
169            Some('(') => {
170                self.advance();
171                Ok((
172                    Token::HashFn,
173                    self.span_from(start_pos, start_line, start_col),
174                ))
175            }
176            Some('{') => {
177                self.advance();
178                Ok((
179                    Token::HashSet,
180                    self.span_from(start_pos, start_line, start_col),
181                ))
182            }
183            Some('\'') => {
184                self.advance();
185                Ok((
186                    Token::HashVar,
187                    self.span_from(start_pos, start_line, start_col),
188                ))
189            }
190            Some('_') => {
191                self.advance();
192                Ok((
193                    Token::HashDiscard,
194                    self.span_from(start_pos, start_line, start_col),
195                ))
196            }
197            Some('"') => self.lex_regex(start_pos, start_line, start_col),
198            Some('?') => {
199                self.advance(); // consume '?'
200                if self.peek() == Some('@') {
201                    self.advance();
202                    Ok((
203                        Token::ReaderCondSplice,
204                        self.span_from(start_pos, start_line, start_col),
205                    ))
206                } else {
207                    Ok((
208                        Token::ReaderCond,
209                        self.span_from(start_pos, start_line, start_col),
210                    ))
211                }
212            }
213            Some('#') => self.lex_symbolic(start_pos, start_line, start_col),
214            Some(':') => self.lex_namespaced_map(start_pos, start_line, start_col),
215            Some(c) if is_symbol_start(c) => {
216                let name = self.read_symbol_chars();
217                Ok((
218                    Token::TaggedLiteral(name),
219                    self.span_from(start_pos, start_line, start_col),
220                ))
221            }
222            other => {
223                let span = self.span_from(start_pos, start_line, start_col);
224                Err(self.make_error(format!("unknown # dispatch character: {:?}", other), span))
225            }
226        }
227    }
228
229    fn lex_regex(
230        &mut self,
231        start_pos: usize,
232        start_line: u32,
233        start_col: u32,
234    ) -> CljxResult<(Token, Span)> {
235        self.advance(); // consume opening '"'
236        let mut buf = String::new();
237        loop {
238            match self.advance() {
239                None => {
240                    let span = self.span_from(start_pos, start_line, start_col);
241                    return Err(self.make_error("unterminated regex literal", span));
242                }
243                Some('"') => break,
244                Some('\\') => {
245                    // Store escape verbatim (two chars) — no processing.
246                    buf.push('\\');
247                    match self.advance() {
248                        Some(c) => buf.push(c),
249                        None => {
250                            let span = self.span_from(start_pos, start_line, start_col);
251                            return Err(self.make_error("unterminated regex literal", span));
252                        }
253                    }
254                }
255                Some(c) => buf.push(c),
256            }
257        }
258        Ok((
259            Token::Regex(buf),
260            self.span_from(start_pos, start_line, start_col),
261        ))
262    }
263
264    fn lex_symbolic(
265        &mut self,
266        start_pos: usize,
267        start_line: u32,
268        start_col: u32,
269    ) -> CljxResult<(Token, Span)> {
270        self.advance(); // consume second '#'
271        let name = self.read_symbol_chars();
272        match name.as_str() {
273            "Inf" | "-Inf" | "NaN" => Ok((
274                Token::Symbolic(name),
275                self.span_from(start_pos, start_line, start_col),
276            )),
277            _ => {
278                let span = self.span_from(start_pos, start_line, start_col);
279                Err(self.make_error(format!("unknown symbolic value: ##{name}"), span))
280            }
281        }
282    }
283
284    // ── String literal ────────────────────────────────────────────────────
285
286    fn lex_string(
287        &mut self,
288        start_pos: usize,
289        start_line: u32,
290        start_col: u32,
291    ) -> CljxResult<(Token, Span)> {
292        self.advance(); // consume opening '"'
293        let mut buf = String::new();
294        loop {
295            match self.advance() {
296                None => {
297                    let span = self.span_from(start_pos, start_line, start_col);
298                    return Err(self.make_error("unterminated string literal", span));
299                }
300                Some('"') => break,
301                Some('\\') => match self.advance() {
302                    Some('n') => buf.push('\n'),
303                    Some('t') => buf.push('\t'),
304                    Some('r') => buf.push('\r'),
305                    Some('b') => buf.push('\x08'),
306                    Some('f') => buf.push('\x0C'),
307                    Some('\\') => buf.push('\\'),
308                    Some('"') => buf.push('"'),
309                    Some('u') => {
310                        let ch = self.read_unicode_escape(start_pos, start_line, start_col)?;
311                        buf.push(ch);
312                    }
313                    Some(c) => {
314                        let span = self.span_from(start_pos, start_line, start_col);
315                        return Err(self.make_error(format!("unknown string escape: \\{c}"), span));
316                    }
317                    None => {
318                        let span = self.span_from(start_pos, start_line, start_col);
319                        return Err(self.make_error("unterminated string literal", span));
320                    }
321                },
322                Some(c) => buf.push(c),
323            }
324        }
325        Ok((
326            Token::Str(buf),
327            self.span_from(start_pos, start_line, start_col),
328        ))
329    }
330
331    /// Read exactly 4 hex digits after `\u` and return the corresponding char.
332    fn read_unicode_escape(
333        &mut self,
334        start_pos: usize,
335        start_line: u32,
336        start_col: u32,
337    ) -> CljxResult<char> {
338        let mut hex = String::with_capacity(4);
339        for _ in 0..4 {
340            match self.advance() {
341                Some(c) if c.is_ascii_hexdigit() => hex.push(c),
342                Some(c) => {
343                    let span = self.span_from(start_pos, start_line, start_col);
344                    return Err(self.make_error(
345                        format!("invalid \\u escape: expected hex digit, got {c:?}"),
346                        span,
347                    ));
348                }
349                None => {
350                    let span = self.span_from(start_pos, start_line, start_col);
351                    return Err(self.make_error("unterminated \\u escape", span));
352                }
353            }
354        }
355        let code = u32::from_str_radix(&hex, 16).unwrap();
356        char::from_u32(code).ok_or_else(|| {
357            let span = self.span_from(start_pos, start_line, start_col);
358            self.make_error(format!("invalid unicode code point: \\u{hex}"), span)
359        })
360    }
361
362    // ── Character literal `\X` ────────────────────────────────────────────
363
364    fn lex_char_literal(
365        &mut self,
366        start_pos: usize,
367        start_line: u32,
368        start_col: u32,
369    ) -> CljxResult<(Token, Span)> {
370        self.advance(); // consume '\'
371
372        // Peek ahead at all symbol-constituent chars to figure out the name.
373        let rest_start = self.pos;
374        let rest: String = self.source[rest_start..]
375            .chars()
376            .take_while(|&c| c.is_alphanumeric() || c == '-')
377            .collect();
378
379        let ch = match rest.as_str() {
380            "newline" => {
381                self.pos += "newline".len();
382                self.col += "newline".len() as u32;
383                '\n'
384            }
385            "space" => {
386                self.pos += "space".len();
387                self.col += "space".len() as u32;
388                ' '
389            }
390            "tab" => {
391                self.pos += "tab".len();
392                self.col += "tab".len() as u32;
393                '\t'
394            }
395            "backspace" => {
396                self.pos += "backspace".len();
397                self.col += "backspace".len() as u32;
398                '\x08'
399            }
400            "formfeed" => {
401                self.pos += "formfeed".len();
402                self.col += "formfeed".len() as u32;
403                '\x0C'
404            }
405            "return" => {
406                self.pos += "return".len();
407                self.col += "return".len() as u32;
408                '\r'
409            }
410            _ if rest.starts_with('u') && rest.len() >= 5 => {
411                // Try \uXXXX
412                let hex_part = &rest[1..5];
413                if hex_part.chars().all(|c| c.is_ascii_hexdigit()) {
414                    let code = u32::from_str_radix(hex_part, 16).unwrap();
415                    let c = char::from_u32(code).ok_or_else(|| {
416                        let span = self.span_from(start_pos, start_line, start_col);
417                        self.make_error(
418                            format!("invalid unicode code point in char literal: \\u{hex_part}"),
419                            span,
420                        )
421                    })?;
422                    // advance 5 bytes: 'u' + 4 hex digits
423                    self.pos += 5;
424                    self.col += 5;
425                    c
426                } else {
427                    let span = self.span_from(start_pos, start_line, start_col);
428                    return Err(self.make_error(format!("unknown character name: {rest}"), span));
429                }
430            }
431            _ if rest.starts_with('o') && rest.len() > 1 => {
432                let digits = &rest[1..];
433                if digits.is_empty()
434                    || digits.len() > 3
435                    || !digits.chars().all(|c| matches!(c, '0'..='7'))
436                {
437                    let span = self.span_from(start_pos, start_line, start_col);
438                    return Err(self.make_error(format!("invalid octal character: {rest}"), span));
439                }
440                let code = u32::from_str_radix(digits, 8).unwrap();
441                if code > 0o377 {
442                    let span = self.span_from(start_pos, start_line, start_col);
443                    return Err(
444                        self.make_error(format!("octal character out of range: {rest}"), span)
445                    );
446                }
447                self.pos += rest.len();
448                self.col += rest.len() as u32;
449                char::from_u32(code).unwrap()
450            }
451            _ if rest.len() == 1 => {
452                // Single ASCII or first char
453                let c = self.source[rest_start..].chars().next().unwrap();
454                self.pos += c.len_utf8();
455                self.col += c.len_utf8() as u32;
456                c
457            }
458            _ if rest.is_empty() => {
459                // Nothing after backslash — try a single non-alphanumeric char
460                match self.source[rest_start..].chars().next() {
461                    Some(c) => {
462                        self.pos += c.len_utf8();
463                        self.col += c.len_utf8() as u32;
464                        c
465                    }
466                    None => {
467                        let span = self.span_from(start_pos, start_line, start_col);
468                        return Err(self.make_error("unexpected end of file after \\", span));
469                    }
470                }
471            }
472            _ => {
473                let span = self.span_from(start_pos, start_line, start_col);
474                return Err(self.make_error(format!("unknown character name: {rest}"), span));
475            }
476        };
477
478        Ok((
479            Token::Char(ch),
480            self.span_from(start_pos, start_line, start_col),
481        ))
482    }
483
484    /// Lex the `#:ns` / `#::` / `#::alias` prefix of a namespaced map. `#` is
485    /// already consumed and the next char is `:`. The `{` is left in the stream
486    /// so the parser reads the body through the ordinary map path.
487    fn lex_namespaced_map(
488        &mut self,
489        start_pos: usize,
490        start_line: u32,
491        start_col: u32,
492    ) -> CljxResult<(Token, Span)> {
493        self.advance(); // consume first ':'
494        let auto = self.peek() == Some(':');
495        if auto {
496            self.advance(); // consume second ':'
497        }
498        let text = self.read_symbol_chars();
499        let ns = crate::namespaced_map::MapNs::parse(&text, auto).map_err(|msg| {
500            let span = self.span_from(start_pos, start_line, start_col);
501            self.make_error(msg, span)
502        })?;
503        // The JVM reader skips whitespace between the prefix and the map, but
504        // not comments: `#:foo {…}` and `#:foo, {…}` are the same literal.
505        self.skip_whitespace_only();
506        if self.peek() != Some('{') {
507            let span = self.span_from(start_pos, start_line, start_col);
508            return Err(self.make_error("namespaced map literal must be followed by a map", span));
509        }
510        Ok((
511            Token::NamespacedMap(ns),
512            self.span_from(start_pos, start_line, start_col),
513        ))
514    }
515
516    /// Skip spaces, tabs, newlines and commas - and nothing else.
517    fn skip_whitespace_only(&mut self) {
518        while let Some(ch) = self.peek() {
519            if ch.is_whitespace() || ch == ',' {
520                self.advance();
521            } else {
522                break;
523            }
524        }
525    }
526
527    // ── Keyword ───────────────────────────────────────────────────────────
528
529    fn lex_keyword(
530        &mut self,
531        start_pos: usize,
532        start_line: u32,
533        start_col: u32,
534    ) -> CljxResult<(Token, Span)> {
535        self.advance(); // consume first ':'
536        if self.peek() == Some(':') {
537            self.advance(); // consume second ':'
538            let name = self.read_symbol_chars();
539            if name.is_empty() {
540                let span = self.span_from(start_pos, start_line, start_col);
541                return Err(self.make_error("empty auto-resolved keyword", span));
542            }
543            Ok((
544                Token::AutoKeyword(name),
545                self.span_from(start_pos, start_line, start_col),
546            ))
547        } else {
548            let name = self.read_symbol_chars();
549            if name.is_empty() {
550                let span = self.span_from(start_pos, start_line, start_col);
551                return Err(self.make_error("empty keyword", span));
552            }
553            Ok((
554                Token::Keyword(name),
555                self.span_from(start_pos, start_line, start_col),
556            ))
557        }
558    }
559
560    // ── Symbol (and nil/true/false) ────────────────────────────────────────
561
562    fn lex_symbol(
563        &mut self,
564        start_pos: usize,
565        start_line: u32,
566        start_col: u32,
567    ) -> CljxResult<(Token, Span)> {
568        let mut name = self.read_symbol_chars();
569
570        // Peek for a version suffix: `@<commit-hash>`.  We only consume the `@`
571        // when it is immediately followed by 7–40 hex characters so that a
572        // standalone `@expr` (deref reader macro) is never affected — deref
573        // always starts a *new* form where `@` is the first character, not a
574        // mid-symbol suffix.
575        if self.peek() == Some('@') {
576            let version_candidate = self.peek_version_hash();
577            if let Some(hash) = version_candidate {
578                self.advance(); // consume '@'
579                for _ in 0..hash.len() {
580                    self.advance();
581                }
582                name.push('@');
583                name.push_str(&hash);
584            }
585        }
586
587        let tok = match name.as_str() {
588            "nil" => Token::Nil,
589            "true" => Token::Bool(true),
590            "false" => Token::Bool(false),
591            _ => Token::Symbol(name),
592        };
593        Ok((tok, self.span_from(start_pos, start_line, start_col)))
594    }
595
596    /// Look ahead past the `@` that `peek()` just returned and collect
597    /// characters as long as they are ASCII hex digits, up to 40.  Returns
598    /// `Some(hash)` if the candidate is 7–40 hex chars followed by a
599    /// non-hex-digit (or EOF), `None` otherwise.  Does **not** advance the
600    /// cursor.
601    fn peek_version_hash(&self) -> Option<String> {
602        // Start one byte past the current `@`.
603        let at_byte = self.pos + 1; // '@' is single-byte ASCII
604        let rest = &self.source[at_byte..];
605        let hash: String = rest
606            .chars()
607            .take(40)
608            .take_while(|c| c.is_ascii_hexdigit())
609            .collect();
610        if hash.len() >= 7 {
611            // Make sure the character after the hash is a delimiter (or EOF).
612            let after = rest[hash.len()..].chars().next();
613            let is_delimited = after.is_none_or(|c| !c.is_ascii_hexdigit());
614            if is_delimited {
615                return Some(hash);
616            }
617        }
618        None
619    }
620
621    // ── Number ────────────────────────────────────────────────────────────
622
623    fn lex_number(
624        &mut self,
625        start_pos: usize,
626        start_line: u32,
627        start_col: u32,
628    ) -> CljxResult<(Token, Span)> {
629        // Optional sign
630        let negative = match self.peek() {
631            Some('-') => {
632                self.advance();
633                true
634            }
635            Some('+') => {
636                self.advance();
637                false
638            }
639            _ => false,
640        };
641        let sign_str = if negative { "-" } else { "" };
642
643        // Integer part (decimal digits)
644        let mut int_part = String::new();
645        while let Some(c) = self.peek() {
646            if c.is_ascii_digit() {
647                int_part.push(c);
648                self.advance();
649            } else {
650                break;
651            }
652        }
653
654        // Hex literal: 0x / 0X  (also -0x…)
655        if int_part == "0" && matches!(self.peek(), Some('x') | Some('X')) {
656            self.advance(); // consume 'x'/'X'
657            let mut hex = String::new();
658            while let Some(c) = self.peek() {
659                if c.is_ascii_hexdigit() {
660                    hex.push(c);
661                    self.advance();
662                } else {
663                    break;
664                }
665            }
666            if hex.is_empty() {
667                let span = self.span_from(start_pos, start_line, start_col);
668                return Err(self.make_error("expected hex digits after 0x", span));
669            }
670            let value = u128::from_str_radix(&hex, 16).unwrap_or(u128::MAX);
671            let span = self.span_from(start_pos, start_line, start_col);
672            return if negative {
673                // -0x8000000000000000 == i64::MIN is valid; anything larger overflows.
674                if value <= (i64::MAX as u128) + 1 {
675                    Ok((Token::Int(0i64.wrapping_sub(value as i64)), span))
676                } else {
677                    // Store as signed decimal string for BigInt.
678                    Ok((Token::BigInt(format!("-{value}")), span))
679                }
680            } else if value <= i64::MAX as u128 {
681                Ok((Token::Int(value as i64), span))
682            } else {
683                Ok((Token::BigInt(value.to_string()), span))
684            };
685        }
686
687        // Radix literal: NNrDIGITS
688        if matches!(self.peek(), Some('r') | Some('R')) {
689            let radix: u32 = int_part.parse().unwrap_or(0);
690            self.advance(); // consume 'r'/'R'
691            let mut digits = String::new();
692            while let Some(c) = self.peek() {
693                if c.is_ascii_alphanumeric() {
694                    digits.push(c);
695                    self.advance();
696                } else {
697                    break;
698                }
699            }
700            let mut value: u128 = 0;
701            for c in digits.chars() {
702                let d = c.to_digit(radix).ok_or_else(|| {
703                    let span = self.span_from(start_pos, start_line, start_col);
704                    self.make_error(format!("invalid digit {c:?} for radix {radix}"), span)
705                })?;
706                value = value.wrapping_mul(radix as u128).wrapping_add(d as u128);
707            }
708            if negative {
709                // Check if it fits as negative i64
710                if value <= (i64::MAX as u128) + 1 {
711                    let signed = -(value as i64);
712                    return Ok((
713                        Token::Int(signed),
714                        self.span_from(start_pos, start_line, start_col),
715                    ));
716                } else {
717                    // Store as decimal string with sign
718                    return Ok((
719                        Token::BigInt(format!("-{value}")),
720                        self.span_from(start_pos, start_line, start_col),
721                    ));
722                }
723            } else if value <= i64::MAX as u128 {
724                return Ok((
725                    Token::Int(value as i64),
726                    self.span_from(start_pos, start_line, start_col),
727                ));
728            } else {
729                return Ok((
730                    Token::BigInt(value.to_string()),
731                    self.span_from(start_pos, start_line, start_col),
732                ));
733            }
734        }
735
736        // BigInt suffix 'N'
737        if self.peek() == Some('N') {
738            self.advance();
739            return Ok((
740                Token::BigInt(format!("{sign_str}{int_part}")),
741                self.span_from(start_pos, start_line, start_col),
742            ));
743        }
744
745        // BigDecimal suffix 'M' on integer literal (e.g. 4M)
746        if self.peek() == Some('M') {
747            self.advance();
748            return Ok((
749                Token::BigDecimal(format!("{sign_str}{int_part}")),
750                self.span_from(start_pos, start_line, start_col),
751            ));
752        }
753
754        // Float: decimal point or exponent
755        if matches!(self.peek(), Some('.') | Some('e') | Some('E')) {
756            let mut raw = format!("{sign_str}{int_part}");
757            if self.peek() == Some('.') {
758                raw.push('.');
759                self.advance();
760                while let Some(c) = self.peek() {
761                    if c.is_ascii_digit() {
762                        raw.push(c);
763                        self.advance();
764                    } else {
765                        break;
766                    }
767                }
768            }
769            if matches!(self.peek(), Some('e') | Some('E')) {
770                raw.push('e');
771                self.advance();
772                if matches!(self.peek(), Some('+') | Some('-')) {
773                    raw.push(self.peek().unwrap());
774                    self.advance();
775                }
776                while let Some(c) = self.peek() {
777                    if c.is_ascii_digit() {
778                        raw.push(c);
779                        self.advance();
780                    } else {
781                        break;
782                    }
783                }
784            }
785            // BigDecimal suffix 'M'
786            if self.peek() == Some('M') {
787                self.advance();
788                return Ok((
789                    Token::BigDecimal(raw),
790                    self.span_from(start_pos, start_line, start_col),
791                ));
792            }
793            let val: f64 = raw.parse().map_err(|_| {
794                let span = self.span_from(start_pos, start_line, start_col);
795                self.make_error(format!("invalid float: {raw}"), span)
796            })?;
797            return Ok((
798                Token::Float(val),
799                self.span_from(start_pos, start_line, start_col),
800            ));
801        }
802
803        // Ratio: INT/DIGITS — only if next char after '/' is a digit
804        if self.peek() == Some('/') && matches!(self.peek_next(), Some(c) if c.is_ascii_digit()) {
805            self.advance(); // consume '/'
806            let mut denom = String::new();
807            while let Some(c) = self.peek() {
808                if c.is_ascii_digit() {
809                    denom.push(c);
810                    self.advance();
811                } else {
812                    break;
813                }
814            }
815            return Ok((
816                Token::Ratio(format!("{sign_str}{int_part}/{denom}")),
817                self.span_from(start_pos, start_line, start_col),
818            ));
819        }
820
821        // Plain integer
822        let full = format!("{sign_str}{int_part}");
823        if int_part.len() > 1 && int_part.starts_with('0') {
824            if !int_part.chars().all(|c| matches!(c, '0'..='7')) {
825                let span = self.span_from(start_pos, start_line, start_col);
826                return Err(self.make_error(format!("invalid octal integer: {full}"), span));
827            }
828            let magnitude = u128::from_str_radix(&int_part, 8).unwrap_or(u128::MAX);
829            let signed = if negative {
830                -(magnitude as i128)
831            } else {
832                magnitude as i128
833            };
834            return if let Ok(value) = i64::try_from(signed) {
835                Ok((
836                    Token::Int(value),
837                    self.span_from(start_pos, start_line, start_col),
838                ))
839            } else {
840                Ok((
841                    Token::BigInt(signed.to_string()),
842                    self.span_from(start_pos, start_line, start_col),
843                ))
844            };
845        }
846        match full.parse::<i64>() {
847            Ok(n) => Ok((
848                Token::Int(n),
849                self.span_from(start_pos, start_line, start_col),
850            )),
851            Err(_) => {
852                // Overflow: store decimal string
853                Ok((
854                    Token::BigInt(full),
855                    self.span_from(start_pos, start_line, start_col),
856                ))
857            }
858        }
859    }
860
861    // ── Top-level token dispatch ──────────────────────────────────────────
862
863    pub fn next_token(&mut self) -> CljxResult<(Token, Span)> {
864        self.skip_whitespace_and_comments();
865
866        let start_pos = self.pos;
867        let start_line = self.line;
868        let start_col = self.col;
869
870        let ch = match self.peek() {
871            None => {
872                return Ok((Token::Eof, self.span_from(start_pos, start_line, start_col)));
873            }
874            Some(c) => c,
875        };
876
877        match ch {
878            '(' => {
879                self.advance();
880                Ok((
881                    Token::LParen,
882                    self.span_from(start_pos, start_line, start_col),
883                ))
884            }
885            ')' => {
886                self.advance();
887                Ok((
888                    Token::RParen,
889                    self.span_from(start_pos, start_line, start_col),
890                ))
891            }
892            '[' => {
893                self.advance();
894                Ok((
895                    Token::LBracket,
896                    self.span_from(start_pos, start_line, start_col),
897                ))
898            }
899            ']' => {
900                self.advance();
901                Ok((
902                    Token::RBracket,
903                    self.span_from(start_pos, start_line, start_col),
904                ))
905            }
906            '{' => {
907                self.advance();
908                Ok((
909                    Token::LBrace,
910                    self.span_from(start_pos, start_line, start_col),
911                ))
912            }
913            '}' => {
914                self.advance();
915                Ok((
916                    Token::RBrace,
917                    self.span_from(start_pos, start_line, start_col),
918                ))
919            }
920            '\'' => {
921                self.advance();
922                Ok((
923                    Token::Quote,
924                    self.span_from(start_pos, start_line, start_col),
925                ))
926            }
927            '`' => {
928                self.advance();
929                Ok((
930                    Token::SyntaxQuote,
931                    self.span_from(start_pos, start_line, start_col),
932                ))
933            }
934            '@' => {
935                self.advance();
936                Ok((
937                    Token::Deref,
938                    self.span_from(start_pos, start_line, start_col),
939                ))
940            }
941            '^' => {
942                self.advance();
943                Ok((
944                    Token::Meta,
945                    self.span_from(start_pos, start_line, start_col),
946                ))
947            }
948            '~' => self.lex_unquote(start_pos, start_line, start_col),
949            '#' => self.lex_hash(start_pos, start_line, start_col),
950            '"' => self.lex_string(start_pos, start_line, start_col),
951            '\\' => self.lex_char_literal(start_pos, start_line, start_col),
952            ':' => self.lex_keyword(start_pos, start_line, start_col),
953            c if c.is_ascii_digit() => self.lex_number(start_pos, start_line, start_col),
954            '+' | '-' if matches!(self.peek_next(), Some(d) if d.is_ascii_digit()) => {
955                self.lex_number(start_pos, start_line, start_col)
956            }
957            c if is_symbol_start(c) => self.lex_symbol(start_pos, start_line, start_col),
958            // '+' and '-' alone (or before non-digit) are symbols
959            '+' | '-' => self.lex_symbol(start_pos, start_line, start_col),
960            c => {
961                self.advance();
962                let span = self.span_from(start_pos, start_line, start_col);
963                Err(self.make_error(format!("unexpected character: {c:?}"), span))
964            }
965        }
966    }
967}
968
969impl Iterator for Lexer {
970    type Item = CljxResult<(Token, Span)>;
971
972    fn next(&mut self) -> Option<Self::Item> {
973        match self.next_token() {
974            Ok((Token::Eof, _)) => None,
975            result => Some(result),
976        }
977    }
978}
979
980// ─── Tests ────────────────────────────────────────────────────────────────────
981
982#[cfg(test)]
983mod tests {
984    use super::*;
985
986    fn lex_all(src: &str) -> Vec<Token> {
987        Lexer::new(src.to_string(), "<test>".to_string())
988            .map(|r: CljxResult<(Token, Span)>| r.expect("lex error").0)
989            .collect()
990    }
991
992    fn lex_one(src: &str) -> Token {
993        let mut l = Lexer::new(src.to_string(), "<test>".to_string());
994        l.next_token().expect("lex error").0
995    }
996
997    fn lex_err(src: &str) -> String {
998        let mut l = Lexer::new(src.to_string(), "<test>".to_string());
999        loop {
1000            match l.next_token() {
1001                Err(CljxError::ReadError { message, .. }) => return message,
1002                Err(e) => panic!("unexpected error type: {e}"),
1003                Ok((Token::Eof, _)) => panic!("expected an error but got Eof"),
1004                Ok(_) => {}
1005            }
1006        }
1007    }
1008
1009    // ── nil / bool ────────────────────────────────────────────────────────
1010
1011    #[test]
1012    fn test_nil() {
1013        assert_eq!(lex_one("nil"), Token::Nil);
1014    }
1015
1016    #[test]
1017    fn test_bool() {
1018        assert_eq!(lex_one("true"), Token::Bool(true));
1019        assert_eq!(lex_one("false"), Token::Bool(false));
1020    }
1021
1022    // ── Integers ──────────────────────────────────────────────────────────
1023
1024    #[test]
1025    fn test_int_plain() {
1026        assert_eq!(lex_one("42"), Token::Int(42));
1027        assert_eq!(lex_one("-42"), Token::Int(-42));
1028        assert_eq!(lex_one("+42"), Token::Int(42));
1029        assert_eq!(lex_one("0"), Token::Int(0));
1030    }
1031
1032    #[test]
1033    fn test_bigint_suffix() {
1034        assert_eq!(lex_one("42N"), Token::BigInt("42".to_string()));
1035        assert_eq!(lex_one("-42N"), Token::BigInt("-42".to_string()));
1036    }
1037
1038    #[test]
1039    fn test_hex_literal() {
1040        assert_eq!(lex_one("0xff"), Token::Int(255));
1041        assert_eq!(lex_one("0xFF"), Token::Int(255));
1042        assert_eq!(lex_one("0x0"), Token::Int(0));
1043        assert_eq!(lex_one("0x7FFFFFFFFFFFFFFF"), Token::Int(i64::MAX));
1044        assert_eq!(lex_one("-0x8000000000000000"), Token::Int(i64::MIN));
1045        assert_eq!(lex_one("-0xff"), Token::Int(-255));
1046        // Overflow → BigInt
1047        match lex_one("0xFFFFFFFFFFFFFFFF") {
1048            Token::BigInt(_) => {}
1049            other => panic!("expected BigInt for 0xFFFF…, got {other:?}"),
1050        }
1051    }
1052
1053    #[test]
1054    fn test_radix() {
1055        assert_eq!(lex_one("2r1010"), Token::Int(10));
1056        assert_eq!(lex_one("8r77"), Token::Int(63));
1057        assert_eq!(lex_one("16rFF"), Token::Int(255));
1058        assert_eq!(lex_one("16rff"), Token::Int(255));
1059        assert_eq!(lex_one("36rZ"), Token::Int(35));
1060    }
1061
1062    #[test]
1063    fn test_radix_overflow() {
1064        // 2^64 fits in u128 but not i64
1065        let tok = lex_one("10r18446744073709551616");
1066        match tok {
1067            Token::BigInt(_) => {}
1068            other => panic!("expected BigInt, got {other:?}"),
1069        }
1070    }
1071
1072    // ── Floats ────────────────────────────────────────────────────────────
1073
1074    #[test]
1075    #[allow(clippy::approx_constant)]
1076    fn test_floats() {
1077        assert_eq!(lex_one("3.14"), Token::Float(3.14));
1078        assert_eq!(lex_one("1e10"), Token::Float(1e10));
1079        assert_eq!(lex_one("1.5e-3"), Token::Float(1.5e-3));
1080        assert_eq!(lex_one("-0.5"), Token::Float(-0.5));
1081    }
1082
1083    #[test]
1084    fn test_bigdecimal() {
1085        assert_eq!(lex_one("3.14M"), Token::BigDecimal("3.14".to_string()));
1086        assert_eq!(lex_one("1e5M"), Token::BigDecimal("1e5".to_string()));
1087    }
1088
1089    // ── Ratio ────────────────────────────────────────────────────────────
1090
1091    #[test]
1092    fn test_ratio() {
1093        assert_eq!(lex_one("3/4"), Token::Ratio("3/4".to_string()));
1094        assert_eq!(lex_one("-1/2"), Token::Ratio("-1/2".to_string()));
1095    }
1096
1097    #[test]
1098    fn test_ratio_vs_symbol() {
1099        // "3/foo" should lex as Int(3) then Symbol("/foo") — not a ratio
1100        let toks = lex_all("3/foo");
1101        assert_eq!(toks[0], Token::Int(3));
1102        assert_eq!(toks[1], Token::Symbol("/foo".to_string()));
1103    }
1104
1105    // ── Char literals ────────────────────────────────────────────────────
1106
1107    #[test]
1108    fn test_char_simple() {
1109        assert_eq!(lex_one("\\a"), Token::Char('a'));
1110    }
1111
1112    #[test]
1113    fn test_char_named() {
1114        assert_eq!(lex_one("\\newline"), Token::Char('\n'));
1115        assert_eq!(lex_one("\\space"), Token::Char(' '));
1116        assert_eq!(lex_one("\\tab"), Token::Char('\t'));
1117        assert_eq!(lex_one("\\backspace"), Token::Char('\x08'));
1118        assert_eq!(lex_one("\\formfeed"), Token::Char('\x0C'));
1119        assert_eq!(lex_one("\\return"), Token::Char('\r'));
1120    }
1121
1122    #[test]
1123    fn test_char_unicode() {
1124        assert_eq!(lex_one("\\u0041"), Token::Char('A'));
1125        assert_eq!(lex_one("\\u00e9"), Token::Char('é'));
1126    }
1127
1128    #[test]
1129    fn test_char_octal() {
1130        assert_eq!(lex_one("\\o"), Token::Char('o'));
1131        assert_eq!(lex_one("\\o0"), Token::Char('\0'));
1132        assert_eq!(lex_one("\\o101"), Token::Char('A'));
1133        assert_eq!(lex_one("\\o377"), Token::Char('\u{ff}'));
1134    }
1135
1136    // ── Strings ──────────────────────────────────────────────────────────
1137
1138    #[test]
1139    fn test_string_basic() {
1140        assert_eq!(lex_one("\"hello\""), Token::Str("hello".to_string()));
1141    }
1142
1143    #[test]
1144    fn test_string_escapes() {
1145        assert_eq!(
1146            lex_one(r#""\n\t\r\b\f\\\"" "#),
1147            Token::Str("\n\t\r\x08\x0C\\\"".to_string())
1148        );
1149    }
1150
1151    #[test]
1152    fn test_string_unicode_escape() {
1153        assert_eq!(lex_one("\"\\u0041\""), Token::Str("A".to_string()));
1154    }
1155
1156    // ── Symbols ──────────────────────────────────────────────────────────
1157
1158    #[test]
1159    fn test_symbols() {
1160        assert_eq!(lex_one("foo"), Token::Symbol("foo".to_string()));
1161        assert_eq!(lex_one("ns/name"), Token::Symbol("ns/name".to_string()));
1162        assert_eq!(lex_one("/"), Token::Symbol("/".to_string()));
1163        assert_eq!(lex_one(".."), Token::Symbol("..".to_string()));
1164        assert_eq!(lex_one(".method"), Token::Symbol(".method".to_string()));
1165        assert_eq!(lex_one("+"), Token::Symbol("+".to_string()));
1166        assert_eq!(lex_one("-"), Token::Symbol("-".to_string()));
1167        assert_eq!(lex_one("+foo"), Token::Symbol("+foo".to_string()));
1168    }
1169
1170    #[test]
1171    fn test_auto_gensym_symbol() {
1172        // `x#` is a single symbol token (trailing `#` is auto-gensym syntax,
1173        // not a `#`-dispatch macro — `#` is non-terminating mid-token).
1174        assert_eq!(lex_one("x#"), Token::Symbol("x#".to_string()));
1175        let toks = lex_all("`(let [x# 1] x#)");
1176        assert_eq!(
1177            toks,
1178            vec![
1179                Token::SyntaxQuote,
1180                Token::LParen,
1181                Token::Symbol("let".to_string()),
1182                Token::LBracket,
1183                Token::Symbol("x#".to_string()),
1184                Token::Int(1),
1185                Token::RBracket,
1186                Token::Symbol("x#".to_string()),
1187                Token::RParen,
1188            ]
1189        );
1190    }
1191
1192    // ── Keywords ─────────────────────────────────────────────────────────
1193
1194    #[test]
1195    fn test_keyword() {
1196        assert_eq!(lex_one(":foo"), Token::Keyword("foo".to_string()));
1197        assert_eq!(lex_one(":ns/name"), Token::Keyword("ns/name".to_string()));
1198    }
1199
1200    #[test]
1201    fn test_keyword_with_embedded_colon() {
1202        // An embedded `:` is a literal name character, not a sub-token
1203        // boundary — `:xlink:href` is one keyword, not two.
1204        assert_eq!(
1205            lex_one(":xlink:href"),
1206            Token::Keyword("xlink:href".to_string())
1207        );
1208        let toks = lex_all("[:xlink:href]");
1209        assert_eq!(
1210            toks,
1211            vec![
1212                Token::LBracket,
1213                Token::Keyword("xlink:href".to_string()),
1214                Token::RBracket,
1215            ]
1216        );
1217    }
1218
1219    #[test]
1220    fn test_auto_keyword() {
1221        assert_eq!(lex_one("::foo"), Token::AutoKeyword("foo".to_string()));
1222        assert_eq!(
1223            lex_one("::ns/alias"),
1224            Token::AutoKeyword("ns/alias".to_string())
1225        );
1226    }
1227
1228    #[test]
1229    fn test_namespaced_map_prefix() {
1230        use crate::namespaced_map::MapNs;
1231        // The `{` is deliberately NOT consumed - the parser reads the body
1232        // through the ordinary map path.
1233        assert_eq!(
1234            lex_one("#:adt{:a 1}"),
1235            Token::NamespacedMap(MapNs::Literal("adt".to_string()))
1236        );
1237        assert_eq!(lex_one("#::{:a 1}"), Token::NamespacedMap(MapNs::CurrentNs));
1238        assert_eq!(
1239            lex_one("#::al{:a 1}"),
1240            Token::NamespacedMap(MapNs::Alias("al".to_string()))
1241        );
1242    }
1243
1244    #[test]
1245    fn test_namespaced_map_prefix_skips_whitespace_before_the_brace() {
1246        use crate::namespaced_map::MapNs;
1247        // The JVM accepts whitespace and commas between the prefix and `{`.
1248        for src in ["#:adt {:a 1}", "#:adt, {:a 1}", "#:adt\n{:a 1}"] {
1249            assert_eq!(
1250                lex_one(src),
1251                Token::NamespacedMap(MapNs::Literal("adt".to_string())),
1252                "{src}"
1253            );
1254        }
1255        assert_eq!(
1256            lex_one("#:: {:a 1}"),
1257            Token::NamespacedMap(MapNs::CurrentNs)
1258        );
1259        assert_eq!(
1260            lex_one("#::al {:a 1}"),
1261            Token::NamespacedMap(MapNs::Alias("al".to_string()))
1262        );
1263    }
1264
1265    #[test]
1266    fn test_namespaced_map_prefix_does_not_skip_comments() {
1267        // Whitespace only - a comment between the prefix and `{` is not a map.
1268        assert!(lex_err("#:adt ;; here\n{:a 1}").contains("must be followed by a map"));
1269    }
1270
1271    #[test]
1272    fn test_namespaced_map_prefix_errors() {
1273        assert!(lex_err("#:{:a 1}").contains("requires a namespace"));
1274        assert!(lex_err("#:adt [1]").contains("must be followed by a map"));
1275    }
1276
1277    #[test]
1278    fn test_namespaced_map_prefix_must_be_an_unqualified_symbol() {
1279        for src in [
1280            "#:foo/bar{:a 1}",
1281            "#:1{:a 1}",
1282            "#:nil{:a 1}",
1283            "#::foo/bar{:a 1}",
1284        ] {
1285            let err = lex_err(src);
1286            assert!(err.contains("unqualified symbol"), "{src}: {err}");
1287        }
1288    }
1289
1290    // ── Delimiters ───────────────────────────────────────────────────────
1291
1292    #[test]
1293    fn test_delimiters() {
1294        assert_eq!(
1295            lex_all("([{}])"),
1296            vec![
1297                Token::LParen,
1298                Token::LBracket,
1299                Token::LBrace,
1300                Token::RBrace,
1301                Token::RBracket,
1302                Token::RParen,
1303            ]
1304        );
1305    }
1306
1307    // ── Reader macros ────────────────────────────────────────────────────
1308
1309    #[test]
1310    fn test_reader_macros() {
1311        assert_eq!(lex_one("'x"), Token::Quote);
1312        assert_eq!(lex_one("`x"), Token::SyntaxQuote);
1313        assert_eq!(lex_one("~x"), Token::Unquote);
1314        assert_eq!(lex_one("~@x"), Token::UnquoteSplice);
1315        assert_eq!(lex_one("@x"), Token::Deref);
1316        assert_eq!(lex_one("^x"), Token::Meta);
1317    }
1318
1319    // ── `#` dispatch ─────────────────────────────────────────────────────
1320
1321    #[test]
1322    fn test_hash_dispatch() {
1323        assert_eq!(lex_one("#("), Token::HashFn);
1324        assert_eq!(lex_one("#{"), Token::HashSet);
1325        assert_eq!(lex_one("#'"), Token::HashVar);
1326        assert_eq!(lex_one("#_"), Token::HashDiscard);
1327        assert_eq!(lex_one("#?"), Token::ReaderCond);
1328        assert_eq!(lex_one("#?@"), Token::ReaderCondSplice);
1329    }
1330
1331    #[test]
1332    fn test_regex() {
1333        assert_eq!(lex_one("#\"[a-z]+\""), Token::Regex("[a-z]+".to_string()));
1334    }
1335
1336    #[test]
1337    fn test_symbolic() {
1338        assert_eq!(lex_one("##Inf"), Token::Symbolic("Inf".to_string()));
1339        assert_eq!(lex_one("##-Inf"), Token::Symbolic("-Inf".to_string()));
1340        assert_eq!(lex_one("##NaN"), Token::Symbolic("NaN".to_string()));
1341    }
1342
1343    #[test]
1344    fn test_tagged_literal() {
1345        assert_eq!(lex_one("#mytag"), Token::TaggedLiteral("mytag".to_string()));
1346    }
1347
1348    // ── Multi-token ──────────────────────────────────────────────────────
1349
1350    #[test]
1351    fn test_multi_token() {
1352        let toks = lex_all("(+ 1 2)");
1353        assert_eq!(
1354            toks,
1355            vec![
1356                Token::LParen,
1357                Token::Symbol("+".to_string()),
1358                Token::Int(1),
1359                Token::Int(2),
1360                Token::RParen,
1361            ]
1362        );
1363    }
1364
1365    // ── Whitespace / comments ────────────────────────────────────────────
1366
1367    #[test]
1368    fn test_comma_skipped() {
1369        assert_eq!(lex_all("{,,,}"), vec![Token::LBrace, Token::RBrace]);
1370    }
1371
1372    #[test]
1373    fn test_comment_skipped() {
1374        assert_eq!(lex_all("; this is a comment\n42"), vec![Token::Int(42)]);
1375    }
1376
1377    #[test]
1378    fn test_shebang_skipped() {
1379        assert_eq!(lex_all("#!/usr/bin/env cljx\n42"), vec![Token::Int(42)]);
1380    }
1381
1382    // ── Span tracking ────────────────────────────────────────────────────
1383
1384    #[test]
1385    fn test_span_col() {
1386        let mut l = Lexer::new("  foo".to_string(), "<test>".to_string());
1387        let (_tok, span) = l.next_token().unwrap();
1388        assert_eq!(span.start, 2);
1389        assert_eq!(span.col, 3);
1390    }
1391
1392    #[test]
1393    fn test_span_newline() {
1394        let mut l = Lexer::new("a\nb".to_string(), "<test>".to_string());
1395        l.next_token().unwrap(); // consume 'a'
1396        let (_tok, span) = l.next_token().unwrap(); // 'b'
1397        assert_eq!(span.line, 2);
1398        assert_eq!(span.col, 1);
1399    }
1400
1401    // ── Errors ───────────────────────────────────────────────────────────
1402
1403    #[test]
1404    fn test_error_unterminated_string() {
1405        let msg = lex_err("\"unterminated");
1406        assert!(msg.contains("unterminated string"));
1407    }
1408
1409    #[test]
1410    fn test_error_bad_hash_dispatch() {
1411        // '#1' is invalid: '1' is not a symbol start and not a special dispatch char
1412        let msg = lex_err("#1");
1413        assert!(msg.contains("unknown # dispatch"));
1414    }
1415
1416    #[test]
1417    fn test_error_bad_unicode_escape_in_string() {
1418        let msg = lex_err("\"\\uGHIJ\"");
1419        assert!(msg.contains("invalid") || msg.contains("hex"));
1420    }
1421
1422    #[test]
1423    fn test_error_unknown_char_name() {
1424        let msg = lex_err("\\bogus");
1425        assert!(msg.contains("unknown character name"));
1426    }
1427
1428    #[test]
1429    fn test_error_unknown_symbolic() {
1430        let msg = lex_err("##Bogus");
1431        assert!(msg.contains("unknown symbolic value"));
1432    }
1433
1434    #[test]
1435    fn test_error_bad_string_escape() {
1436        let msg = lex_err("\"\\q\"");
1437        assert!(msg.contains("unknown string escape"));
1438    }
1439}
cljrs_reader/lexer.rs

cljrs_reader/
lexer.rs
