Skip to main content

nybl/
lexer.rs

1#[cfg(all(feature = "no_std", not(feature = "std")))]
2use alloc::{format, string::String, vec::Vec};
3
4use crate::error::NyblError;
5
6pub(crate) const I64_MIN_MAGNITUDE: u64 = (i64::MAX as u64) + 1;
7pub(crate) const I64_MIN_MAGNITUDE_TEXT: &str = "9223372036854775808";
8
9pub(crate) fn integer_literal_out_of_range(digits: &str) -> String {
10    format!("Integer literal out of range for i64: {digits}")
11}
12
13#[derive(Debug, Clone, PartialEq)]
14pub enum StringPart {
15    Literal(String),
16    Variable(String),
17}
18
19#[derive(Debug, Clone, PartialEq)]
20pub enum Token {
21    // Literals
22    /// Integer literal — a digit sequence with no `.` part.
23    /// Lexed to `i64` at scan time; the parser maps it to
24    /// `ExprKind::Int` which the engines evaluate as
25    /// `Value::Int`.
26    Int(i64),
27    /// The sole unsigned integer magnitude that does not fit in
28    /// `i64` but can form `i64::MIN` when immediately negated.
29    /// The parser must consume this sentinel; it never reaches
30    /// the AST or an execution engine.
31    IntMinMagnitude,
32    Number(f64),
33    Str(String),
34    StringInterp(Vec<StringPart>),
35    True,
36    False,
37    None,
38
39    // Identifiers & Keywords
40    Ident(String),
41    Let,
42    Const,
43    Fn,
44    Pub,
45    Return,
46    If,
47    Else,
48    While,
49    For,
50    In,
51    Repeat,
52    Break,
53    Continue,
54    Use,
55    As,
56    Struct,
57    Enum,
58    Match,
59    Try,
60    Ref,
61
62    // Operators
63    Plus,
64    Minus,
65    Star,
66    Slash,
67    Percent,
68    EqEq,
69    BangEq,
70    Lt,
71    Gt,
72    LtEq,
73    GtEq,
74    AmpAmp,
75    PipePipe,
76    Bang,
77    Eq,
78    PlusEq,
79    MinusEq,
80    StarEq,
81    SlashEq,
82    PercentEq,
83
84    // Delimiters
85    LParen,
86    RParen,
87    LBracket,
88    RBracket,
89    LBrace,
90    RBrace,
91    Comma,
92    Colon,
93    ColonColon,
94    Dot,
95    DotDot,
96    Semicolon,
97    FatArrow,
98    Pipe,
99
100    // Internal (removed after auto-semicolons)
101    Newline,
102
103    Eof,
104}
105
106// Define keyword spelling once and derive both lexer recognition and
107// diagnostic metadata from it. Keeping these directions in one declaration
108// prevents consumers such as the parser and public precheck API from growing
109// hand-maintained keyword lists that drift from the lexer.
110macro_rules! define_keywords {
111    ($($spelling:literal => $variant:ident),+ $(,)?) => {
112        impl Token {
113            pub(crate) fn from_keyword(value: &str) -> Option<Self> {
114                match value {
115                    $($spelling => Some(Self::$variant),)+
116                    _ => None,
117                }
118            }
119
120            pub(crate) fn keyword_name(&self) -> Option<&'static str> {
121                match self {
122                    $(Self::$variant => Some($spelling),)+
123                    _ => None,
124                }
125            }
126        }
127
128        #[cfg(test)]
129        pub(crate) const KEYWORD_NAMES: &[&str] = &[$($spelling),+];
130    };
131}
132
133define_keywords! {
134    "let" => Let,
135    "const" => Const,
136    "fn" => Fn,
137    "pub" => Pub,
138    "return" => Return,
139    "if" => If,
140    "else" => Else,
141    "while" => While,
142    "for" => For,
143    "in" => In,
144    "repeat" => Repeat,
145    "break" => Break,
146    "continue" => Continue,
147    "use" => Use,
148    "as" => As,
149    "struct" => Struct,
150    "enum" => Enum,
151    "match" => Match,
152    "try" => Try,
153    "ref" => Ref,
154    "true" => True,
155    "false" => False,
156    "none" => None,
157}
158
159#[derive(Debug, Clone)]
160pub struct SpannedToken {
161    pub token: Token,
162    pub line: u32,
163    /// 1-indexed column where the token starts. Used by the
164    /// parser and runtime to point error carets at the exact
165    /// offending character rather than just the line.
166    pub column: u32,
167}
168
169/// Steps a module load costs: one per lexer token, excluding end of input.
170/// Every engine charges it once per load, after lexing and before parsing.
171pub fn module_load_steps(tokens: &[SpannedToken]) -> u64 {
172    tokens
173        .iter()
174        .filter(|token| !matches!(token.token, Token::Eof))
175        .count() as u64
176}
177
178pub fn lex(source: &str) -> Result<Vec<SpannedToken>, NyblError> {
179    let mut lexer = Lexer::new(source);
180    let raw = lexer.lex_all()?;
181    Ok(insert_semicolons(raw))
182}
183
184fn triggers_semicolon(token: &Token) -> bool {
185    matches!(
186        token,
187        Token::Ident(_)
188            | Token::Int(_)
189            | Token::IntMinMagnitude
190            | Token::Number(_)
191            | Token::Str(_)
192            | Token::StringInterp(_)
193            | Token::True
194            | Token::False
195            | Token::None
196            | Token::Break
197            | Token::Continue
198            | Token::Return
199            | Token::RParen
200            | Token::RBracket
201            | Token::RBrace
202    )
203}
204
205fn supports_dot_continuation(token: &Token) -> bool {
206    matches!(
207        token,
208        Token::Ident(_)
209            | Token::Int(_)
210            | Token::IntMinMagnitude
211            | Token::Number(_)
212            | Token::Str(_)
213            | Token::StringInterp(_)
214            | Token::True
215            | Token::False
216            | Token::None
217            | Token::RParen
218            | Token::RBracket
219            | Token::RBrace
220    )
221}
222
223#[derive(Clone, Copy, PartialEq, Eq)]
224enum Delimiter {
225    Paren,
226    Bracket,
227    Brace,
228}
229
230fn insert_semicolons(raw: Vec<SpannedToken>) -> Vec<SpannedToken> {
231    let mut result: Vec<SpannedToken> = Vec::new();
232    let mut delimiters: Vec<Delimiter> = Vec::new();
233    let mut tokens = raw.into_iter().peekable();
234    while let Some(token) = tokens.next() {
235        if token.token == Token::Newline {
236            // Coalesce the run so next-token lookahead stays linear
237            // even across large blocks of blank lines.
238            while tokens
239                .peek()
240                .is_some_and(|next| next.token == Token::Newline)
241            {
242                tokens.next();
243            }
244            if let Some(last) = result.last() {
245                // The innermost delimiter decides: a function block
246                // inside an array still needs separators between its
247                // statements even though the outer bracket is open.
248                let inside_soft_delimiter = matches!(
249                    delimiters.last(),
250                    Some(Delimiter::Paren | Delimiter::Bracket)
251                );
252                let next = tokens.peek().map(|next| &next.token);
253                let before_closing_delimiter =
254                    matches!(next, Some(Token::RParen | Token::RBracket | Token::RBrace));
255                // A newline before `else` is branch layout, not a statement
256                // boundary. Suppressing its synthetic semicolon here keeps an
257                // explicit `;` distinguishable (and therefore invalid) at the
258                // same parser boundary.
259                let before_else =
260                    matches!(last.token, Token::RBrace) && matches!(next, Some(Token::Else));
261                let before_dot_continuation =
262                    matches!(next, Some(Token::Dot)) && supports_dot_continuation(&last.token);
263                if !inside_soft_delimiter
264                    && !before_closing_delimiter
265                    && !before_else
266                    && !before_dot_continuation
267                    && triggers_semicolon(&last.token)
268                {
269                    result.push(SpannedToken {
270                        token: Token::Semicolon,
271                        line: token.line,
272                        column: token.column,
273                    });
274                }
275            }
276        } else {
277            match &token.token {
278                Token::LParen => delimiters.push(Delimiter::Paren),
279                Token::LBracket => delimiters.push(Delimiter::Bracket),
280                Token::LBrace => delimiters.push(Delimiter::Brace),
281                Token::RParen if delimiters.last() == Some(&Delimiter::Paren) => {
282                    delimiters.pop();
283                }
284                Token::RBracket if delimiters.last() == Some(&Delimiter::Bracket) => {
285                    delimiters.pop();
286                }
287                Token::RBrace if delimiters.last() == Some(&Delimiter::Brace) => {
288                    delimiters.pop();
289                }
290                _ => {}
291            }
292            result.push(token);
293        }
294    }
295    result
296}
297
298/// A zero-copy cursor over UTF-8 source text.
299///
300/// `byte_pos` is always kept on a character boundary by [`advance`], while
301/// callers continue to count source columns in characters rather than bytes.
302struct SourceCursor<'source> {
303    source: &'source str,
304    chars: core::str::CharIndices<'source>,
305    current: Option<(usize, char)>,
306    next: Option<(usize, char)>,
307    byte_pos: usize,
308}
309
310impl<'source> SourceCursor<'source> {
311    fn new(source: &'source str) -> Self {
312        let mut chars = source.char_indices();
313        let current = chars.next();
314        let next = chars.next();
315        Self {
316            source,
317            chars,
318            current,
319            next,
320            byte_pos: 0,
321        }
322    }
323
324    fn peek(&self) -> Option<char> {
325        self.current.map(|(_, ch)| ch)
326    }
327
328    fn peek_next(&self) -> Option<char> {
329        self.next.map(|(_, ch)| ch)
330    }
331
332    fn advance(&mut self) -> Option<char> {
333        let (_, ch) = self.current?;
334        self.current = self.next;
335        self.next = self.chars.next();
336        self.byte_pos = self
337            .current
338            .map(|(byte_pos, _)| byte_pos)
339            .unwrap_or(self.source.len());
340        Some(ch)
341    }
342}
343
344struct Lexer<'source> {
345    cursor: SourceCursor<'source>,
346    line: u32,
347    /// 1-indexed column of the *next* character to consume.
348    /// Reset to 1 after each newline; incremented by `advance`.
349    column: u32,
350}
351
352impl<'source> Lexer<'source> {
353    fn new(source: &'source str) -> Self {
354        Self {
355            cursor: SourceCursor::new(source),
356            line: 1,
357            column: 1,
358        }
359    }
360
361    fn peek(&self) -> Option<char> {
362        self.cursor.peek()
363    }
364
365    fn peek_next(&self) -> Option<char> {
366        self.cursor.peek_next()
367    }
368
369    fn advance(&mut self) -> Option<char> {
370        let ch = self.cursor.advance()?;
371        if ch == '\n' {
372            // The newline itself belongs to the line it
373            // terminates; the next character starts at column 1
374            // of the *following* line. `line` gets bumped by the
375            // lexer's dispatch loop when it sees `\n`, so we
376            // only reset the column here.
377            self.column = 1;
378        } else {
379            self.column += 1;
380        }
381        Some(ch)
382    }
383
384    fn error(&self, message: impl Into<String>) -> NyblError {
385        NyblError {
386            line: Some(self.line),
387            column: Some(self.column),
388            message: message.into(),
389            friendly_hint: None,
390            source_context: None,
391            is_fatal: false,
392            is_try_return: false,
393        }
394    }
395
396    fn error_at(&self, line: u32, column: u32, message: impl Into<String>) -> NyblError {
397        NyblError {
398            line: Some(line),
399            column: Some(column),
400            message: message.into(),
401            friendly_hint: None,
402            source_context: None,
403            is_fatal: false,
404            is_try_return: false,
405        }
406    }
407
408    fn error_with_hint(&self, message: impl Into<String>, hint: impl Into<String>) -> NyblError {
409        NyblError {
410            line: Some(self.line),
411            column: Some(self.column),
412            message: message.into(),
413            friendly_hint: Some(hint.into()),
414            source_context: None,
415            is_fatal: false,
416            is_try_return: false,
417        }
418    }
419
420    fn lex_all(&mut self) -> Result<Vec<SpannedToken>, NyblError> {
421        let mut tokens = Vec::new();
422
423        loop {
424            // Skip whitespace (not newlines)
425            while let Some(ch) = self.peek() {
426                if ch == ' ' || ch == '\t' || ch == '\r' {
427                    self.advance();
428                } else {
429                    break;
430                }
431            }
432
433            let Some(ch) = self.peek() else {
434                tokens.push(SpannedToken {
435                    token: Token::Eof,
436                    line: self.line,
437                    column: self.column,
438                });
439                break;
440            };
441
442            // Capture the token's start position before we start
443            // consuming characters — `self.line` / `self.column`
444            // move as we advance.
445            let line = self.line;
446            let column = self.column;
447
448            match ch {
449                '\n' => {
450                    self.advance();
451                    self.line += 1;
452                    tokens.push(SpannedToken {
453                        token: Token::Newline,
454                        line,
455                        column,
456                    });
457                }
458
459                '"' => {
460                    tokens.push(SpannedToken {
461                        token: self.lex_string()?,
462                        line,
463                        column,
464                    });
465                }
466
467                '0'..='9' => {
468                    tokens.push(SpannedToken {
469                        token: self.lex_number(line, column)?,
470                        line,
471                        column,
472                    });
473                }
474
475                'a'..='z' | 'A'..='Z' | '_' => {
476                    tokens.push(SpannedToken {
477                        token: self.lex_ident_or_keyword(),
478                        line,
479                        column,
480                    });
481                }
482
483                '+' => {
484                    self.advance();
485                    if self.peek() == Some('=') {
486                        self.advance();
487                        tokens.push(SpannedToken {
488                            token: Token::PlusEq,
489                            line,
490                            column,
491                        });
492                    } else {
493                        tokens.push(SpannedToken {
494                            token: Token::Plus,
495                            line,
496                            column,
497                        });
498                    }
499                }
500                '-' => {
501                    self.advance();
502                    if self.peek() == Some('=') {
503                        self.advance();
504                        tokens.push(SpannedToken {
505                            token: Token::MinusEq,
506                            line,
507                            column,
508                        });
509                    } else {
510                        tokens.push(SpannedToken {
511                            token: Token::Minus,
512                            line,
513                            column,
514                        });
515                    }
516                }
517                '*' => {
518                    self.advance();
519                    if self.peek() == Some('=') {
520                        self.advance();
521                        tokens.push(SpannedToken {
522                            token: Token::StarEq,
523                            line,
524                            column,
525                        });
526                    } else {
527                        tokens.push(SpannedToken {
528                            token: Token::Star,
529                            line,
530                            column,
531                        });
532                    }
533                }
534                '/' => {
535                    self.advance();
536                    if self.peek() == Some('=') {
537                        self.advance();
538                        tokens.push(SpannedToken {
539                            token: Token::SlashEq,
540                            line,
541                            column,
542                        });
543                    } else if self.peek() == Some('/') {
544                        // `//` — line comment. Runs to end of
545                        // line; no block-comment form. (There's
546                        // no integer-division operator: `/`
547                        // always returns a `Number` even for
548                        // `Int / Int`, and users who want an
549                        // integer result write `int(a / b)`.)
550                        self.advance();
551                        while let Some(c) = self.peek() {
552                            if c == '\n' {
553                                break;
554                            }
555                            self.advance();
556                        }
557                    } else {
558                        tokens.push(SpannedToken {
559                            token: Token::Slash,
560                            line,
561                            column,
562                        });
563                    }
564                }
565                '%' => {
566                    self.advance();
567                    if self.peek() == Some('=') {
568                        self.advance();
569                        tokens.push(SpannedToken {
570                            token: Token::PercentEq,
571                            line,
572                            column,
573                        });
574                    } else {
575                        tokens.push(SpannedToken {
576                            token: Token::Percent,
577                            line,
578                            column,
579                        });
580                    }
581                }
582
583                '=' => {
584                    self.advance();
585                    if self.peek() == Some('=') {
586                        self.advance();
587                        tokens.push(SpannedToken {
588                            token: Token::EqEq,
589                            line,
590                            column,
591                        });
592                    } else if self.peek() == Some('>') {
593                        self.advance();
594                        tokens.push(SpannedToken {
595                            token: Token::FatArrow,
596                            line,
597                            column,
598                        });
599                    } else {
600                        tokens.push(SpannedToken {
601                            token: Token::Eq,
602                            line,
603                            column,
604                        });
605                    }
606                }
607                '!' => {
608                    self.advance();
609                    if self.peek() == Some('=') {
610                        self.advance();
611                        tokens.push(SpannedToken {
612                            token: Token::BangEq,
613                            line,
614                            column,
615                        });
616                    } else {
617                        tokens.push(SpannedToken {
618                            token: Token::Bang,
619                            line,
620                            column,
621                        });
622                    }
623                }
624                '<' => {
625                    self.advance();
626                    if self.peek() == Some('=') {
627                        self.advance();
628                        tokens.push(SpannedToken {
629                            token: Token::LtEq,
630                            line,
631                            column,
632                        });
633                    } else {
634                        tokens.push(SpannedToken {
635                            token: Token::Lt,
636                            line,
637                            column,
638                        });
639                    }
640                }
641                '>' => {
642                    self.advance();
643                    if self.peek() == Some('=') {
644                        self.advance();
645                        tokens.push(SpannedToken {
646                            token: Token::GtEq,
647                            line,
648                            column,
649                        });
650                    } else {
651                        tokens.push(SpannedToken {
652                            token: Token::Gt,
653                            line,
654                            column,
655                        });
656                    }
657                }
658
659                '&' => {
660                    self.advance();
661                    if self.peek() == Some('&') {
662                        self.advance();
663                        tokens.push(SpannedToken {
664                            token: Token::AmpAmp,
665                            line,
666                            column,
667                        });
668                    } else {
669                        return Err(
670                            self.error_with_hint("Unexpected `&`", "Did you mean `&&` (and)?")
671                        );
672                    }
673                }
674                '|' => {
675                    self.advance();
676                    if self.peek() == Some('|') {
677                        self.advance();
678                        tokens.push(SpannedToken {
679                            token: Token::PipePipe,
680                            line,
681                            column,
682                        });
683                    } else {
684                        // Single `|` is now the or-pattern
685                        // separator inside `match` arms. Parser
686                        // decides whether it's accepted in the
687                        // current context.
688                        tokens.push(SpannedToken {
689                            token: Token::Pipe,
690                            line,
691                            column,
692                        });
693                    }
694                }
695
696                '(' => {
697                    self.advance();
698                    tokens.push(SpannedToken {
699                        token: Token::LParen,
700                        line,
701                        column,
702                    });
703                }
704                ')' => {
705                    self.advance();
706                    tokens.push(SpannedToken {
707                        token: Token::RParen,
708                        line,
709                        column,
710                    });
711                }
712                '[' => {
713                    self.advance();
714                    tokens.push(SpannedToken {
715                        token: Token::LBracket,
716                        line,
717                        column,
718                    });
719                }
720                ']' => {
721                    self.advance();
722                    tokens.push(SpannedToken {
723                        token: Token::RBracket,
724                        line,
725                        column,
726                    });
727                }
728                '{' => {
729                    self.advance();
730                    tokens.push(SpannedToken {
731                        token: Token::LBrace,
732                        line,
733                        column,
734                    });
735                }
736                '}' => {
737                    self.advance();
738                    tokens.push(SpannedToken {
739                        token: Token::RBrace,
740                        line,
741                        column,
742                    });
743                }
744                ',' => {
745                    self.advance();
746                    tokens.push(SpannedToken {
747                        token: Token::Comma,
748                        line,
749                        column,
750                    });
751                }
752                ':' => {
753                    self.advance();
754                    if self.peek() == Some(':') {
755                        self.advance();
756                        tokens.push(SpannedToken {
757                            token: Token::ColonColon,
758                            line,
759                            column,
760                        });
761                    } else {
762                        tokens.push(SpannedToken {
763                            token: Token::Colon,
764                            line,
765                            column,
766                        });
767                    }
768                }
769                '.' => {
770                    self.advance();
771                    if self.peek() == Some('.') {
772                        self.advance();
773                        tokens.push(SpannedToken {
774                            token: Token::DotDot,
775                            line,
776                            column,
777                        });
778                    } else {
779                        tokens.push(SpannedToken {
780                            token: Token::Dot,
781                            line,
782                            column,
783                        });
784                    }
785                }
786                ';' => {
787                    self.advance();
788                    tokens.push(SpannedToken {
789                        token: Token::Semicolon,
790                        line,
791                        column,
792                    });
793                }
794
795                _ => {
796                    return Err(self.error(format!("I don't understand the character `{ch}`")));
797                }
798            }
799        }
800
801        Ok(tokens)
802    }
803
804    fn lex_number(&mut self, line: u32, column: u32) -> Result<Token, NyblError> {
805        let mut s = String::new();
806        while let Some(ch) = self.peek() {
807            if ch.is_ascii_digit() {
808                s.push(ch);
809                self.advance();
810            } else {
811                break;
812            }
813        }
814        // A trailing `.<digit>` promotes to a float; `42..foo`
815        // or `42.` at EOF stays an Int so chained method calls
816        // (`42.str()`) and inclusive array-rest patterns still
817        // parse.
818        let is_float =
819            if self.peek() == Some('.') && self.peek_next().is_some_and(|c| c.is_ascii_digit()) {
820                s.push('.');
821                self.advance();
822                while let Some(ch) = self.peek() {
823                    if ch.is_ascii_digit() {
824                        s.push(ch);
825                        self.advance();
826                    } else {
827                        break;
828                    }
829                }
830                true
831            } else {
832                false
833            };
834        if is_float {
835            let n: f64 = s
836                .parse()
837                .map_err(|_| self.error_at(line, column, format!("Invalid number: {s}")))?;
838            Ok(Token::Number(n))
839        } else {
840            // Preserve one otherwise-unrepresentable magnitude
841            // for the parser to fold only under unary `-`. Every
842            // other out-of-range value still fails here rather
843            // than wrapping or degrading to `f64`.
844            match s.parse::<u64>() {
845                Ok(n) if n <= i64::MAX as u64 => Ok(Token::Int(n as i64)),
846                Ok(I64_MIN_MAGNITUDE) => Ok(Token::IntMinMagnitude),
847                _ => Err(self.error_at(line, column, integer_literal_out_of_range(&s))),
848            }
849        }
850    }
851
852    fn lex_ident_or_keyword(&mut self) -> Token {
853        let mut s = String::new();
854        while let Some(ch) = self.peek() {
855            if ch.is_ascii_alphanumeric() || ch == '_' {
856                s.push(ch);
857                self.advance();
858            } else {
859                break;
860            }
861        }
862        Token::from_keyword(&s).unwrap_or(Token::Ident(s))
863    }
864
865    fn lex_string(&mut self) -> Result<Token, NyblError> {
866        self.advance(); // consume opening "
867        let mut parts: Vec<StringPart> = Vec::new();
868        let mut current = String::new();
869
870        loop {
871            match self.peek() {
872                None | Some('\n') => {
873                    return Err(self.error_with_hint(
874                        "This string is missing its closing `\"`",
875                        "Every string needs to start and end with quotes.",
876                    ));
877                }
878                Some('"') => {
879                    self.advance();
880                    break;
881                }
882                Some('\\') => {
883                    self.advance();
884                    match self.peek() {
885                        Some('"') => {
886                            current.push('"');
887                            self.advance();
888                        }
889                        Some('\\') => {
890                            current.push('\\');
891                            self.advance();
892                        }
893                        Some('n') => {
894                            current.push('\n');
895                            self.advance();
896                        }
897                        Some('t') => {
898                            current.push('\t');
899                            self.advance();
900                        }
901                        Some('r') => {
902                            // Windows / HTTP line endings; also
903                            // needed so `std.json` can stringify
904                            // and parse carriage returns inside
905                            // strings.
906                            current.push('\r');
907                            self.advance();
908                        }
909                        Some('{') => {
910                            current.push('{');
911                            self.advance();
912                        }
913                        Some('}') => {
914                            current.push('}');
915                            self.advance();
916                        }
917                        Some(c) => {
918                            return Err(self.error(format!("Unknown escape sequence `\\{c}`")));
919                        }
920                        None => {
921                            return Err(self.error("Unexpected end of string after `\\`"));
922                        }
923                    }
924                }
925                Some('{')
926                    if self
927                        .peek_next()
928                        .is_some_and(|c| c.is_ascii_alphabetic() || c == '_') =>
929                {
930                    self.advance(); // consume {
931                    // Read variable name
932                    let mut var = String::new();
933                    while let Some(ch) = self.peek() {
934                        if ch.is_ascii_alphanumeric() || ch == '_' {
935                            var.push(ch);
936                            self.advance();
937                        } else {
938                            break;
939                        }
940                    }
941                    if self.peek() != Some('}') {
942                        return Err(self.error_with_hint(
943                            format!("Missing `}}` after `{{{var}`"),
944                            "String interpolation needs a closing `}`, like: \"{name}\"",
945                        ));
946                    }
947                    self.advance(); // consume }
948                    if !current.is_empty() {
949                        parts.push(StringPart::Literal(core::mem::take(&mut current)));
950                    }
951                    parts.push(StringPart::Variable(var));
952                }
953                Some(ch) => {
954                    current.push(ch);
955                    self.advance();
956                }
957            }
958        }
959
960        if parts.is_empty() {
961            // Plain string, no interpolation
962            Ok(Token::Str(current))
963        } else {
964            if !current.is_empty() {
965                parts.push(StringPart::Literal(current));
966            }
967            Ok(Token::StringInterp(parts))
968        }
969    }
970}
971
972#[cfg(test)]
973mod tests {
974    use super::*;
975
976    /// Lex and strip Eof, returning just token variants
977    fn toks(code: &str) -> Vec<Token> {
978        lex(code)
979            .unwrap()
980            .into_iter()
981            .map(|t| t.token)
982            .filter(|t| !matches!(t, Token::Eof))
983            .collect()
984    }
985
986    fn lex_err(code: &str) -> String {
987        lex(code).unwrap_err().message
988    }
989
990    #[test]
991    fn source_cursor_borrows_utf8_and_advances_on_character_boundaries() {
992        let source = "é🙂x";
993        let mut lexer = Lexer::new(source);
994
995        assert_eq!(lexer.cursor.source.as_ptr(), source.as_ptr());
996        assert_eq!(lexer.cursor.byte_pos, 0);
997
998        assert_eq!(lexer.advance(), Some('é'));
999        assert_eq!(lexer.cursor.byte_pos, 'é'.len_utf8());
1000        assert_eq!(lexer.column, 2);
1001
1002        assert_eq!(lexer.advance(), Some('🙂'));
1003        assert_eq!(lexer.cursor.byte_pos, 'é'.len_utf8() + '🙂'.len_utf8());
1004        assert_eq!(lexer.column, 3);
1005        assert_eq!(lexer.peek(), Some('x'));
1006    }
1007
1008    #[test]
1009    fn utf8_string_contents_preserve_character_based_token_columns() {
1010        let tokens = lex("\"é🙂\" next").expect("source should lex");
1011        assert!(matches!(&tokens[0].token, Token::Str(value) if value == "é🙂"));
1012        assert_eq!((tokens[0].line, tokens[0].column), (1, 1));
1013        assert!(matches!(&tokens[1].token, Token::Ident(name) if name == "next"));
1014        assert_eq!((tokens[1].line, tokens[1].column), (1, 6));
1015    }
1016
1017    // ─── Numbers ───────────────────────────────────────────────────
1018
1019    #[test]
1020    fn integer() {
1021        // Integer literals now lex to `Token::Int` (phase 6).
1022        assert_eq!(toks("42"), vec![Token::Int(42)]);
1023    }
1024
1025    #[test]
1026    fn i64_min_magnitude_is_preserved_for_unary_minus_parsing() {
1027        assert_eq!(toks(I64_MIN_MAGNITUDE_TEXT), vec![Token::IntMinMagnitude]);
1028    }
1029
1030    #[test]
1031    fn integer_beyond_i64_min_magnitude_errors_at_the_literal_start() {
1032        let error = lex("\n  9223372036854775809").expect_err("magnitude must be rejected");
1033        assert_eq!(error.line, Some(2));
1034        assert_eq!(error.column, Some(3));
1035        assert_eq!(
1036            error.message,
1037            "Integer literal out of range for i64: 9223372036854775809"
1038        );
1039    }
1040
1041    #[test]
1042    fn float() {
1043        assert_eq!(toks("3.125"), vec![Token::Number(3.125)]);
1044    }
1045
1046    #[test]
1047    fn leading_zero_float() {
1048        assert_eq!(toks("0.5"), vec![Token::Number(0.5)]);
1049    }
1050
1051    // ─── Strings ───────────────────────────────────────────────────
1052
1053    #[test]
1054    fn plain_string() {
1055        assert_eq!(toks(r#""hello""#), vec![Token::Str("hello".into())]);
1056    }
1057
1058    #[test]
1059    fn escape_sequences() {
1060        assert_eq!(
1061            toks(r#""a\nb\t\\\"c""#),
1062            vec![Token::Str("a\nb\t\\\"c".into())]
1063        );
1064    }
1065
1066    #[test]
1067    fn escape_sequence_cr() {
1068        // `\r` was added alongside `std.json` so carriage
1069        // returns can live in string literals (Windows line
1070        // endings, HTTP headers, etc.).
1071        assert_eq!(toks(r#""a\rb""#), vec![Token::Str("a\rb".into())]);
1072    }
1073
1074    #[test]
1075    fn string_interpolation() {
1076        assert_eq!(
1077            toks(r#""hi {name}!""#),
1078            vec![Token::StringInterp(vec![
1079                StringPart::Literal("hi ".into()),
1080                StringPart::Variable("name".into()),
1081                StringPart::Literal("!".into()),
1082            ])]
1083        );
1084    }
1085
1086    #[test]
1087    fn string_interpolation_multiple_vars() {
1088        assert_eq!(
1089            toks(r#""{x},{y}""#),
1090            vec![Token::StringInterp(vec![
1091                StringPart::Variable("x".into()),
1092                StringPart::Literal(",".into()),
1093                StringPart::Variable("y".into()),
1094            ])]
1095        );
1096    }
1097
1098    #[test]
1099    fn unterminated_string() {
1100        assert!(lex_err(r#""hello"#).contains("missing its closing"));
1101    }
1102
1103    #[test]
1104    fn unknown_escape() {
1105        assert!(lex_err(r#""hello\q""#).contains("Unknown escape"));
1106    }
1107
1108    // ─── Keywords vs Identifiers ───────────────────────────────────
1109
1110    #[test]
1111    fn keywords() {
1112        assert_eq!(
1113            toks("let fn return if else while for in repeat break continue true false none"),
1114            vec![
1115                Token::Let,
1116                Token::Fn,
1117                Token::Return,
1118                Token::If,
1119                Token::Else,
1120                Token::While,
1121                Token::For,
1122                Token::In,
1123                Token::Repeat,
1124                Token::Break,
1125                Token::Continue,
1126                Token::True,
1127                Token::False,
1128                Token::None,
1129            ]
1130        );
1131    }
1132
1133    #[test]
1134    fn keyword_metadata_round_trips_the_lexer_vocabulary() {
1135        assert!(KEYWORD_NAMES.contains(&"const"));
1136
1137        for &name in KEYWORD_NAMES {
1138            let token = Token::from_keyword(name).expect("declared keyword should lex");
1139            assert_eq!(token.keyword_name(), Some(name));
1140            assert_eq!(toks(name), vec![token]);
1141        }
1142    }
1143
1144    #[test]
1145    fn identifiers() {
1146        assert_eq!(
1147            toks("foo bar_baz _x abc123"),
1148            vec![
1149                Token::Ident("foo".into()),
1150                Token::Ident("bar_baz".into()),
1151                Token::Ident("_x".into()),
1152                Token::Ident("abc123".into()),
1153            ]
1154        );
1155    }
1156
1157    // ─── Operators ─────────────────────────────────────────────────
1158
1159    #[test]
1160    fn single_char_ops() {
1161        assert_eq!(
1162            toks("+ - * / % = ! < > ( ) [ ] { } , : . ;"),
1163            vec![
1164                Token::Plus,
1165                Token::Minus,
1166                Token::Star,
1167                Token::Slash,
1168                Token::Percent,
1169                Token::Eq,
1170                Token::Bang,
1171                Token::Lt,
1172                Token::Gt,
1173                Token::LParen,
1174                Token::RParen,
1175                Token::LBracket,
1176                Token::RBracket,
1177                Token::LBrace,
1178                Token::RBrace,
1179                Token::Comma,
1180                Token::Colon,
1181                Token::Dot,
1182                Token::Semicolon,
1183            ]
1184        );
1185    }
1186
1187    #[test]
1188    fn double_char_ops() {
1189        assert_eq!(
1190            toks("== != <= >= && || += -= *= /= %="),
1191            vec![
1192                Token::EqEq,
1193                Token::BangEq,
1194                Token::LtEq,
1195                Token::GtEq,
1196                Token::AmpAmp,
1197                Token::PipePipe,
1198                Token::PlusEq,
1199                Token::MinusEq,
1200                Token::StarEq,
1201                Token::SlashEq,
1202                Token::PercentEq,
1203            ]
1204        );
1205    }
1206
1207    #[test]
1208    fn lone_ampersand_error() {
1209        assert!(lex_err("&x").contains("Unexpected `&`"));
1210    }
1211
1212    #[test]
1213    fn lone_pipe_lexes_as_or_pattern_separator() {
1214        // `|` is now the or-pattern separator inside `match`
1215        // arms. It parses at the lexer level regardless of
1216        // context; the parser decides whether it's accepted.
1217        assert_eq!(toks("|"), vec![Token::Pipe]);
1218    }
1219
1220    // ─── Comments ──────────────────────────────────────────────────
1221
1222    #[test]
1223    fn line_comment_skipped() {
1224        assert_eq!(
1225            toks("1 // comment\n2"),
1226            vec![Token::Int(1), Token::Semicolon, Token::Int(2)]
1227        );
1228    }
1229
1230    #[test]
1231    fn comment_at_end() {
1232        assert_eq!(toks("x // done"), vec![Token::Ident("x".into())]);
1233    }
1234
1235    #[test]
1236    fn hash_is_not_a_comment() {
1237        // `#` used to be the line-comment token; comments are
1238        // now `//` and `#` is just an unrecognised character.
1239        assert!(lex_err("x # nope").contains("don't understand"));
1240    }
1241
1242    // ─── Auto-semicolons ──────────────────────────────────────────
1243
1244    #[test]
1245    fn auto_semi_after_ident() {
1246        assert_eq!(
1247            toks("x\ny"),
1248            vec![
1249                Token::Ident("x".into()),
1250                Token::Semicolon,
1251                Token::Ident("y".into()),
1252            ]
1253        );
1254    }
1255
1256    #[test]
1257    fn auto_semi_after_number() {
1258        assert_eq!(
1259            toks("42\n10"),
1260            vec![Token::Int(42), Token::Semicolon, Token::Int(10)]
1261        );
1262    }
1263
1264    #[test]
1265    fn auto_semi_after_rparen() {
1266        assert_eq!(
1267            toks("f()\ng()"),
1268            vec![
1269                Token::Ident("f".into()),
1270                Token::LParen,
1271                Token::RParen,
1272                Token::Semicolon,
1273                Token::Ident("g".into()),
1274                Token::LParen,
1275                Token::RParen,
1276            ]
1277        );
1278    }
1279
1280    #[test]
1281    fn auto_semi_after_rbrace() {
1282        assert_eq!(
1283            toks("{\n}\nx"),
1284            vec![
1285                Token::LBrace,
1286                Token::RBrace,
1287                Token::Semicolon,
1288                Token::Ident("x".into()),
1289            ]
1290        );
1291    }
1292
1293    #[test]
1294    fn no_semi_after_open_delim() {
1295        assert_eq!(toks("{\nx"), vec![Token::LBrace, Token::Ident("x".into()),]);
1296    }
1297
1298    #[test]
1299    fn no_semi_after_operator() {
1300        assert_eq!(
1301            toks("x +\ny"),
1302            vec![
1303                Token::Ident("x".into()),
1304                Token::Plus,
1305                Token::Ident("y".into()),
1306            ]
1307        );
1308    }
1309
1310    #[test]
1311    fn auto_semi_after_break_continue_return() {
1312        assert_eq!(
1313            toks("break\ncontinue\nreturn"),
1314            vec![
1315                Token::Break,
1316                Token::Semicolon,
1317                Token::Continue,
1318                Token::Semicolon,
1319                Token::Return,
1320            ]
1321        );
1322    }
1323
1324    #[test]
1325    fn auto_semi_after_true_false_none() {
1326        assert_eq!(
1327            toks("true\nfalse\nnone"),
1328            vec![
1329                Token::True,
1330                Token::Semicolon,
1331                Token::False,
1332                Token::Semicolon,
1333                Token::None,
1334            ]
1335        );
1336    }
1337
1338    #[test]
1339    fn no_auto_semi_inside_parentheses_or_brackets() {
1340        assert_eq!(
1341            toks("[\n1,\n2\n]\nf(\n3,\n4\n)"),
1342            vec![
1343                Token::LBracket,
1344                Token::Int(1),
1345                Token::Comma,
1346                Token::Int(2),
1347                Token::RBracket,
1348                Token::Semicolon,
1349                Token::Ident("f".into()),
1350                Token::LParen,
1351                Token::Int(3),
1352                Token::Comma,
1353                Token::Int(4),
1354                Token::RParen,
1355            ]
1356        );
1357    }
1358
1359    #[test]
1360    fn braces_inside_parentheses_keep_statement_separation() {
1361        assert_eq!(
1362            toks("(fn() {\nx\ny\n})"),
1363            vec![
1364                Token::LParen,
1365                Token::Fn,
1366                Token::LParen,
1367                Token::RParen,
1368                Token::LBrace,
1369                Token::Ident("x".into()),
1370                Token::Semicolon,
1371                Token::Ident("y".into()),
1372                Token::RBrace,
1373                Token::RParen,
1374            ]
1375        );
1376    }
1377
1378    #[test]
1379    fn braces_inside_brackets_keep_statement_separation() {
1380        assert_eq!(
1381            toks("[fn() {\nx\ny\n}]"),
1382            vec![
1383                Token::LBracket,
1384                Token::Fn,
1385                Token::LParen,
1386                Token::RParen,
1387                Token::LBrace,
1388                Token::Ident("x".into()),
1389                Token::Semicolon,
1390                Token::Ident("y".into()),
1391                Token::RBrace,
1392                Token::RBracket,
1393            ]
1394        );
1395    }
1396
1397    #[test]
1398    fn newline_before_closing_delimiter_does_not_insert_semicolon() {
1399        assert_eq!(
1400            toks("{\nx\ny\n}"),
1401            vec![
1402                Token::LBrace,
1403                Token::Ident("x".into()),
1404                Token::Semicolon,
1405                Token::Ident("y".into()),
1406                Token::RBrace,
1407            ]
1408        );
1409    }
1410
1411    #[test]
1412    fn newline_before_else_does_not_insert_semicolon() {
1413        assert_eq!(
1414            toks("if true { 1 }\nelse { 2 }"),
1415            vec![
1416                Token::If,
1417                Token::True,
1418                Token::LBrace,
1419                Token::Int(1),
1420                Token::RBrace,
1421                Token::Else,
1422                Token::LBrace,
1423                Token::Int(2),
1424                Token::RBrace,
1425            ]
1426        );
1427
1428        assert_eq!(
1429            toks("if true { 1 };\nelse { 2 }"),
1430            vec![
1431                Token::If,
1432                Token::True,
1433                Token::LBrace,
1434                Token::Int(1),
1435                Token::RBrace,
1436                Token::Semicolon,
1437                Token::Else,
1438                Token::LBrace,
1439                Token::Int(2),
1440                Token::RBrace,
1441            ]
1442        );
1443    }
1444
1445    #[test]
1446    fn leading_dot_continues_a_value_across_comments_and_blank_lines() {
1447        assert_eq!(
1448            toks("value\n// explain the chain\n\n.len()"),
1449            vec![
1450                Token::Ident("value".into()),
1451                Token::Dot,
1452                Token::Ident("len".into()),
1453                Token::LParen,
1454                Token::RParen,
1455            ]
1456        );
1457    }
1458
1459    #[test]
1460    fn return_newline_remains_a_bare_return() {
1461        assert_eq!(
1462            toks("return\nvalue"),
1463            vec![
1464                Token::Return,
1465                Token::Semicolon,
1466                Token::Ident("value".into()),
1467            ]
1468        );
1469    }
1470
1471    #[test]
1472    fn delimiter_characters_inside_strings_do_not_affect_asi() {
1473        assert_eq!(
1474            toks("\"([\"\nx\n\"{name}]\"\ny"),
1475            vec![
1476                Token::Str("([".into()),
1477                Token::Semicolon,
1478                Token::Ident("x".into()),
1479                Token::Semicolon,
1480                Token::StringInterp(vec![
1481                    StringPart::Variable("name".into()),
1482                    StringPart::Literal("]".into()),
1483                ]),
1484                Token::Semicolon,
1485                Token::Ident("y".into()),
1486            ]
1487        );
1488    }
1489
1490    #[test]
1491    fn unmatched_closing_delimiter_does_not_corrupt_depth() {
1492        assert_eq!(
1493            toks("]\nx"),
1494            vec![Token::RBracket, Token::Semicolon, Token::Ident("x".into()),]
1495        );
1496    }
1497
1498    // ─── Line tracking ─────────────────────────────────────────────
1499
1500    #[test]
1501    fn line_numbers() {
1502        let tokens = lex("x\ny\nz").unwrap();
1503        let lines: Vec<u32> = tokens.iter().map(|t| t.line).collect();
1504        // x(L1), ;(L1), y(L2), ;(L2), z(L3), Eof(L3)
1505        assert_eq!(lines, vec![1, 1, 2, 2, 3, 3]);
1506    }
1507
1508    // ─── Unknown character ─────────────────────────────────────────
1509
1510    #[test]
1511    fn unknown_char() {
1512        assert!(lex_err("@").contains("don't understand"));
1513    }
1514}