leo_parser_rowan/
lexer.rs

1// Copyright (C) 2019-2026 Provable Inc.
2// This file is part of the Leo library.
3
4// The Leo library is free software: you can redistribute it and/or modify
5// it under the terms of the GNU General Public License as published by
6// the Free Software Foundation, either version 3 of the License, or
7// (at your option) any later version.
8
9// The Leo library is distributed in the hope that it will be useful,
10// but WITHOUT ANY WARRANTY; without even the implied warranty of
11// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12// GNU General Public License for more details.
13
14// You should have received a copy of the GNU General Public License
15// along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
16
17//! Lexer for the rowan-based Leo parser.
18//!
19//! This module provides a logos-based lexer that produces tokens suitable for
20//! use with rowan's GreenNodeBuilder. All tokens, including whitespace and
21//! comments (trivia), are explicitly represented.
22
23use crate::{SyntaxKind, SyntaxKind::*};
24use logos::Logos;
25use rowan::{TextRange, TextSize};
26
27/// A token produced by the lexer.
28#[derive(Debug, Clone, Copy, PartialEq, Eq)]
29pub struct Token {
30    /// The kind of token.
31    pub kind: SyntaxKind,
32    /// The length in bytes of the token text.
33    pub len: u32,
34}
35
36/// The kind of lexer error, carrying any structured data needed for diagnostics.
37#[derive(Debug, Clone, PartialEq, Eq)]
38pub enum LexErrorKind {
39    /// A digit was invalid for the given radix (e.g. `9` in an octal literal).
40    InvalidDigit { digit: char, radix: u32, token: String },
41    /// A token could not be lexed at all.
42    CouldNotLex { content: String },
43    /// A Unicode bidi override code point was encountered.
44    BidiOverride,
45}
46
47/// An error encountered during lexing.
48#[derive(Debug, Clone, PartialEq, Eq)]
49pub struct LexError {
50    /// The text range where the error occurred.
51    pub range: TextRange,
52    /// Structured error kind.
53    pub kind: LexErrorKind,
54}
55
56/// Callback for parsing block comments.
57///
58/// Block comments can't be matched with a simple regex due to the need to find
59/// the closing `*/`. This also detects bidi override characters for security.
60/// Always returns true to produce a token; unterminated comments are detected
61/// by checking if the slice ends with `*/` in the lex() function.
62fn comment_block(lex: &mut logos::Lexer<LogosToken>) -> bool {
63    let mut last_asterisk = false;
64    for (index, c) in lex.remainder().char_indices() {
65        if c == '*' {
66            last_asterisk = true;
67        } else if c == '/' && last_asterisk {
68            lex.bump(index + 1);
69            return true;
70        } else if matches!(c, '\u{202A}'..='\u{202E}' | '\u{2066}'..='\u{2069}') {
71            // Bidi character detected - end the comment token here
72            // so we can report that error separately.
73            lex.bump(index);
74            return true;
75        } else {
76            last_asterisk = false;
77        }
78    }
79    // Unterminated block comment - consume all remaining input
80    let remaining = lex.remainder().len();
81    lex.bump(remaining);
82    true
83}
84
85/// Internal logos token enum.
86///
87/// This is mapped to `SyntaxKind` during lexing. We use a separate enum here
88/// because logos requires ownership of the token type during lexing.
89#[derive(Clone, Copy, Debug, PartialEq, Eq, Logos)]
90enum LogosToken {
91    // =========================================================================
92    // Trivia
93    // =========================================================================
94    #[regex(r"[ \t\f]+")]
95    Whitespace,
96
97    #[regex(r"\r?\n")]
98    Linebreak,
99
100    // Comments don't include line breaks or bidi characters
101    #[regex(r"//[^\r\n\u{202A}-\u{202E}\u{2066}-\u{2069}]*")]
102    CommentLine,
103
104    #[token(r"/*", comment_block)]
105    CommentBlock,
106
107    // =========================================================================
108    // Literals
109    // =========================================================================
110    // Address literals: aleo1...
111    // We lex any length and validate later
112    #[regex(r"aleo1[a-z0-9]*")]
113    AddressLiteral,
114
115    // Integer literals with various radixes
116    // The regex includes type suffixes (u8, i32, field, etc.) to lex as a single token.
117    // We use permissive patterns that allow invalid digits (e.g., G in hex) so we can
118    // report specific validation errors rather than failing to lex.
119    #[regex(r"0x[0-9A-Za-z_]+([ui](8|16|32|64|128)|field|group|scalar)?", priority = 3)]
120    #[regex(r"0o[0-9A-Za-z_]+([ui](8|16|32|64|128)|field|group|scalar)?", priority = 3)]
121    #[regex(r"0b[0-9A-Za-z_]+([ui](8|16|32|64|128)|field|group|scalar)?", priority = 3)]
122    #[regex(r"[0-9][0-9A-Za-z_]*([ui](8|16|32|64|128)|field|group|scalar)?")]
123    Integer,
124
125    #[regex(r#""[^"]*""#)]
126    StaticString,
127
128    // Identifier literals: 'foo', 'bar_baz'
129    #[regex(r"'[a-zA-Z][a-zA-Z0-9_]*'")]
130    IdentifierLiteral,
131
132    // =========================================================================
133    // Identifiers and Keywords
134    // =========================================================================
135    // Note: Complex identifiers (paths like foo::bar, program IDs like foo.aleo,
136    // locators like foo.aleo::bar) are deferred to Phase 2. The lexer produces
137    // simple tokens; the parser handles disambiguation.
138    //
139    // We need special cases for `group::abc`, `signature::abc`, and `Future::abc`
140    // as otherwise these would be keywords followed by ::.
141    #[regex(r"group::[a-zA-Z][a-zA-Z0-9_]*")]
142    #[regex(r"signature::[a-zA-Z][a-zA-Z0-9_]*")]
143    #[regex(r"Future::[a-zA-Z][a-zA-Z0-9_]*")]
144    PathSpecial,
145
146    // Identifiers starting with underscore (intrinsic names)
147    #[regex(r"_[a-zA-Z][a-zA-Z0-9_]*")]
148    IdentIntrinsic,
149
150    // Regular identifiers (keywords are matched by checking the slice)
151    #[regex(r"[a-zA-Z][a-zA-Z0-9_]*")]
152    Ident,
153
154    // =========================================================================
155    // Operators (multi-character first for correct priority)
156    // =========================================================================
157    #[token("**=")]
158    PowAssign,
159    #[token("&&=")]
160    AndAssign,
161    #[token("||=")]
162    OrAssign,
163    #[token("<<=")]
164    ShlAssign,
165    #[token(">>=")]
166    ShrAssign,
167
168    #[token("**")]
169    Pow,
170    #[token("&&")]
171    And,
172    #[token("||")]
173    Or,
174    #[token("<<")]
175    Shl,
176    #[token(">>")]
177    Shr,
178    #[token("==")]
179    EqEq,
180    #[token("!=")]
181    NotEq,
182    #[token("<=")]
183    LtEq,
184    #[token(">=")]
185    GtEq,
186    #[token("+=")]
187    AddAssign,
188    #[token("-=")]
189    SubAssign,
190    #[token("*=")]
191    MulAssign,
192    #[token("/=")]
193    DivAssign,
194    #[token("%=")]
195    RemAssign,
196    #[token("&=")]
197    BitAndAssign,
198    #[token("|=")]
199    BitOrAssign,
200    #[token("^=")]
201    BitXorAssign,
202
203    #[token("->")]
204    Arrow,
205    #[token("=>")]
206    FatArrow,
207    #[token("..=")]
208    DotDotEq,
209    #[token("..")]
210    DotDot,
211    #[token("::")]
212    ColonColon,
213
214    // Single character operators and punctuation
215    #[token("=")]
216    Eq,
217    #[token("!")]
218    Bang,
219    #[token("<")]
220    Lt,
221    #[token(">")]
222    Gt,
223    #[token("+")]
224    Plus,
225    #[token("-")]
226    Minus,
227    #[token("*")]
228    Star,
229    #[token("/")]
230    Slash,
231    #[token("%")]
232    Percent,
233    #[token("&")]
234    Amp,
235    #[token("|")]
236    Pipe,
237    #[token("^")]
238    Caret,
239
240    // =========================================================================
241    // Punctuation
242    // =========================================================================
243    #[token("(")]
244    LParen,
245    #[token(")")]
246    RParen,
247    #[token("[")]
248    LBracket,
249    #[token("]")]
250    RBracket,
251    #[token("{")]
252    LBrace,
253    #[token("}")]
254    RBrace,
255    #[token(",")]
256    Comma,
257    #[token(".")]
258    Dot,
259    #[token(";")]
260    Semicolon,
261    #[token(":")]
262    Colon,
263    #[token("?")]
264    Question,
265    #[token("_")]
266    Underscore,
267    #[token("@")]
268    At,
269
270    // =========================================================================
271    // Security
272    // =========================================================================
273    // Unicode bidirectional control characters are a security risk.
274    // We detect them so we can report an error.
275    #[regex(r"[\u{202A}-\u{202E}\u{2066}-\u{2069}]")]
276    Bidi,
277}
278
279/// Convert an identifier slice to the appropriate SyntaxKind (keyword or IDENT).
280fn ident_to_kind(s: &str) -> SyntaxKind {
281    match s {
282        // Literal keywords
283        "true" => KW_TRUE,
284        "false" => KW_FALSE,
285        "none" => KW_NONE,
286        // Type keywords
287        "address" => KW_ADDRESS,
288        "bool" => KW_BOOL,
289        "field" => KW_FIELD,
290        "group" => KW_GROUP,
291        "scalar" => KW_SCALAR,
292        "signature" => KW_SIGNATURE,
293        "string" => KW_STRING,
294        "record" => KW_RECORD,
295        "dyn" => KW_DYN,
296        "identifier" => KW_IDENTIFIER,
297        "i8" => KW_I8,
298        "i16" => KW_I16,
299        "i32" => KW_I32,
300        "i64" => KW_I64,
301        "i128" => KW_I128,
302        "u8" => KW_U8,
303        "u16" => KW_U16,
304        "u32" => KW_U32,
305        "u64" => KW_U64,
306        "u128" => KW_U128,
307        // Control flow keywords
308        "if" => KW_IF,
309        "else" => KW_ELSE,
310        "for" => KW_FOR,
311        "in" => KW_IN,
312        "return" => KW_RETURN,
313        // Declaration keywords
314        "let" => KW_LET,
315        "const" => KW_CONST,
316        "constant" => KW_CONSTANT,
317        "final" => KW_FINAL,
318        "Final" => KW_FINAL_UPPER,
319        "view" => KW_VIEW,
320        "fn" => KW_FN,
321        "Fn" => KW_FN_UPPER,
322        "struct" => KW_STRUCT,
323        "constructor" => KW_CONSTRUCTOR,
324        "interface" => KW_INTERFACE,
325        // Program structure keywords
326        "program" => KW_PROGRAM,
327        "import" => KW_IMPORT,
328        "mapping" => KW_MAPPING,
329        "storage" => KW_STORAGE,
330        "network" => KW_NETWORK,
331        "aleo" => KW_ALEO,
332        "script" => KW_SCRIPT,
333        "block" => KW_BLOCK,
334        // Visibility & assertion keywords
335        "public" => KW_PUBLIC,
336        "private" => KW_PRIVATE,
337        "export" => KW_EXPORT,
338        "as" => KW_AS,
339        "self" => KW_SELF,
340        "Self" => KW_SELF_UPPER,
341        "assert" => KW_ASSERT,
342        "assert_eq" => KW_ASSERT_EQ,
343        "assert_neq" => KW_ASSERT_NEQ,
344        // Not a keyword
345        _ => IDENT,
346    }
347}
348
349pub(crate) fn is_keyword(s: &str) -> bool {
350    ident_to_kind(s).is_keyword()
351}
352
353/// Strip integer type suffix from a string, returning the numeric part.
354fn strip_int_suffix(s: &str) -> Option<&str> {
355    // Check for integer type suffixes (longest first to match correctly)
356    let suffixes = ["u128", "i128", "u64", "i64", "u32", "i32", "u16", "i16", "u8", "i8"];
357    for suffix in suffixes {
358        if let Some(prefix) = s.strip_suffix(suffix) {
359            return Some(prefix);
360        }
361    }
362    None
363}
364
365/// Validate integer literal digits for the appropriate radix.
366/// Adds errors to the error vector if invalid digits are found.
367fn validate_integer_digits(text: &str, offset: usize, errors: &mut Vec<LexError>) {
368    // Strip type suffix if present (field, group, scalar, or integer types)
369    let num_part = text
370        .strip_suffix("field")
371        .or_else(|| text.strip_suffix("group"))
372        .or_else(|| text.strip_suffix("scalar"))
373        .or_else(|| strip_int_suffix(text))
374        .unwrap_or(text);
375
376    // Determine the radix and get the digit part
377    let (digits, radix, _prefix_len): (&str, u32, usize) = if let Some(s) = num_part.strip_prefix("0x") {
378        (s, 16, 2)
379    } else if let Some(s) = num_part.strip_prefix("0X") {
380        (s, 16, 2)
381    } else if let Some(s) = num_part.strip_prefix("0o") {
382        (s, 8, 2)
383    } else if let Some(s) = num_part.strip_prefix("0O") {
384        (s, 8, 2)
385    } else if let Some(s) = num_part.strip_prefix("0b") {
386        (s, 2, 2)
387    } else if let Some(s) = num_part.strip_prefix("0B") {
388        (s, 2, 2)
389    } else {
390        // Decimal - no prefix
391        (num_part, 10, 0)
392    };
393
394    // Find the first invalid digit
395    for (_, c) in digits.char_indices() {
396        if c == '_' {
397            continue; // Underscores are allowed
398        }
399        if !c.is_digit(radix) {
400            // Found an invalid digit - span covers the entire numeric part (like LALRPOP)
401            let error_end = offset + num_part.len();
402            errors.push(LexError {
403                range: TextRange::new(TextSize::new(offset as u32), TextSize::new(error_end as u32)),
404                kind: LexErrorKind::InvalidDigit { digit: c, radix, token: num_part.to_string() },
405            });
406            return; // Only report the first invalid digit
407        }
408    }
409}
410
411/// Lex the given source text into a sequence of tokens.
412///
413/// Returns a vector of tokens and any errors encountered. Even if errors
414/// occur, tokens are still produced to enable error recovery in the parser.
415pub fn lex(source: &str) -> (Vec<Token>, Vec<LexError>) {
416    let mut tokens = Vec::new();
417    let mut errors = Vec::new();
418    let mut lexer = LogosToken::lexer(source);
419
420    while let Some(result) = lexer.next() {
421        let span = lexer.span();
422        let len = (span.end - span.start) as u32;
423        let slice = lexer.slice();
424
425        let kind = match result {
426            Ok(token) => match token {
427                // Trivia
428                LogosToken::Whitespace => WHITESPACE,
429                LogosToken::Linebreak => LINEBREAK,
430                LogosToken::CommentLine => COMMENT_LINE,
431                LogosToken::CommentBlock => {
432                    // Check if block comment was properly terminated
433                    if !slice.ends_with("*/") {
434                        let preview_len = slice.len().min(10);
435                        let preview = &slice[..preview_len];
436                        errors.push(LexError {
437                            range: TextRange::new(
438                                TextSize::new(span.start as u32),
439                                TextSize::new((span.start + 2) as u32), // Just the /*
440                            ),
441                            kind: LexErrorKind::CouldNotLex { content: preview.to_string() },
442                        });
443                    }
444                    COMMENT_BLOCK
445                }
446
447                // Literals
448                LogosToken::AddressLiteral => ADDRESS_LIT,
449                LogosToken::Integer => INTEGER,
450                LogosToken::StaticString => STRING,
451                LogosToken::IdentifierLiteral => IDENT_LIT,
452
453                // Identifiers (check for keywords)
454                LogosToken::Ident => ident_to_kind(slice),
455                LogosToken::IdentIntrinsic => IDENT,
456                LogosToken::PathSpecial => IDENT, // Treat as IDENT for now (Phase 2)
457
458                // Multi-char operators
459                LogosToken::PowAssign => STAR2_EQ,
460                LogosToken::AndAssign => AMP2_EQ,
461                LogosToken::OrAssign => PIPE2_EQ,
462                LogosToken::ShlAssign => SHL_EQ,
463                LogosToken::ShrAssign => SHR_EQ,
464                LogosToken::Pow => STAR2,
465                LogosToken::And => AMP2,
466                LogosToken::Or => PIPE2,
467                LogosToken::Shl => SHL,
468                LogosToken::Shr => SHR,
469                LogosToken::EqEq => EQ2,
470                LogosToken::NotEq => BANG_EQ,
471                LogosToken::LtEq => LT_EQ,
472                LogosToken::GtEq => GT_EQ,
473                LogosToken::AddAssign => PLUS_EQ,
474                LogosToken::SubAssign => MINUS_EQ,
475                LogosToken::MulAssign => STAR_EQ,
476                LogosToken::DivAssign => SLASH_EQ,
477                LogosToken::RemAssign => PERCENT_EQ,
478                LogosToken::BitAndAssign => AMP_EQ,
479                LogosToken::BitOrAssign => PIPE_EQ,
480                LogosToken::BitXorAssign => CARET_EQ,
481                LogosToken::Arrow => ARROW,
482                LogosToken::FatArrow => FAT_ARROW,
483                LogosToken::DotDotEq => DOT_DOT_EQ,
484                LogosToken::DotDot => DOT_DOT,
485                LogosToken::ColonColon => COLON_COLON,
486
487                // Single-char operators
488                LogosToken::Eq => EQ,
489                LogosToken::Bang => BANG,
490                LogosToken::Lt => LT,
491                LogosToken::Gt => GT,
492                LogosToken::Plus => PLUS,
493                LogosToken::Minus => MINUS,
494                LogosToken::Star => STAR,
495                LogosToken::Slash => SLASH,
496                LogosToken::Percent => PERCENT,
497                LogosToken::Amp => AMP,
498                LogosToken::Pipe => PIPE,
499                LogosToken::Caret => CARET,
500
501                // Punctuation
502                LogosToken::LParen => L_PAREN,
503                LogosToken::RParen => R_PAREN,
504                LogosToken::LBracket => L_BRACKET,
505                LogosToken::RBracket => R_BRACKET,
506                LogosToken::LBrace => L_BRACE,
507                LogosToken::RBrace => R_BRACE,
508                LogosToken::Comma => COMMA,
509                LogosToken::Dot => DOT,
510                LogosToken::Semicolon => SEMICOLON,
511                LogosToken::Colon => COLON,
512                LogosToken::Question => QUESTION,
513                LogosToken::Underscore => UNDERSCORE,
514                LogosToken::At => AT,
515
516                // Security: bidi characters
517                LogosToken::Bidi => {
518                    errors.push(LexError {
519                        range: TextRange::new(TextSize::new(span.start as u32), TextSize::new(span.end as u32)),
520                        kind: LexErrorKind::BidiOverride,
521                    });
522                    ERROR
523                }
524            },
525            Err(()) => {
526                errors.push(LexError {
527                    range: TextRange::new(TextSize::new(span.start as u32), TextSize::new(span.end as u32)),
528                    kind: LexErrorKind::CouldNotLex { content: slice.to_string() },
529                });
530                ERROR
531            }
532        };
533
534        // Validate integer literal digits for the appropriate radix
535        if kind == INTEGER {
536            validate_integer_digits(slice, span.start, &mut errors);
537        }
538
539        tokens.push(Token { kind, len });
540    }
541
542    // Add EOF token
543    tokens.push(Token { kind: EOF, len: 0 });
544
545    (tokens, errors)
546}
547
548#[cfg(test)]
549mod tests {
550    use super::*;
551    use expect_test::{Expect, expect};
552
553    /// Helper to format tokens for snapshot testing.
554    fn check_lex(input: &str, expect: Expect) {
555        let (tokens, _errors) = lex(input);
556        let mut output = String::new();
557        let mut offset = 0usize;
558        for token in &tokens {
559            let text = &input[offset..offset + token.len as usize];
560            output.push_str(&format!("{:?} {:?}\n", token.kind, text));
561            offset += token.len as usize;
562        }
563        expect.assert_eq(&output);
564    }
565
566    /// Helper to check that lexing produces expected errors.
567    fn check_lex_errors(input: &str, expect: Expect) {
568        let (_tokens, errors) = lex(input);
569        let output = errors
570            .iter()
571            .map(|e| format!("{}..{}:{:?}", u32::from(e.range.start()), u32::from(e.range.end()), e.kind))
572            .collect::<Vec<_>>()
573            .join("\n");
574        expect.assert_eq(&output);
575    }
576
577    #[test]
578    fn lex_empty() {
579        check_lex("", expect![[r#"
580            EOF ""
581        "#]]);
582    }
583
584    #[test]
585    fn lex_whitespace() {
586        check_lex("  \t  ", expect![[r#"
587            WHITESPACE "  \t  "
588            EOF ""
589        "#]]);
590    }
591
592    #[test]
593    fn lex_linebreaks() {
594        check_lex("\n\r\n\n", expect![[r#"
595            LINEBREAK "\n"
596            LINEBREAK "\r\n"
597            LINEBREAK "\n"
598            EOF ""
599"#]]);
600    }
601
602    #[test]
603    fn lex_mixed_whitespace() {
604        check_lex("  \n  \t\n", expect![[r#"
605            WHITESPACE "  "
606            LINEBREAK "\n"
607            WHITESPACE "  \t"
608            LINEBREAK "\n"
609            EOF ""
610        "#]]);
611    }
612
613    #[test]
614    fn lex_line_comments() {
615        check_lex("// hello\n// world", expect![[r#"
616            COMMENT_LINE "// hello"
617            LINEBREAK "\n"
618            COMMENT_LINE "// world"
619            EOF ""
620        "#]]);
621    }
622
623    #[test]
624    fn lex_block_comments() {
625        check_lex("/* hello */ /* multi\nline */", expect![[r#"
626            COMMENT_BLOCK "/* hello */"
627            WHITESPACE " "
628            COMMENT_BLOCK "/* multi\nline */"
629            EOF ""
630        "#]]);
631    }
632
633    #[test]
634    fn lex_identifiers() {
635        check_lex("foo Bar _baz x123", expect![[r#"
636            IDENT "foo"
637            WHITESPACE " "
638            IDENT "Bar"
639            WHITESPACE " "
640            IDENT "_baz"
641            WHITESPACE " "
642            IDENT "x123"
643            EOF ""
644        "#]]);
645    }
646
647    #[test]
648    fn lex_keywords() {
649        check_lex("let fn if return true false", expect![[r#"
650            KW_LET "let"
651            WHITESPACE " "
652            KW_FN "fn"
653            WHITESPACE " "
654            KW_IF "if"
655            WHITESPACE " "
656            KW_RETURN "return"
657            WHITESPACE " "
658            KW_TRUE "true"
659            WHITESPACE " "
660            KW_FALSE "false"
661            EOF ""
662        "#]]);
663    }
664
665    #[test]
666    fn lex_type_keywords() {
667        check_lex("u8 u16 u32 u64 u128 i8 i16 i32 i64 i128", expect![[r#"
668            KW_U8 "u8"
669            WHITESPACE " "
670            KW_U16 "u16"
671            WHITESPACE " "
672            KW_U32 "u32"
673            WHITESPACE " "
674            KW_U64 "u64"
675            WHITESPACE " "
676            KW_U128 "u128"
677            WHITESPACE " "
678            KW_I8 "i8"
679            WHITESPACE " "
680            KW_I16 "i16"
681            WHITESPACE " "
682            KW_I32 "i32"
683            WHITESPACE " "
684            KW_I64 "i64"
685            WHITESPACE " "
686            KW_I128 "i128"
687            EOF ""
688        "#]]);
689    }
690
691    #[test]
692    fn lex_more_type_keywords() {
693        check_lex("bool field group scalar address signature string record", expect![[r#"
694            KW_BOOL "bool"
695            WHITESPACE " "
696            KW_FIELD "field"
697            WHITESPACE " "
698            KW_GROUP "group"
699            WHITESPACE " "
700            KW_SCALAR "scalar"
701            WHITESPACE " "
702            KW_ADDRESS "address"
703            WHITESPACE " "
704            KW_SIGNATURE "signature"
705            WHITESPACE " "
706            KW_STRING "string"
707            WHITESPACE " "
708            KW_RECORD "record"
709            EOF ""
710        "#]]);
711    }
712
713    #[test]
714    fn lex_identifier_literal() {
715        check_lex("'foo' 'bar_baz' 'x'", expect![[r#"
716            IDENT_LIT "'foo'"
717            WHITESPACE " "
718            IDENT_LIT "'bar_baz'"
719            WHITESPACE " "
720            IDENT_LIT "'x'"
721            EOF ""
722        "#]]);
723    }
724
725    #[test]
726    fn lex_identifier_keyword() {
727        check_lex("identifier", expect![[r#"
728            KW_IDENTIFIER "identifier"
729            EOF ""
730        "#]]);
731    }
732
733    #[test]
734    fn keyword_predicate_matches_identifier_keywords() {
735        assert!(super::is_keyword("view"));
736        assert!(super::is_keyword("dyn"));
737        assert!(super::is_keyword("in"));
738        assert!(!super::is_keyword("inside"));
739    }
740
741    #[test]
742    fn lex_integers() {
743        check_lex("123 0xFF 0b101 0o77", expect![[r#"
744            INTEGER "123"
745            WHITESPACE " "
746            INTEGER "0xFF"
747            WHITESPACE " "
748            INTEGER "0b101"
749            WHITESPACE " "
750            INTEGER "0o77"
751            EOF ""
752        "#]]);
753    }
754
755    #[test]
756    fn lex_integers_with_underscores() {
757        check_lex("1_000_000 0xFF_FF", expect![[r#"
758            INTEGER "1_000_000"
759            WHITESPACE " "
760            INTEGER "0xFF_FF"
761            EOF ""
762        "#]]);
763    }
764
765    #[test]
766    fn lex_address_literal() {
767        check_lex("aleo1abc123", expect![[r#"
768            ADDRESS_LIT "aleo1abc123"
769            EOF ""
770        "#]]);
771    }
772
773    #[test]
774    fn lex_strings() {
775        check_lex(r#""hello" "world""#, expect![[r#"
776            STRING "\"hello\""
777            WHITESPACE " "
778            STRING "\"world\""
779            EOF ""
780        "#]]);
781    }
782
783    #[test]
784    fn lex_punctuation() {
785        check_lex("( ) [ ] { } , . ; : :: ? -> => _ @", expect![[r#"
786            L_PAREN "("
787            WHITESPACE " "
788            R_PAREN ")"
789            WHITESPACE " "
790            L_BRACKET "["
791            WHITESPACE " "
792            R_BRACKET "]"
793            WHITESPACE " "
794            L_BRACE "{"
795            WHITESPACE " "
796            R_BRACE "}"
797            WHITESPACE " "
798            COMMA ","
799            WHITESPACE " "
800            DOT "."
801            WHITESPACE " "
802            SEMICOLON ";"
803            WHITESPACE " "
804            COLON ":"
805            WHITESPACE " "
806            COLON_COLON "::"
807            WHITESPACE " "
808            QUESTION "?"
809            WHITESPACE " "
810            ARROW "->"
811            WHITESPACE " "
812            FAT_ARROW "=>"
813            WHITESPACE " "
814            UNDERSCORE "_"
815            WHITESPACE " "
816            AT "@"
817            EOF ""
818        "#]]);
819    }
820
821    #[test]
822    fn lex_arithmetic_operators() {
823        check_lex("+ - * / % **", expect![[r#"
824            PLUS "+"
825            WHITESPACE " "
826            MINUS "-"
827            WHITESPACE " "
828            STAR "*"
829            WHITESPACE " "
830            SLASH "/"
831            WHITESPACE " "
832            PERCENT "%"
833            WHITESPACE " "
834            STAR2 "**"
835            EOF ""
836        "#]]);
837    }
838
839    #[test]
840    fn lex_comparison_operators() {
841        check_lex("== != < <= > >=", expect![[r#"
842            EQ2 "=="
843            WHITESPACE " "
844            BANG_EQ "!="
845            WHITESPACE " "
846            LT "<"
847            WHITESPACE " "
848            LT_EQ "<="
849            WHITESPACE " "
850            GT ">"
851            WHITESPACE " "
852            GT_EQ ">="
853            EOF ""
854        "#]]);
855    }
856
857    #[test]
858    fn lex_logical_operators() {
859        check_lex("&& || !", expect![[r#"
860            AMP2 "&&"
861            WHITESPACE " "
862            PIPE2 "||"
863            WHITESPACE " "
864            BANG "!"
865            EOF ""
866        "#]]);
867    }
868
869    #[test]
870    fn lex_bitwise_operators() {
871        check_lex("& | ^ << >>", expect![[r#"
872            AMP "&"
873            WHITESPACE " "
874            PIPE "|"
875            WHITESPACE " "
876            CARET "^"
877            WHITESPACE " "
878            SHL "<<"
879            WHITESPACE " "
880            SHR ">>"
881            EOF ""
882        "#]]);
883    }
884
885    #[test]
886    fn lex_assignment_operators() {
887        check_lex("= += -= *= /= %= **= &&= ||=", expect![[r#"
888            EQ "="
889            WHITESPACE " "
890            PLUS_EQ "+="
891            WHITESPACE " "
892            MINUS_EQ "-="
893            WHITESPACE " "
894            STAR_EQ "*="
895            WHITESPACE " "
896            SLASH_EQ "/="
897            WHITESPACE " "
898            PERCENT_EQ "%="
899            WHITESPACE " "
900            STAR2_EQ "**="
901            WHITESPACE " "
902            AMP2_EQ "&&="
903            WHITESPACE " "
904            PIPE2_EQ "||="
905            EOF ""
906        "#]]);
907    }
908
909    #[test]
910    fn lex_more_assignment_operators() {
911        check_lex("&= |= ^= <<= >>=", expect![[r#"
912            AMP_EQ "&="
913            WHITESPACE " "
914            PIPE_EQ "|="
915            WHITESPACE " "
916            CARET_EQ "^="
917            WHITESPACE " "
918            SHL_EQ "<<="
919            WHITESPACE " "
920            SHR_EQ ">>="
921            EOF ""
922        "#]]);
923    }
924
925    #[test]
926    fn lex_dot_dot() {
927        check_lex("0..10", expect![[r#"
928            INTEGER "0"
929            DOT_DOT ".."
930            INTEGER "10"
931            EOF ""
932        "#]]);
933    }
934
935    #[test]
936    fn lex_simple_expression() {
937        check_lex("x + y * 2", expect![[r#"
938            IDENT "x"
939            WHITESPACE " "
940            PLUS "+"
941            WHITESPACE " "
942            IDENT "y"
943            WHITESPACE " "
944            STAR "*"
945            WHITESPACE " "
946            INTEGER "2"
947            EOF ""
948        "#]]);
949    }
950
951    #[test]
952    fn lex_function_call() {
953        check_lex("foo(a, b)", expect![[r#"
954            IDENT "foo"
955            L_PAREN "("
956            IDENT "a"
957            COMMA ","
958            WHITESPACE " "
959            IDENT "b"
960            R_PAREN ")"
961            EOF ""
962        "#]]);
963    }
964
965    #[test]
966    fn lex_function_definition() {
967        check_lex("fn add(x: u32) -> u32 {", expect![[r#"
968            KW_FN "fn"
969            WHITESPACE " "
970            IDENT "add"
971            L_PAREN "("
972            IDENT "x"
973            COLON ":"
974            WHITESPACE " "
975            KW_U32 "u32"
976            R_PAREN ")"
977            WHITESPACE " "
978            ARROW "->"
979            WHITESPACE " "
980            KW_U32 "u32"
981            WHITESPACE " "
982            L_BRACE "{"
983            EOF ""
984        "#]]);
985    }
986
987    #[test]
988    fn lex_let_statement() {
989        check_lex("let x: u32 = 42;", expect![[r#"
990            KW_LET "let"
991            WHITESPACE " "
992            IDENT "x"
993            COLON ":"
994            WHITESPACE " "
995            KW_U32 "u32"
996            WHITESPACE " "
997            EQ "="
998            WHITESPACE " "
999            INTEGER "42"
1000            SEMICOLON ";"
1001            EOF ""
1002        "#]]);
1003    }
1004
1005    #[test]
1006    fn lex_typed_integers() {
1007        // Integer literals with type suffixes should be lexed as single tokens
1008        check_lex("1000u32 42i64 0u8 255u128", expect![[r#"
1009            INTEGER "1000u32"
1010            WHITESPACE " "
1011            INTEGER "42i64"
1012            WHITESPACE " "
1013            INTEGER "0u8"
1014            WHITESPACE " "
1015            INTEGER "255u128"
1016            EOF ""
1017        "#]]);
1018    }
1019
1020    #[test]
1021    fn lex_typed_integers_field() {
1022        // Field, group, and scalar suffixes
1023        check_lex("123field 456group 789scalar", expect![[r#"
1024            INTEGER "123field"
1025            WHITESPACE " "
1026            INTEGER "456group"
1027            WHITESPACE " "
1028            INTEGER "789scalar"
1029            EOF ""
1030        "#]]);
1031    }
1032
1033    #[test]
1034    fn lex_special_paths() {
1035        // These are special cases where keywords are followed by ::
1036        check_lex("group::GEN signature::verify Future::await", expect![[r#"
1037            IDENT "group::GEN"
1038            WHITESPACE " "
1039            IDENT "signature::verify"
1040            WHITESPACE " "
1041            IDENT "Future::await"
1042            EOF ""
1043        "#]]);
1044    }
1045
1046    #[test]
1047    fn lex_typed_integer_range() {
1048        // Integer with type suffix followed by range operator
1049        check_lex("0u8..STOP", expect![[r#"
1050            INTEGER "0u8"
1051            DOT_DOT ".."
1052            IDENT "STOP"
1053            EOF ""
1054        "#]]);
1055    }
1056
1057    #[test]
1058    fn lex_error_unknown_char() {
1059        check_lex_errors("hello $ world", expect![[r#"6..7:CouldNotLex { content: "$" }"#]]);
1060    }
1061
1062    #[test]
1063    fn lex_invalid_hex_digit() {
1064        let (tokens, errors) = lex("0xGAu32");
1065        assert_eq!(tokens.len(), 2); // INTEGER + EOF
1066        assert!(!errors.is_empty());
1067        assert!(matches!(errors[0].kind, LexErrorKind::InvalidDigit { digit: 'G', radix: 16, .. }));
1068    }
1069
1070    #[test]
1071    fn lex_invalid_octal_digit() {
1072        let (_, errors) = lex("0o9u32");
1073        assert!(!errors.is_empty());
1074        assert!(matches!(errors[0].kind, LexErrorKind::InvalidDigit { digit: '9', radix: 8, .. }));
1075    }
1076
1077    #[test]
1078    fn lex_invalid_binary_digit() {
1079        let (_, errors) = lex("0b2u32");
1080        assert!(!errors.is_empty());
1081        assert!(matches!(errors[0].kind, LexErrorKind::InvalidDigit { digit: '2', radix: 2, .. }));
1082    }
1083
1084    #[test]
1085    fn lex_valid_hex_is_ok() {
1086        let (_, errors) = lex("0xDEADBEEFu64");
1087        assert!(errors.is_empty());
1088    }
1089
1090    #[test]
1091    fn lex_invalid_hex_lowercase() {
1092        // Lowercase hex digits beyond f are invalid
1093        let (_, errors) = lex("0xghu32");
1094        assert!(!errors.is_empty());
1095        assert!(matches!(errors[0].kind, LexErrorKind::InvalidDigit { digit: 'g', radix: 16, .. }));
1096    }
1097
1098    #[test]
1099    fn lex_bidi_override_error() {
1100        let (_, errors) = lex("let x\u{202E} = 1;");
1101        assert!(!errors.is_empty());
1102        assert!(matches!(errors[0].kind, LexErrorKind::BidiOverride));
1103    }
1104
1105    #[test]
1106    fn lex_unclosed_block_comment() {
1107        let (tokens, errors) = lex("/* unclosed");
1108        assert!(!errors.is_empty());
1109        assert!(matches!(errors[0].kind, LexErrorKind::CouldNotLex { .. }));
1110        // Should still produce a COMMENT_BLOCK token
1111        assert!(tokens.iter().any(|t| t.kind == COMMENT_BLOCK));
1112    }
1113
1114    #[test]
1115    fn lex_nested_comment_not_supported() {
1116        // Leo doesn't support nested comments - the first */ closes the comment
1117        let (tokens, errors) = lex("/* outer /* inner */");
1118        // No errors - the first */ terminates the comment
1119        assert!(errors.is_empty());
1120        // The "outer /* inner " part is the comment, then "*/" closes it
1121        // But actually let's check what happens...
1122        // "/* outer /* inner */" - this finds the first */ which is at position 18
1123        // So the comment is "/* outer /* inner */" which IS terminated
1124        assert!(tokens.iter().any(|t| t.kind == COMMENT_BLOCK));
1125    }
1126
1127    #[test]
1128    fn lex_closed_comment_ok() {
1129        let (_, errors) = lex("/* closed */");
1130        assert!(errors.is_empty());
1131    }
1132}
leo_parser_rowan/lexer.rs

leo_parser_rowan/
lexer.rs
