leo_parser_rowan/
lexer.rs

1// Copyright (C) 2019-2026 Provable Inc.
2// This file is part of the Leo library.
3
4// The Leo library is free software: you can redistribute it and/or modify
5// it under the terms of the GNU General Public License as published by
6// the Free Software Foundation, either version 3 of the License, or
7// (at your option) any later version.
8
9// The Leo library is distributed in the hope that it will be useful,
10// but WITHOUT ANY WARRANTY; without even the implied warranty of
11// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12// GNU General Public License for more details.
13
14// You should have received a copy of the GNU General Public License
15// along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
16
17//! Lexer for the rowan-based Leo parser.
18//!
19//! This module provides a logos-based lexer that produces tokens suitable for
20//! use with rowan's GreenNodeBuilder. All tokens, including whitespace and
21//! comments (trivia), are explicitly represented.
22
23use crate::{SyntaxKind, SyntaxKind::*};
24use logos::Logos;
25use rowan::{TextRange, TextSize};
26
27/// A token produced by the lexer.
28#[derive(Debug, Clone, Copy, PartialEq, Eq)]
29pub struct Token {
30    /// The kind of token.
31    pub kind: SyntaxKind,
32    /// The length in bytes of the token text.
33    pub len: u32,
34}
35
36/// The kind of lexer error, carrying any structured data needed for diagnostics.
37#[derive(Debug, Clone, PartialEq, Eq)]
38pub enum LexErrorKind {
39    /// A digit was invalid for the given radix (e.g. `9` in an octal literal).
40    InvalidDigit { digit: char, radix: u32, token: String },
41    /// A token could not be lexed at all.
42    CouldNotLex { content: String },
43    /// A Unicode bidi override code point was encountered.
44    BidiOverride,
45}
46
47/// An error encountered during lexing.
48#[derive(Debug, Clone, PartialEq, Eq)]
49pub struct LexError {
50    /// The text range where the error occurred.
51    pub range: TextRange,
52    /// Structured error kind.
53    pub kind: LexErrorKind,
54}
55
56/// Callback for parsing block comments.
57///
58/// Block comments can't be matched with a simple regex due to the need to find
59/// the closing `*/`. This also detects bidi override characters for security.
60/// Always returns true to produce a token; unterminated comments are detected
61/// by checking if the slice ends with `*/` in the lex() function.
62fn comment_block(lex: &mut logos::Lexer<LogosToken>) -> bool {
63    let mut last_asterisk = false;
64    for (index, c) in lex.remainder().char_indices() {
65        if c == '*' {
66            last_asterisk = true;
67        } else if c == '/' && last_asterisk {
68            lex.bump(index + 1);
69            return true;
70        } else if matches!(c, '\u{202A}'..='\u{202E}' | '\u{2066}'..='\u{2069}') {
71            // Bidi character detected - end the comment token here
72            // so we can report that error separately.
73            lex.bump(index);
74            return true;
75        } else {
76            last_asterisk = false;
77        }
78    }
79    // Unterminated block comment - consume all remaining input
80    let remaining = lex.remainder().len();
81    lex.bump(remaining);
82    true
83}
84
85/// Internal logos token enum.
86///
87/// This is mapped to `SyntaxKind` during lexing. We use a separate enum here
88/// because logos requires ownership of the token type during lexing.
89#[derive(Clone, Copy, Debug, PartialEq, Eq, Logos)]
90enum LogosToken {
91    // =========================================================================
92    // Trivia
93    // =========================================================================
94    #[regex(r"[ \t\f]+")]
95    Whitespace,
96
97    #[regex(r"\r?\n")]
98    Linebreak,
99
100    // Comments don't include line breaks or bidi characters
101    #[regex(r"//[^\r\n\u{202A}-\u{202E}\u{2066}-\u{2069}]*")]
102    CommentLine,
103
104    #[token(r"/*", comment_block)]
105    CommentBlock,
106
107    // =========================================================================
108    // Literals
109    // =========================================================================
110    // Address literals: aleo1...
111    // We lex any length and validate later
112    #[regex(r"aleo1[a-z0-9]*")]
113    AddressLiteral,
114
115    // Integer literals with various radixes
116    // The regex includes type suffixes (u8, i32, field, etc.) to lex as a single token.
117    // We use permissive patterns that allow invalid digits (e.g., G in hex) so we can
118    // report specific validation errors rather than failing to lex.
119    #[regex(r"0x[0-9A-Za-z_]+([ui](8|16|32|64|128)|field|group|scalar)?", priority = 3)]
120    #[regex(r"0o[0-9A-Za-z_]+([ui](8|16|32|64|128)|field|group|scalar)?", priority = 3)]
121    #[regex(r"0b[0-9A-Za-z_]+([ui](8|16|32|64|128)|field|group|scalar)?", priority = 3)]
122    #[regex(r"[0-9][0-9A-Za-z_]*([ui](8|16|32|64|128)|field|group|scalar)?")]
123    Integer,
124
125    #[regex(r#""[^"]*""#)]
126    StaticString,
127
128    // Identifier literals: 'foo', 'bar_baz'
129    #[regex(r"'[a-zA-Z][a-zA-Z0-9_]*'")]
130    IdentifierLiteral,
131
132    // =========================================================================
133    // Identifiers and Keywords
134    // =========================================================================
135    // Note: Complex identifiers (paths like foo::bar, program IDs like foo.aleo,
136    // locators like foo.aleo::bar) are deferred to Phase 2. The lexer produces
137    // simple tokens; the parser handles disambiguation.
138    //
139    // We need special cases for `group::abc`, `signature::abc`, and `Future::abc`
140    // as otherwise these would be keywords followed by ::.
141    #[regex(r"group::[a-zA-Z][a-zA-Z0-9_]*")]
142    #[regex(r"signature::[a-zA-Z][a-zA-Z0-9_]*")]
143    #[regex(r"Future::[a-zA-Z][a-zA-Z0-9_]*")]
144    PathSpecial,
145
146    // Identifiers starting with underscore (intrinsic names)
147    #[regex(r"_[a-zA-Z][a-zA-Z0-9_]*")]
148    IdentIntrinsic,
149
150    // Regular identifiers (keywords are matched by checking the slice)
151    #[regex(r"[a-zA-Z][a-zA-Z0-9_]*")]
152    Ident,
153
154    // =========================================================================
155    // Operators (multi-character first for correct priority)
156    // =========================================================================
157    #[token("**=")]
158    PowAssign,
159    #[token("&&=")]
160    AndAssign,
161    #[token("||=")]
162    OrAssign,
163    #[token("<<=")]
164    ShlAssign,
165    #[token(">>=")]
166    ShrAssign,
167
168    #[token("**")]
169    Pow,
170    #[token("&&")]
171    And,
172    #[token("||")]
173    Or,
174    #[token("<<")]
175    Shl,
176    #[token(">>")]
177    Shr,
178    #[token("==")]
179    EqEq,
180    #[token("!=")]
181    NotEq,
182    #[token("<=")]
183    LtEq,
184    #[token(">=")]
185    GtEq,
186    #[token("+=")]
187    AddAssign,
188    #[token("-=")]
189    SubAssign,
190    #[token("*=")]
191    MulAssign,
192    #[token("/=")]
193    DivAssign,
194    #[token("%=")]
195    RemAssign,
196    #[token("&=")]
197    BitAndAssign,
198    #[token("|=")]
199    BitOrAssign,
200    #[token("^=")]
201    BitXorAssign,
202
203    #[token("->")]
204    Arrow,
205    #[token("=>")]
206    FatArrow,
207    #[token("..=")]
208    DotDotEq,
209    #[token("..")]
210    DotDot,
211    #[token("::")]
212    ColonColon,
213
214    // Single character operators and punctuation
215    #[token("=")]
216    Eq,
217    #[token("!")]
218    Bang,
219    #[token("<")]
220    Lt,
221    #[token(">")]
222    Gt,
223    #[token("+")]
224    Plus,
225    #[token("-")]
226    Minus,
227    #[token("*")]
228    Star,
229    #[token("/")]
230    Slash,
231    #[token("%")]
232    Percent,
233    #[token("&")]
234    Amp,
235    #[token("|")]
236    Pipe,
237    #[token("^")]
238    Caret,
239
240    // =========================================================================
241    // Punctuation
242    // =========================================================================
243    #[token("(")]
244    LParen,
245    #[token(")")]
246    RParen,
247    #[token("[")]
248    LBracket,
249    #[token("]")]
250    RBracket,
251    #[token("{")]
252    LBrace,
253    #[token("}")]
254    RBrace,
255    #[token(",")]
256    Comma,
257    #[token(".")]
258    Dot,
259    #[token(";")]
260    Semicolon,
261    #[token(":")]
262    Colon,
263    #[token("?")]
264    Question,
265    #[token("_")]
266    Underscore,
267    #[token("@")]
268    At,
269
270    // =========================================================================
271    // Security
272    // =========================================================================
273    // Unicode bidirectional control characters are a security risk.
274    // We detect them so we can report an error.
275    #[regex(r"[\u{202A}-\u{202E}\u{2066}-\u{2069}]")]
276    Bidi,
277}
278
279/// Convert an identifier slice to the appropriate SyntaxKind (keyword or IDENT).
280fn ident_to_kind(s: &str) -> SyntaxKind {
281    match s {
282        // Literal keywords
283        "true" => KW_TRUE,
284        "false" => KW_FALSE,
285        "none" => KW_NONE,
286        // Type keywords
287        "address" => KW_ADDRESS,
288        "bool" => KW_BOOL,
289        "field" => KW_FIELD,
290        "group" => KW_GROUP,
291        "scalar" => KW_SCALAR,
292        "signature" => KW_SIGNATURE,
293        "string" => KW_STRING,
294        "record" => KW_RECORD,
295        "dyn" => KW_DYN,
296        "identifier" => KW_IDENTIFIER,
297        "i8" => KW_I8,
298        "i16" => KW_I16,
299        "i32" => KW_I32,
300        "i64" => KW_I64,
301        "i128" => KW_I128,
302        "u8" => KW_U8,
303        "u16" => KW_U16,
304        "u32" => KW_U32,
305        "u64" => KW_U64,
306        "u128" => KW_U128,
307        // Control flow keywords
308        "if" => KW_IF,
309        "else" => KW_ELSE,
310        "for" => KW_FOR,
311        "in" => KW_IN,
312        "return" => KW_RETURN,
313        // Declaration keywords
314        "let" => KW_LET,
315        "const" => KW_CONST,
316        "constant" => KW_CONSTANT,
317        "final" => KW_FINAL,
318        "Final" => KW_FINAL_UPPER,
319        "view" => KW_VIEW,
320        "fn" => KW_FN,
321        "Fn" => KW_FN_UPPER,
322        "struct" => KW_STRUCT,
323        "constructor" => KW_CONSTRUCTOR,
324        "interface" => KW_INTERFACE,
325        // Program structure keywords
326        "program" => KW_PROGRAM,
327        "import" => KW_IMPORT,
328        "mapping" => KW_MAPPING,
329        "storage" => KW_STORAGE,
330        "network" => KW_NETWORK,
331        "aleo" => KW_ALEO,
332        "script" => KW_SCRIPT,
333        "block" => KW_BLOCK,
334        // Visibility & assertion keywords
335        "public" => KW_PUBLIC,
336        "private" => KW_PRIVATE,
337        "as" => KW_AS,
338        "self" => KW_SELF,
339        "assert" => KW_ASSERT,
340        "assert_eq" => KW_ASSERT_EQ,
341        "assert_neq" => KW_ASSERT_NEQ,
342        // Not a keyword
343        _ => IDENT,
344    }
345}
346
347pub(crate) fn is_keyword(s: &str) -> bool {
348    ident_to_kind(s).is_keyword()
349}
350
351/// Strip integer type suffix from a string, returning the numeric part.
352fn strip_int_suffix(s: &str) -> Option<&str> {
353    // Check for integer type suffixes (longest first to match correctly)
354    let suffixes = ["u128", "i128", "u64", "i64", "u32", "i32", "u16", "i16", "u8", "i8"];
355    for suffix in suffixes {
356        if let Some(prefix) = s.strip_suffix(suffix) {
357            return Some(prefix);
358        }
359    }
360    None
361}
362
363/// Validate integer literal digits for the appropriate radix.
364/// Adds errors to the error vector if invalid digits are found.
365fn validate_integer_digits(text: &str, offset: usize, errors: &mut Vec<LexError>) {
366    // Strip type suffix if present (field, group, scalar, or integer types)
367    let num_part = text
368        .strip_suffix("field")
369        .or_else(|| text.strip_suffix("group"))
370        .or_else(|| text.strip_suffix("scalar"))
371        .or_else(|| strip_int_suffix(text))
372        .unwrap_or(text);
373
374    // Determine the radix and get the digit part
375    let (digits, radix, _prefix_len): (&str, u32, usize) = if let Some(s) = num_part.strip_prefix("0x") {
376        (s, 16, 2)
377    } else if let Some(s) = num_part.strip_prefix("0X") {
378        (s, 16, 2)
379    } else if let Some(s) = num_part.strip_prefix("0o") {
380        (s, 8, 2)
381    } else if let Some(s) = num_part.strip_prefix("0O") {
382        (s, 8, 2)
383    } else if let Some(s) = num_part.strip_prefix("0b") {
384        (s, 2, 2)
385    } else if let Some(s) = num_part.strip_prefix("0B") {
386        (s, 2, 2)
387    } else {
388        // Decimal - no prefix
389        (num_part, 10, 0)
390    };
391
392    // Find the first invalid digit
393    for (_, c) in digits.char_indices() {
394        if c == '_' {
395            continue; // Underscores are allowed
396        }
397        if !c.is_digit(radix) {
398            // Found an invalid digit - span covers the entire numeric part (like LALRPOP)
399            let error_end = offset + num_part.len();
400            errors.push(LexError {
401                range: TextRange::new(TextSize::new(offset as u32), TextSize::new(error_end as u32)),
402                kind: LexErrorKind::InvalidDigit { digit: c, radix, token: num_part.to_string() },
403            });
404            return; // Only report the first invalid digit
405        }
406    }
407}
408
409/// Lex the given source text into a sequence of tokens.
410///
411/// Returns a vector of tokens and any errors encountered. Even if errors
412/// occur, tokens are still produced to enable error recovery in the parser.
413pub fn lex(source: &str) -> (Vec<Token>, Vec<LexError>) {
414    let mut tokens = Vec::new();
415    let mut errors = Vec::new();
416    let mut lexer = LogosToken::lexer(source);
417
418    while let Some(result) = lexer.next() {
419        let span = lexer.span();
420        let len = (span.end - span.start) as u32;
421        let slice = lexer.slice();
422
423        let kind = match result {
424            Ok(token) => match token {
425                // Trivia
426                LogosToken::Whitespace => WHITESPACE,
427                LogosToken::Linebreak => LINEBREAK,
428                LogosToken::CommentLine => COMMENT_LINE,
429                LogosToken::CommentBlock => {
430                    // Check if block comment was properly terminated
431                    if !slice.ends_with("*/") {
432                        let preview_len = slice.len().min(10);
433                        let preview = &slice[..preview_len];
434                        errors.push(LexError {
435                            range: TextRange::new(
436                                TextSize::new(span.start as u32),
437                                TextSize::new((span.start + 2) as u32), // Just the /*
438                            ),
439                            kind: LexErrorKind::CouldNotLex { content: preview.to_string() },
440                        });
441                    }
442                    COMMENT_BLOCK
443                }
444
445                // Literals
446                LogosToken::AddressLiteral => ADDRESS_LIT,
447                LogosToken::Integer => INTEGER,
448                LogosToken::StaticString => STRING,
449                LogosToken::IdentifierLiteral => IDENT_LIT,
450
451                // Identifiers (check for keywords)
452                LogosToken::Ident => ident_to_kind(slice),
453                LogosToken::IdentIntrinsic => IDENT,
454                LogosToken::PathSpecial => IDENT, // Treat as IDENT for now (Phase 2)
455
456                // Multi-char operators
457                LogosToken::PowAssign => STAR2_EQ,
458                LogosToken::AndAssign => AMP2_EQ,
459                LogosToken::OrAssign => PIPE2_EQ,
460                LogosToken::ShlAssign => SHL_EQ,
461                LogosToken::ShrAssign => SHR_EQ,
462                LogosToken::Pow => STAR2,
463                LogosToken::And => AMP2,
464                LogosToken::Or => PIPE2,
465                LogosToken::Shl => SHL,
466                LogosToken::Shr => SHR,
467                LogosToken::EqEq => EQ2,
468                LogosToken::NotEq => BANG_EQ,
469                LogosToken::LtEq => LT_EQ,
470                LogosToken::GtEq => GT_EQ,
471                LogosToken::AddAssign => PLUS_EQ,
472                LogosToken::SubAssign => MINUS_EQ,
473                LogosToken::MulAssign => STAR_EQ,
474                LogosToken::DivAssign => SLASH_EQ,
475                LogosToken::RemAssign => PERCENT_EQ,
476                LogosToken::BitAndAssign => AMP_EQ,
477                LogosToken::BitOrAssign => PIPE_EQ,
478                LogosToken::BitXorAssign => CARET_EQ,
479                LogosToken::Arrow => ARROW,
480                LogosToken::FatArrow => FAT_ARROW,
481                LogosToken::DotDotEq => DOT_DOT_EQ,
482                LogosToken::DotDot => DOT_DOT,
483                LogosToken::ColonColon => COLON_COLON,
484
485                // Single-char operators
486                LogosToken::Eq => EQ,
487                LogosToken::Bang => BANG,
488                LogosToken::Lt => LT,
489                LogosToken::Gt => GT,
490                LogosToken::Plus => PLUS,
491                LogosToken::Minus => MINUS,
492                LogosToken::Star => STAR,
493                LogosToken::Slash => SLASH,
494                LogosToken::Percent => PERCENT,
495                LogosToken::Amp => AMP,
496                LogosToken::Pipe => PIPE,
497                LogosToken::Caret => CARET,
498
499                // Punctuation
500                LogosToken::LParen => L_PAREN,
501                LogosToken::RParen => R_PAREN,
502                LogosToken::LBracket => L_BRACKET,
503                LogosToken::RBracket => R_BRACKET,
504                LogosToken::LBrace => L_BRACE,
505                LogosToken::RBrace => R_BRACE,
506                LogosToken::Comma => COMMA,
507                LogosToken::Dot => DOT,
508                LogosToken::Semicolon => SEMICOLON,
509                LogosToken::Colon => COLON,
510                LogosToken::Question => QUESTION,
511                LogosToken::Underscore => UNDERSCORE,
512                LogosToken::At => AT,
513
514                // Security: bidi characters
515                LogosToken::Bidi => {
516                    errors.push(LexError {
517                        range: TextRange::new(TextSize::new(span.start as u32), TextSize::new(span.end as u32)),
518                        kind: LexErrorKind::BidiOverride,
519                    });
520                    ERROR
521                }
522            },
523            Err(()) => {
524                errors.push(LexError {
525                    range: TextRange::new(TextSize::new(span.start as u32), TextSize::new(span.end as u32)),
526                    kind: LexErrorKind::CouldNotLex { content: slice.to_string() },
527                });
528                ERROR
529            }
530        };
531
532        // Validate integer literal digits for the appropriate radix
533        if kind == INTEGER {
534            validate_integer_digits(slice, span.start, &mut errors);
535        }
536
537        tokens.push(Token { kind, len });
538    }
539
540    // Add EOF token
541    tokens.push(Token { kind: EOF, len: 0 });
542
543    (tokens, errors)
544}
545
546#[cfg(test)]
547mod tests {
548    use super::*;
549    use expect_test::{Expect, expect};
550
551    /// Helper to format tokens for snapshot testing.
552    fn check_lex(input: &str, expect: Expect) {
553        let (tokens, _errors) = lex(input);
554        let mut output = String::new();
555        let mut offset = 0usize;
556        for token in &tokens {
557            let text = &input[offset..offset + token.len as usize];
558            output.push_str(&format!("{:?} {:?}\n", token.kind, text));
559            offset += token.len as usize;
560        }
561        expect.assert_eq(&output);
562    }
563
564    /// Helper to check that lexing produces expected errors.
565    fn check_lex_errors(input: &str, expect: Expect) {
566        let (_tokens, errors) = lex(input);
567        let output = errors
568            .iter()
569            .map(|e| format!("{}..{}:{:?}", u32::from(e.range.start()), u32::from(e.range.end()), e.kind))
570            .collect::<Vec<_>>()
571            .join("\n");
572        expect.assert_eq(&output);
573    }
574
575    #[test]
576    fn lex_empty() {
577        check_lex("", expect![[r#"
578            EOF ""
579        "#]]);
580    }
581
582    #[test]
583    fn lex_whitespace() {
584        check_lex("  \t  ", expect![[r#"
585            WHITESPACE "  \t  "
586            EOF ""
587        "#]]);
588    }
589
590    #[test]
591    fn lex_linebreaks() {
592        check_lex("\n\r\n\n", expect![[r#"
593            LINEBREAK "\n"
594            LINEBREAK "\r\n"
595            LINEBREAK "\n"
596            EOF ""
597"#]]);
598    }
599
600    #[test]
601    fn lex_mixed_whitespace() {
602        check_lex("  \n  \t\n", expect![[r#"
603            WHITESPACE "  "
604            LINEBREAK "\n"
605            WHITESPACE "  \t"
606            LINEBREAK "\n"
607            EOF ""
608        "#]]);
609    }
610
611    #[test]
612    fn lex_line_comments() {
613        check_lex("// hello\n// world", expect![[r#"
614            COMMENT_LINE "// hello"
615            LINEBREAK "\n"
616            COMMENT_LINE "// world"
617            EOF ""
618        "#]]);
619    }
620
621    #[test]
622    fn lex_block_comments() {
623        check_lex("/* hello */ /* multi\nline */", expect![[r#"
624            COMMENT_BLOCK "/* hello */"
625            WHITESPACE " "
626            COMMENT_BLOCK "/* multi\nline */"
627            EOF ""
628        "#]]);
629    }
630
631    #[test]
632    fn lex_identifiers() {
633        check_lex("foo Bar _baz x123", expect![[r#"
634            IDENT "foo"
635            WHITESPACE " "
636            IDENT "Bar"
637            WHITESPACE " "
638            IDENT "_baz"
639            WHITESPACE " "
640            IDENT "x123"
641            EOF ""
642        "#]]);
643    }
644
645    #[test]
646    fn lex_keywords() {
647        check_lex("let fn if return true false", expect![[r#"
648            KW_LET "let"
649            WHITESPACE " "
650            KW_FN "fn"
651            WHITESPACE " "
652            KW_IF "if"
653            WHITESPACE " "
654            KW_RETURN "return"
655            WHITESPACE " "
656            KW_TRUE "true"
657            WHITESPACE " "
658            KW_FALSE "false"
659            EOF ""
660        "#]]);
661    }
662
663    #[test]
664    fn lex_type_keywords() {
665        check_lex("u8 u16 u32 u64 u128 i8 i16 i32 i64 i128", expect![[r#"
666            KW_U8 "u8"
667            WHITESPACE " "
668            KW_U16 "u16"
669            WHITESPACE " "
670            KW_U32 "u32"
671            WHITESPACE " "
672            KW_U64 "u64"
673            WHITESPACE " "
674            KW_U128 "u128"
675            WHITESPACE " "
676            KW_I8 "i8"
677            WHITESPACE " "
678            KW_I16 "i16"
679            WHITESPACE " "
680            KW_I32 "i32"
681            WHITESPACE " "
682            KW_I64 "i64"
683            WHITESPACE " "
684            KW_I128 "i128"
685            EOF ""
686        "#]]);
687    }
688
689    #[test]
690    fn lex_more_type_keywords() {
691        check_lex("bool field group scalar address signature string record", expect![[r#"
692            KW_BOOL "bool"
693            WHITESPACE " "
694            KW_FIELD "field"
695            WHITESPACE " "
696            KW_GROUP "group"
697            WHITESPACE " "
698            KW_SCALAR "scalar"
699            WHITESPACE " "
700            KW_ADDRESS "address"
701            WHITESPACE " "
702            KW_SIGNATURE "signature"
703            WHITESPACE " "
704            KW_STRING "string"
705            WHITESPACE " "
706            KW_RECORD "record"
707            EOF ""
708        "#]]);
709    }
710
711    #[test]
712    fn lex_identifier_literal() {
713        check_lex("'foo' 'bar_baz' 'x'", expect![[r#"
714            IDENT_LIT "'foo'"
715            WHITESPACE " "
716            IDENT_LIT "'bar_baz'"
717            WHITESPACE " "
718            IDENT_LIT "'x'"
719            EOF ""
720        "#]]);
721    }
722
723    #[test]
724    fn lex_identifier_keyword() {
725        check_lex("identifier", expect![[r#"
726            KW_IDENTIFIER "identifier"
727            EOF ""
728        "#]]);
729    }
730
731    #[test]
732    fn keyword_predicate_matches_identifier_keywords() {
733        assert!(super::is_keyword("view"));
734        assert!(super::is_keyword("dyn"));
735        assert!(super::is_keyword("in"));
736        assert!(!super::is_keyword("inside"));
737    }
738
739    #[test]
740    fn lex_integers() {
741        check_lex("123 0xFF 0b101 0o77", expect![[r#"
742            INTEGER "123"
743            WHITESPACE " "
744            INTEGER "0xFF"
745            WHITESPACE " "
746            INTEGER "0b101"
747            WHITESPACE " "
748            INTEGER "0o77"
749            EOF ""
750        "#]]);
751    }
752
753    #[test]
754    fn lex_integers_with_underscores() {
755        check_lex("1_000_000 0xFF_FF", expect![[r#"
756            INTEGER "1_000_000"
757            WHITESPACE " "
758            INTEGER "0xFF_FF"
759            EOF ""
760        "#]]);
761    }
762
763    #[test]
764    fn lex_address_literal() {
765        check_lex("aleo1abc123", expect![[r#"
766            ADDRESS_LIT "aleo1abc123"
767            EOF ""
768        "#]]);
769    }
770
771    #[test]
772    fn lex_strings() {
773        check_lex(r#""hello" "world""#, expect![[r#"
774            STRING "\"hello\""
775            WHITESPACE " "
776            STRING "\"world\""
777            EOF ""
778        "#]]);
779    }
780
781    #[test]
782    fn lex_punctuation() {
783        check_lex("( ) [ ] { } , . ; : :: ? -> => _ @", expect![[r#"
784            L_PAREN "("
785            WHITESPACE " "
786            R_PAREN ")"
787            WHITESPACE " "
788            L_BRACKET "["
789            WHITESPACE " "
790            R_BRACKET "]"
791            WHITESPACE " "
792            L_BRACE "{"
793            WHITESPACE " "
794            R_BRACE "}"
795            WHITESPACE " "
796            COMMA ","
797            WHITESPACE " "
798            DOT "."
799            WHITESPACE " "
800            SEMICOLON ";"
801            WHITESPACE " "
802            COLON ":"
803            WHITESPACE " "
804            COLON_COLON "::"
805            WHITESPACE " "
806            QUESTION "?"
807            WHITESPACE " "
808            ARROW "->"
809            WHITESPACE " "
810            FAT_ARROW "=>"
811            WHITESPACE " "
812            UNDERSCORE "_"
813            WHITESPACE " "
814            AT "@"
815            EOF ""
816        "#]]);
817    }
818
819    #[test]
820    fn lex_arithmetic_operators() {
821        check_lex("+ - * / % **", expect![[r#"
822            PLUS "+"
823            WHITESPACE " "
824            MINUS "-"
825            WHITESPACE " "
826            STAR "*"
827            WHITESPACE " "
828            SLASH "/"
829            WHITESPACE " "
830            PERCENT "%"
831            WHITESPACE " "
832            STAR2 "**"
833            EOF ""
834        "#]]);
835    }
836
837    #[test]
838    fn lex_comparison_operators() {
839        check_lex("== != < <= > >=", expect![[r#"
840            EQ2 "=="
841            WHITESPACE " "
842            BANG_EQ "!="
843            WHITESPACE " "
844            LT "<"
845            WHITESPACE " "
846            LT_EQ "<="
847            WHITESPACE " "
848            GT ">"
849            WHITESPACE " "
850            GT_EQ ">="
851            EOF ""
852        "#]]);
853    }
854
855    #[test]
856    fn lex_logical_operators() {
857        check_lex("&& || !", expect![[r#"
858            AMP2 "&&"
859            WHITESPACE " "
860            PIPE2 "||"
861            WHITESPACE " "
862            BANG "!"
863            EOF ""
864        "#]]);
865    }
866
867    #[test]
868    fn lex_bitwise_operators() {
869        check_lex("& | ^ << >>", expect![[r#"
870            AMP "&"
871            WHITESPACE " "
872            PIPE "|"
873            WHITESPACE " "
874            CARET "^"
875            WHITESPACE " "
876            SHL "<<"
877            WHITESPACE " "
878            SHR ">>"
879            EOF ""
880        "#]]);
881    }
882
883    #[test]
884    fn lex_assignment_operators() {
885        check_lex("= += -= *= /= %= **= &&= ||=", expect![[r#"
886            EQ "="
887            WHITESPACE " "
888            PLUS_EQ "+="
889            WHITESPACE " "
890            MINUS_EQ "-="
891            WHITESPACE " "
892            STAR_EQ "*="
893            WHITESPACE " "
894            SLASH_EQ "/="
895            WHITESPACE " "
896            PERCENT_EQ "%="
897            WHITESPACE " "
898            STAR2_EQ "**="
899            WHITESPACE " "
900            AMP2_EQ "&&="
901            WHITESPACE " "
902            PIPE2_EQ "||="
903            EOF ""
904        "#]]);
905    }
906
907    #[test]
908    fn lex_more_assignment_operators() {
909        check_lex("&= |= ^= <<= >>=", expect![[r#"
910            AMP_EQ "&="
911            WHITESPACE " "
912            PIPE_EQ "|="
913            WHITESPACE " "
914            CARET_EQ "^="
915            WHITESPACE " "
916            SHL_EQ "<<="
917            WHITESPACE " "
918            SHR_EQ ">>="
919            EOF ""
920        "#]]);
921    }
922
923    #[test]
924    fn lex_dot_dot() {
925        check_lex("0..10", expect![[r#"
926            INTEGER "0"
927            DOT_DOT ".."
928            INTEGER "10"
929            EOF ""
930        "#]]);
931    }
932
933    #[test]
934    fn lex_simple_expression() {
935        check_lex("x + y * 2", expect![[r#"
936            IDENT "x"
937            WHITESPACE " "
938            PLUS "+"
939            WHITESPACE " "
940            IDENT "y"
941            WHITESPACE " "
942            STAR "*"
943            WHITESPACE " "
944            INTEGER "2"
945            EOF ""
946        "#]]);
947    }
948
949    #[test]
950    fn lex_function_call() {
951        check_lex("foo(a, b)", expect![[r#"
952            IDENT "foo"
953            L_PAREN "("
954            IDENT "a"
955            COMMA ","
956            WHITESPACE " "
957            IDENT "b"
958            R_PAREN ")"
959            EOF ""
960        "#]]);
961    }
962
963    #[test]
964    fn lex_function_definition() {
965        check_lex("fn add(x: u32) -> u32 {", expect![[r#"
966            KW_FN "fn"
967            WHITESPACE " "
968            IDENT "add"
969            L_PAREN "("
970            IDENT "x"
971            COLON ":"
972            WHITESPACE " "
973            KW_U32 "u32"
974            R_PAREN ")"
975            WHITESPACE " "
976            ARROW "->"
977            WHITESPACE " "
978            KW_U32 "u32"
979            WHITESPACE " "
980            L_BRACE "{"
981            EOF ""
982        "#]]);
983    }
984
985    #[test]
986    fn lex_let_statement() {
987        check_lex("let x: u32 = 42;", expect![[r#"
988            KW_LET "let"
989            WHITESPACE " "
990            IDENT "x"
991            COLON ":"
992            WHITESPACE " "
993            KW_U32 "u32"
994            WHITESPACE " "
995            EQ "="
996            WHITESPACE " "
997            INTEGER "42"
998            SEMICOLON ";"
999            EOF ""
1000        "#]]);
1001    }
1002
1003    #[test]
1004    fn lex_typed_integers() {
1005        // Integer literals with type suffixes should be lexed as single tokens
1006        check_lex("1000u32 42i64 0u8 255u128", expect![[r#"
1007            INTEGER "1000u32"
1008            WHITESPACE " "
1009            INTEGER "42i64"
1010            WHITESPACE " "
1011            INTEGER "0u8"
1012            WHITESPACE " "
1013            INTEGER "255u128"
1014            EOF ""
1015        "#]]);
1016    }
1017
1018    #[test]
1019    fn lex_typed_integers_field() {
1020        // Field, group, and scalar suffixes
1021        check_lex("123field 456group 789scalar", expect![[r#"
1022            INTEGER "123field"
1023            WHITESPACE " "
1024            INTEGER "456group"
1025            WHITESPACE " "
1026            INTEGER "789scalar"
1027            EOF ""
1028        "#]]);
1029    }
1030
1031    #[test]
1032    fn lex_special_paths() {
1033        // These are special cases where keywords are followed by ::
1034        check_lex("group::GEN signature::verify Future::await", expect![[r#"
1035            IDENT "group::GEN"
1036            WHITESPACE " "
1037            IDENT "signature::verify"
1038            WHITESPACE " "
1039            IDENT "Future::await"
1040            EOF ""
1041        "#]]);
1042    }
1043
1044    #[test]
1045    fn lex_typed_integer_range() {
1046        // Integer with type suffix followed by range operator
1047        check_lex("0u8..STOP", expect![[r#"
1048            INTEGER "0u8"
1049            DOT_DOT ".."
1050            IDENT "STOP"
1051            EOF ""
1052        "#]]);
1053    }
1054
1055    #[test]
1056    fn lex_error_unknown_char() {
1057        check_lex_errors("hello $ world", expect![[r#"6..7:CouldNotLex { content: "$" }"#]]);
1058    }
1059
1060    #[test]
1061    fn lex_invalid_hex_digit() {
1062        let (tokens, errors) = lex("0xGAu32");
1063        assert_eq!(tokens.len(), 2); // INTEGER + EOF
1064        assert!(!errors.is_empty());
1065        assert!(matches!(errors[0].kind, LexErrorKind::InvalidDigit { digit: 'G', radix: 16, .. }));
1066    }
1067
1068    #[test]
1069    fn lex_invalid_octal_digit() {
1070        let (_, errors) = lex("0o9u32");
1071        assert!(!errors.is_empty());
1072        assert!(matches!(errors[0].kind, LexErrorKind::InvalidDigit { digit: '9', radix: 8, .. }));
1073    }
1074
1075    #[test]
1076    fn lex_invalid_binary_digit() {
1077        let (_, errors) = lex("0b2u32");
1078        assert!(!errors.is_empty());
1079        assert!(matches!(errors[0].kind, LexErrorKind::InvalidDigit { digit: '2', radix: 2, .. }));
1080    }
1081
1082    #[test]
1083    fn lex_valid_hex_is_ok() {
1084        let (_, errors) = lex("0xDEADBEEFu64");
1085        assert!(errors.is_empty());
1086    }
1087
1088    #[test]
1089    fn lex_invalid_hex_lowercase() {
1090        // Lowercase hex digits beyond f are invalid
1091        let (_, errors) = lex("0xghu32");
1092        assert!(!errors.is_empty());
1093        assert!(matches!(errors[0].kind, LexErrorKind::InvalidDigit { digit: 'g', radix: 16, .. }));
1094    }
1095
1096    #[test]
1097    fn lex_bidi_override_error() {
1098        let (_, errors) = lex("let x\u{202E} = 1;");
1099        assert!(!errors.is_empty());
1100        assert!(matches!(errors[0].kind, LexErrorKind::BidiOverride));
1101    }
1102
1103    #[test]
1104    fn lex_unclosed_block_comment() {
1105        let (tokens, errors) = lex("/* unclosed");
1106        assert!(!errors.is_empty());
1107        assert!(matches!(errors[0].kind, LexErrorKind::CouldNotLex { .. }));
1108        // Should still produce a COMMENT_BLOCK token
1109        assert!(tokens.iter().any(|t| t.kind == COMMENT_BLOCK));
1110    }
1111
1112    #[test]
1113    fn lex_nested_comment_not_supported() {
1114        // Leo doesn't support nested comments - the first */ closes the comment
1115        let (tokens, errors) = lex("/* outer /* inner */");
1116        // No errors - the first */ terminates the comment
1117        assert!(errors.is_empty());
1118        // The "outer /* inner " part is the comment, then "*/" closes it
1119        // But actually let's check what happens...
1120        // "/* outer /* inner */" - this finds the first */ which is at position 18
1121        // So the comment is "/* outer /* inner */" which IS terminated
1122        assert!(tokens.iter().any(|t| t.kind == COMMENT_BLOCK));
1123    }
1124
1125    #[test]
1126    fn lex_closed_comment_ok() {
1127        let (_, errors) = lex("/* closed */");
1128        assert!(errors.is_empty());
1129    }
1130}
leo_parser_rowan/lexer.rs

leo_parser_rowan/
lexer.rs
