pine_lexer/
lib.rs

1use pine_core::PineVersion;
2use thiserror::Error;
3
4#[derive(Debug, Error)]
5pub enum LexerErrorKind {
6    #[error("Unterminated string")]
7    UnterminatedString,
8
9    #[error("Invalid hex color format '{value}'")]
10    InvalidHexColor { value: String },
11
12    #[error("Unexpected character '{ch}'")]
13    UnexpectedCharacter { ch: char },
14
15    #[error("Indentation error")]
16    IndentationError,
17
18    #[error("Invalid number '{value}'")]
19    InvalidNumber { value: String },
20}
21
22/// A lexing error and the 1-based source position it points at.
23#[derive(Debug, Error)]
24#[error("{kind} at line {line}, column {column}")]
25pub struct LexerError {
26    pub line: usize,
27    pub column: usize,
28    pub kind: LexerErrorKind,
29}
30
31impl LexerError {
32    /// The 1-based `(line, column)` of the offending character.
33    pub fn location(&self) -> (u32, u32) {
34        (self.line as u32, self.column as u32)
35    }
36}
37
38// Token types
39#[derive(Debug, Clone, PartialEq)]
40pub enum TokenType {
41    IntLiteral(i64),
42    Number(f64),
43    String(String),
44    Bool(bool),
45    HexColor(String), // #RRGGBB or #RRGGBBAA
46
47    /// A `//` comment's text, verbatim and without the leading `//`. Emitted as
48    /// trivia for tools (the formatter); the parser filters it out.
49    Comment(String),
50    /// An empty source line, emitted as trivia so tools can preserve paragraph
51    /// breaks; the parser filters it out.
52    BlankLine,
53
54    // Identifiers and keywords
55    Ident(String),
56    Var,
57    Varip,
58    Const,
59    Type,
60    Enum,
61    Method,
62    Export,
63    Import,
64    If,
65    Else,
66    For,
67    While,
68    Break,
69    Continue,
70    To,
71    In,
72    Switch, // keywords
73    Int,
74    Float, // type keywords
75    Na,    // special value
76    And,
77    Or,
78    Not, // logical operators
79
80    // Operators
81    Plus,
82    Minus,
83    Star,
84    Slash,
85    Percent,
86    Equal,
87    NotEqual,
88    Less,
89    Greater,
90    LessEqual,
91    GreaterEqual,
92    Assign,
93    ColonAssign,
94    Arrow, // =, :=, =>
95    PlusAssign,
96    MinusAssign,
97    StarAssign,
98    SlashAssign, // +=, -=, *=, /=
99
100    // Delimiters
101    LParen,
102    RParen,
103    LBracket,
104    RBracket,
105    Comma,
106    Dot,
107    Colon,
108    Question,
109    Newline,
110    Indent,
111    Dedent,
112
113    Eof,
114}
115
116#[derive(Debug, Clone)]
117pub struct Token {
118    pub typ: TokenType,
119    pub lexeme: String,
120    pub line: usize,
121    pub column: usize,
122}
123
124pub struct Lexer {
125    input: Vec<char>,
126    current: usize,
127    line: usize,
128    column: usize,
129    indent_stack: Vec<usize>,   // Stack of indentation levels
130    pending_tokens: Vec<Token>, // Queue for Indent/Dedent tokens
131    paren_depth: usize,         // Open parentheses; while > 0, layout tokens are suppressed
132    version: PineVersion,       // Decides which words are keywords rather than identifiers
133}
134
135impl Lexer {
136    /// Lex as [`PineVersion::LATEST`]. Use [`Lexer::with_version`] when the
137    /// script's `//@version=` has already been read — some words are only
138    /// keywords in later versions.
139    pub fn new(input: &str) -> Self {
140        Self::with_version(input, PineVersion::LATEST)
141    }
142
143    pub fn with_version(input: &str, version: PineVersion) -> Self {
144        Self {
145            input: input.chars().collect(),
146            current: 0,
147            line: 1,
148            column: 1,
149            indent_stack: vec![0], // Start with base indentation level
150            pending_tokens: vec![],
151            paren_depth: 0,
152            version,
153        }
154    }
155
156    fn peek(&self) -> Option<char> {
157        self.input.get(self.current).copied()
158    }
159
160    fn advance(&mut self) -> Option<char> {
161        let ch = self.peek()?;
162        self.current += 1;
163        if ch == '\n' {
164            self.line += 1;
165            self.column = 1;
166        } else {
167            self.column += 1;
168        }
169        Some(ch)
170    }
171
172    fn skip_whitespace(&mut self) {
173        while let Some(ch) = self.peek() {
174            if ch == ' ' || ch == '\t' || ch == '\r' {
175                self.advance();
176            } else {
177                break;
178            }
179        }
180    }
181
182    fn scan_number(&mut self) -> Result<Token, LexerError> {
183        let start_line = self.line;
184        let start_col = self.column;
185        let mut num_str = String::new();
186
187        // Handle numbers starting with '.' like .5 or .088
188        if self.peek() == Some('.') {
189            num_str.push('.');
190            self.advance();
191        }
192
193        while let Some(ch) = self.peek() {
194            if ch.is_numeric() {
195                num_str.push(ch);
196                self.advance();
197            } else if ch == '.' && !num_str.contains('.') {
198                // Only consume '.' if we haven't seen one yet and it's followed by a digit
199                if let Some(next_ch) = self.input.get(self.current + 1) {
200                    if next_ch.is_numeric() {
201                        num_str.push(ch);
202                        self.advance();
203                    } else {
204                        break;
205                    }
206                } else {
207                    break;
208                }
209            } else {
210                break;
211            }
212        }
213
214        // No decimal point means an integer literal; Pine treats the two types
215        // differently. (This lexer does not read scientific notation, so a `.`
216        // is the only thing that makes a literal a float.)
217        let typ = if num_str.contains('.') {
218            TokenType::Number(num_str.parse::<f64>().map_err(|_| LexerError {
219                line: start_line,
220                column: start_col,
221                kind: LexerErrorKind::InvalidNumber {
222                    value: num_str.clone(),
223                },
224            })?)
225        } else {
226            TokenType::IntLiteral(num_str.parse::<i64>().map_err(|_| LexerError {
227                line: start_line,
228                column: start_col,
229                kind: LexerErrorKind::InvalidNumber {
230                    value: num_str.clone(),
231                },
232            })?)
233        };
234        Ok(Token {
235            typ,
236            lexeme: num_str,
237            line: start_line,
238            column: start_col,
239        })
240    }
241
242    fn scan_identifier(&mut self) -> Token {
243        let start_line = self.line;
244        let start_col = self.column;
245        let mut ident = String::new();
246
247        while let Some(ch) = self.peek() {
248            if ch.is_alphanumeric() || ch == '_' {
249                ident.push(ch);
250                self.advance();
251            } else {
252                break;
253            }
254        }
255
256        // Check for keywords
257        let typ = match ident.as_str() {
258            "var" => TokenType::Var,
259            "varip" => TokenType::Varip,
260            "const" => TokenType::Const,
261            // `type` introduces a user-defined type from v5 on; before that it
262            // is an ordinary name, and scripts do use it as one.
263            "type" if self.version >= PineVersion::V5 => TokenType::Type,
264            "enum" => TokenType::Enum,
265            "method" => TokenType::Method,
266            "export" => TokenType::Export,
267            "import" => TokenType::Import,
268            "if" => TokenType::If,
269            "else" => TokenType::Else,
270            "true" => TokenType::Bool(true),
271            "false" => TokenType::Bool(false),
272            "for" => TokenType::For,
273            "while" => TokenType::While,
274            "break" => TokenType::Break,
275            "continue" => TokenType::Continue,
276            "to" => TokenType::To,
277            "in" => TokenType::In,
278            "switch" => TokenType::Switch,
279            "int" => TokenType::Int,
280            "float" => TokenType::Float,
281            "na" => TokenType::Na,
282            "and" => TokenType::And,
283            "or" => TokenType::Or,
284            "not" => TokenType::Not,
285            _ => TokenType::Ident(ident.clone()),
286        };
287
288        Token {
289            typ,
290            lexeme: ident,
291            line: start_line,
292            column: start_col,
293        }
294    }
295
296    fn scan_string(&mut self, quote_char: char) -> Result<Token, LexerError> {
297        let start_line = self.line;
298        let start_col = self.column;
299
300        self.advance(); // consume opening quote
301        let mut string = String::new();
302
303        while let Some(ch) = self.peek() {
304            if ch == quote_char {
305                self.advance();
306                return Ok(Token {
307                    typ: TokenType::String(string.clone()),
308                    lexeme: format!("{}{}{}", quote_char, string, quote_char),
309                    line: start_line,
310                    column: start_col,
311                });
312            } else if ch == '\\' {
313                self.advance();
314                if let Some(escaped) = self.advance() {
315                    string.push(match escaped {
316                        'n' => '\n',
317                        't' => '\t',
318                        '"' => '"',
319                        '\'' => '\'',
320                        '\\' => '\\',
321                        _ => escaped,
322                    });
323                }
324            } else {
325                string.push(ch);
326                self.advance();
327            }
328        }
329
330        Err(LexerError {
331            line: start_line,
332            column: start_col,
333            kind: LexerErrorKind::UnterminatedString,
334        })
335    }
336
337    fn scan_hex_color(&mut self) -> Result<Token, LexerError> {
338        let start_line = self.line;
339        let start_col = self.column;
340
341        self.advance(); // consume '#'
342        let mut hex = String::from("#");
343
344        // Hex color format: #RRGGBB or #RRGGBBAA (6 or 8 hex digits)
345        while let Some(ch) = self.peek() {
346            if ch.is_ascii_hexdigit() {
347                hex.push(ch);
348                self.advance();
349            } else {
350                break;
351            }
352        }
353
354        // Validate length (should be 6 or 8 hex digits after #)
355        let hex_len = hex.len() - 1;
356        if hex_len != 6 && hex_len != 8 {
357            return Err(LexerError {
358                line: start_line,
359                column: start_col,
360                kind: LexerErrorKind::InvalidHexColor { value: hex },
361            });
362        }
363
364        Ok(Token {
365            typ: TokenType::HexColor(hex.clone()),
366            lexeme: hex,
367            line: start_line,
368            column: start_col,
369        })
370    }
371
372    fn next_token(&mut self) -> Result<Token, LexerError> {
373        self.skip_whitespace();
374
375        let ch = match self.peek() {
376            Some(c) => c,
377            None => {
378                return Ok(Token {
379                    typ: TokenType::Eof,
380                    lexeme: String::new(),
381                    line: self.line,
382                    column: self.column,
383                });
384            }
385        };
386
387        let line = self.line;
388        let col = self.column;
389
390        let token = match ch {
391            '+' => {
392                self.advance();
393                if self.peek() == Some('=') {
394                    self.advance();
395                    Token {
396                        typ: TokenType::PlusAssign,
397                        lexeme: "+=".to_string(),
398                        line,
399                        column: col,
400                    }
401                } else {
402                    Token {
403                        typ: TokenType::Plus,
404                        lexeme: "+".to_string(),
405                        line,
406                        column: col,
407                    }
408                }
409            }
410            '-' => {
411                self.advance();
412                if self.peek() == Some('=') {
413                    self.advance();
414                    Token {
415                        typ: TokenType::MinusAssign,
416                        lexeme: "-=".to_string(),
417                        line,
418                        column: col,
419                    }
420                } else {
421                    Token {
422                        typ: TokenType::Minus,
423                        lexeme: "-".to_string(),
424                        line,
425                        column: col,
426                    }
427                }
428            }
429            '*' => {
430                self.advance();
431                if self.peek() == Some('=') {
432                    self.advance();
433                    Token {
434                        typ: TokenType::StarAssign,
435                        lexeme: "*=".to_string(),
436                        line,
437                        column: col,
438                    }
439                } else {
440                    Token {
441                        typ: TokenType::Star,
442                        lexeme: "*".to_string(),
443                        line,
444                        column: col,
445                    }
446                }
447            }
448            '/' => {
449                self.advance();
450                if self.peek() == Some('/') {
451                    self.advance(); // consume the second '/'
452                    let mut text = String::new();
453                    while self.peek().is_some() && self.peek() != Some('\n') {
454                        text.push(self.advance().expect("peeked Some"));
455                    }
456                    Token {
457                        typ: TokenType::Comment(text.clone()),
458                        lexeme: format!("//{text}"),
459                        line,
460                        column: col,
461                    }
462                } else if self.peek() == Some('=') {
463                    self.advance();
464                    Token {
465                        typ: TokenType::SlashAssign,
466                        lexeme: "/=".to_string(),
467                        line,
468                        column: col,
469                    }
470                } else {
471                    Token {
472                        typ: TokenType::Slash,
473                        lexeme: "/".to_string(),
474                        line,
475                        column: col,
476                    }
477                }
478            }
479            '%' => {
480                self.advance();
481                Token {
482                    typ: TokenType::Percent,
483                    lexeme: "%".to_string(),
484                    line,
485                    column: col,
486                }
487            }
488            '=' => {
489                self.advance();
490                if self.peek() == Some('=') {
491                    self.advance();
492                    Token {
493                        typ: TokenType::Equal,
494                        lexeme: "==".to_string(),
495                        line,
496                        column: col,
497                    }
498                } else if self.peek() == Some('>') {
499                    self.advance();
500                    Token {
501                        typ: TokenType::Arrow,
502                        lexeme: "=>".to_string(),
503                        line,
504                        column: col,
505                    }
506                } else {
507                    Token {
508                        typ: TokenType::Assign,
509                        lexeme: "=".to_string(),
510                        line,
511                        column: col,
512                    }
513                }
514            }
515            '!' => {
516                self.advance();
517                if self.peek() == Some('=') {
518                    self.advance();
519                    Token {
520                        typ: TokenType::NotEqual,
521                        lexeme: "!=".to_string(),
522                        line,
523                        column: col,
524                    }
525                } else {
526                    return Err(LexerError {
527                        line,
528                        column: col,
529                        kind: LexerErrorKind::UnexpectedCharacter { ch: '!' },
530                    });
531                }
532            }
533            '<' => {
534                self.advance();
535                if self.peek() == Some('=') {
536                    self.advance();
537                    Token {
538                        typ: TokenType::LessEqual,
539                        lexeme: "<=".to_string(),
540                        line,
541                        column: col,
542                    }
543                } else {
544                    Token {
545                        typ: TokenType::Less,
546                        lexeme: "<".to_string(),
547                        line,
548                        column: col,
549                    }
550                }
551            }
552            '>' => {
553                self.advance();
554                if self.peek() == Some('=') {
555                    self.advance();
556                    Token {
557                        typ: TokenType::GreaterEqual,
558                        lexeme: ">=".to_string(),
559                        line,
560                        column: col,
561                    }
562                } else {
563                    Token {
564                        typ: TokenType::Greater,
565                        lexeme: ">".to_string(),
566                        line,
567                        column: col,
568                    }
569                }
570            }
571            '(' => {
572                self.advance();
573                Token {
574                    typ: TokenType::LParen,
575                    lexeme: "(".to_string(),
576                    line,
577                    column: col,
578                }
579            }
580            ')' => {
581                self.advance();
582                Token {
583                    typ: TokenType::RParen,
584                    lexeme: ")".to_string(),
585                    line,
586                    column: col,
587                }
588            }
589            '[' => {
590                self.advance();
591                Token {
592                    typ: TokenType::LBracket,
593                    lexeme: "[".to_string(),
594                    line,
595                    column: col,
596                }
597            }
598            ']' => {
599                self.advance();
600                Token {
601                    typ: TokenType::RBracket,
602                    lexeme: "]".to_string(),
603                    line,
604                    column: col,
605                }
606            }
607            ',' => {
608                self.advance();
609                Token {
610                    typ: TokenType::Comma,
611                    lexeme: ",".to_string(),
612                    line,
613                    column: col,
614                }
615            }
616            '.' => {
617                // Check if this is a decimal number like .5 or .088
618                if let Some(next_ch) = self.input.get(self.current + 1) {
619                    if next_ch.is_numeric() {
620                        // This is a decimal number starting with .
621                        return self.scan_number();
622                    }
623                }
624                self.advance();
625                Token {
626                    typ: TokenType::Dot,
627                    lexeme: ".".to_string(),
628                    line,
629                    column: col,
630                }
631            }
632            ':' => {
633                self.advance();
634                if self.peek() == Some('=') {
635                    self.advance();
636                    Token {
637                        typ: TokenType::ColonAssign,
638                        lexeme: ":=".to_string(),
639                        line,
640                        column: col,
641                    }
642                } else {
643                    Token {
644                        typ: TokenType::Colon,
645                        lexeme: ":".to_string(),
646                        line,
647                        column: col,
648                    }
649                }
650            }
651            '?' => {
652                self.advance();
653                Token {
654                    typ: TokenType::Question,
655                    lexeme: "?".to_string(),
656                    line,
657                    column: col,
658                }
659            }
660            '\n' => {
661                self.advance();
662                Token {
663                    typ: TokenType::Newline,
664                    lexeme: "\\n".to_string(),
665                    line,
666                    column: col,
667                }
668            }
669            '"' => return self.scan_string('"'),
670            '\'' => return self.scan_string('\''),
671            '#' => return self.scan_hex_color(),
672            _ if ch.is_numeric() => return self.scan_number(),
673            _ if ch.is_alphabetic() || ch == '_' => self.scan_identifier(),
674            _ => {
675                return Err(LexerError {
676                    line,
677                    column: col,
678                    kind: LexerErrorKind::UnexpectedCharacter { ch },
679                })
680            }
681        };
682
683        Ok(token)
684    }
685
686    pub fn tokenize(&mut self) -> Result<Vec<Token>, LexerError> {
687        let mut tokens = vec![];
688        let mut at_line_start = true;
689
690        loop {
691            // Check if we have pending tokens (Indent/Dedent)
692            if !self.pending_tokens.is_empty() {
693                tokens.push(self.pending_tokens.remove(0));
694                continue;
695            }
696
697            // Handle indentation at the start of a line
698            if at_line_start {
699                at_line_start = false;
700
701                // Skip blank lines and comments
702                let saved_line = self.line;
703                let saved_col = self.column;
704
705                // Count leading spaces
706                let mut indent_level = 0;
707                while let Some(ch) = self.peek() {
708                    if ch == ' ' {
709                        indent_level += 1;
710                        self.advance();
711                    } else if ch == '\t' {
712                        indent_level += 4; // Treat tab as 4 spaces
713                        self.advance();
714                    } else {
715                        break;
716                    }
717                }
718
719                // Check if this is a blank line or comment
720                if let Some(ch) = self.peek() {
721                    if ch == '\n' || ch == '\r' {
722                        // Blank line - emit trivia and skip the newline.
723                        tokens.push(Token {
724                            typ: TokenType::BlankLine,
725                            lexeme: String::new(),
726                            line: self.line,
727                            column: self.column,
728                        });
729                        self.advance();
730                        at_line_start = true;
731                        continue;
732                    } else if ch == '/' && self.peek_ahead(1) == Some('/') {
733                        // A whole-line comment: capture it as trivia without
734                        // touching the indent stack (its indentation is layout).
735                        let comment_line = self.line;
736                        let comment_col = self.column;
737                        self.advance();
738                        self.advance();
739                        let mut text = String::new();
740                        while let Some(c) = self.peek() {
741                            if c == '\n' {
742                                break;
743                            }
744                            text.push(c);
745                            self.advance();
746                        }
747                        tokens.push(Token {
748                            typ: TokenType::Comment(text.clone()),
749                            lexeme: format!("//{text}"),
750                            line: comment_line,
751                            column: comment_col,
752                        });
753                        if self.peek() == Some('\n') {
754                            self.advance();
755                        }
756                        at_line_start = true;
757                        continue;
758                    }
759                } else {
760                    // EOF - emit dedents for all remaining levels
761                    let current_line = self.line;
762                    let current_col = self.column;
763                    while self.indent_stack.len() > 1 {
764                        self.indent_stack.pop();
765                        tokens.push(Token {
766                            typ: TokenType::Dedent,
767                            lexeme: String::new(),
768                            line: current_line,
769                            column: current_col,
770                        });
771                    }
772                    tokens.push(Token {
773                        typ: TokenType::Eof,
774                        lexeme: String::new(),
775                        line: current_line,
776                        column: current_col,
777                    });
778                    break;
779                }
780
781                if self.paren_depth > 0 {
782                    // Inside parentheses, Pine allows a wrapped line to use any
783                    // indentation, including a multiple of 4. Ignore this line's
784                    // indentation entirely: emit no Indent/Dedent and leave the
785                    // indent stack untouched. The Newline that would have ended
786                    // the previous line is suppressed where it is produced.
787                } else if indent_level % 4 != 0 {
788                    // Pine line-wrapping: a line indented by a non-multiple of
789                    // 4 spaces continues the previous logical line (Pine
790                    // reserves 4-space multiples for local blocks). Join it to
791                    // the previous line: drop the Newline that ended it and
792                    // leave the indent stack untouched.
793                    if matches!(
794                        tokens.last(),
795                        Some(Token {
796                            typ: TokenType::Newline,
797                            ..
798                        })
799                    ) {
800                        tokens.pop();
801                    }
802                } else {
803                    // Handle indent/dedent
804                    // SAFETY: indent_stack is initialized with vec![0] and we never pop the last element
805                    let current_indent = *self.indent_stack.last().unwrap();
806                    let line = saved_line;
807                    let col = saved_col;
808
809                    if indent_level > current_indent {
810                        // Indent
811                        self.indent_stack.push(indent_level);
812                        tokens.push(Token {
813                            typ: TokenType::Indent,
814                            lexeme: String::new(),
815                            line,
816                            column: col,
817                        });
818                    } else if indent_level < current_indent {
819                        // Dedent - possibly multiple levels
820                        // SAFETY: checked by len() > 1
821                        while self.indent_stack.len() > 1
822                            && *self.indent_stack.last().unwrap() > indent_level
823                        {
824                            self.indent_stack.pop();
825                            tokens.push(Token {
826                                typ: TokenType::Dedent,
827                                lexeme: String::new(),
828                                line,
829                                column: col,
830                            });
831                        }
832
833                        // Check for indentation error
834                        // SAFETY: indent_stack always has at least one element
835                        if *self.indent_stack.last().unwrap() != indent_level {
836                            return Err(LexerError {
837                                line,
838                                column: col,
839                                kind: LexerErrorKind::IndentationError,
840                            });
841                        }
842                    }
843                }
844            }
845
846            // Get next token
847            let token = self.next_token()?;
848
849            // Track parenthesis nesting so layout tokens can be suppressed
850            // inside a parenthesised expression (Pine line-wrapping rule).
851            match token.typ {
852                TokenType::LParen => self.paren_depth += 1,
853                TokenType::RParen => self.paren_depth = self.paren_depth.saturating_sub(1),
854                _ => {}
855            }
856
857            // Check if this is a newline
858            if matches!(token.typ, TokenType::Newline) {
859                at_line_start = true;
860                // Inside parentheses a newline does not terminate the logical
861                // line, so drop it; the following line's indentation is ignored
862                // by the layout block above.
863                if self.paren_depth == 0 {
864                    tokens.push(token);
865                }
866            } else if matches!(token.typ, TokenType::Eof) {
867                // Emit dedents for all remaining levels
868                while self.indent_stack.len() > 1 {
869                    self.indent_stack.pop();
870                    tokens.push(Token {
871                        typ: TokenType::Dedent,
872                        lexeme: String::new(),
873                        line: token.line,
874                        column: token.column,
875                    });
876                }
877                tokens.push(token);
878                break;
879            } else {
880                tokens.push(token);
881            }
882        }
883
884        Ok(tokens)
885    }
886
887    fn peek_ahead(&self, offset: usize) -> Option<char> {
888        self.input.get(self.current + offset).copied()
889    }
890}
891
892#[cfg(test)]
893mod tests {
894    use super::*;
895
896    #[test]
897    fn test_line_wrapping_non_multiple_of_4_joins_lines() -> eyre::Result<()> {
898        // Pine line-wrapping rule: a line indented by a non-multiple of 4
899        // spaces continues the previous logical line (4-space multiples are
900        // reserved for local blocks). The wrapped lines must produce NO
901        // Newline before the continuation and NO Indent/Dedent tokens.
902        let mut lexer = Lexer::new("a = x < 2\n         and y\nb = 1");
903        let tokens = lexer.tokenize()?;
904        assert!(
905            !tokens
906                .iter()
907                .any(|t| matches!(t.typ, TokenType::Indent | TokenType::Dedent)),
908            "wrapped continuation must not emit Indent/Dedent: {:?}",
909            tokens.iter().map(|t| &t.typ).collect::<Vec<_>>()
910        );
911        // `a = x < 2 and y` must be one logical line: the only Newline comes
912        // after `y` (plus optionally after `b = 1`).
913        let and_pos = tokens
914            .iter()
915            .position(|t| matches!(t.typ, TokenType::And))
916            .expect("And token present");
917        assert!(
918            !tokens[..and_pos]
919                .iter()
920                .any(|t| matches!(t.typ, TokenType::Newline)),
921            "no Newline may precede the continuation's `and`: {:?}",
922            tokens.iter().map(|t| &t.typ).collect::<Vec<_>>()
923        );
924        Ok(())
925    }
926
927    #[test]
928    fn test_block_indent_multiple_of_4_still_indents() -> eyre::Result<()> {
929        let mut lexer = Lexer::new("if cond\n    x = 1\ny = 2");
930        let tokens = lexer.tokenize()?;
931        assert!(
932            tokens.iter().any(|t| matches!(t.typ, TokenType::Indent)),
933            "4-space block body must still emit Indent"
934        );
935        assert!(
936            tokens.iter().any(|t| matches!(t.typ, TokenType::Dedent)),
937            "return to column 0 must still emit Dedent"
938        );
939        Ok(())
940    }
941
942    #[test]
943    fn test_parens_suppress_layout_at_any_indent() -> eyre::Result<()> {
944        // Inside parentheses a wrapped line may use any indentation, including a
945        // multiple of 4. The whole call is one logical line: no Newline, Indent
946        // or Dedent appears between `(` and `)`.
947        let mut lexer = Lexer::new("plot(\n    a,\n        b\n)");
948        let tokens = lexer.tokenize()?;
949        assert!(matches!(&tokens[0].typ, TokenType::Ident(s) if s == "plot"));
950        assert!(matches!(tokens[1].typ, TokenType::LParen));
951        assert!(matches!(&tokens[2].typ, TokenType::Ident(s) if s == "a"));
952        assert!(matches!(tokens[3].typ, TokenType::Comma));
953        assert!(matches!(&tokens[4].typ, TokenType::Ident(s) if s == "b"));
954        assert!(matches!(tokens[5].typ, TokenType::RParen));
955        assert!(matches!(tokens[6].typ, TokenType::Eof));
956        Ok(())
957    }
958
959    #[test]
960    fn test_newline_after_closing_paren_terminates() -> eyre::Result<()> {
961        // The Newline after the closing paren still terminates the statement,
962        // so a following statement stays separate.
963        let mut lexer = Lexer::new("x = f(\n    a\n)\ny = 1");
964        let tokens = lexer.tokenize()?;
965        assert!(matches!(tokens[5].typ, TokenType::RParen));
966        assert!(matches!(tokens[6].typ, TokenType::Newline));
967        assert!(matches!(&tokens[7].typ, TokenType::Ident(s) if s == "y"));
968        Ok(())
969    }
970
971    #[test]
972    fn test_literals() -> eyre::Result<()> {
973        // Numbers
974        let mut lexer = Lexer::new("42 3.15");
975        let tokens = lexer.tokenize()?;
976        assert!(matches!(tokens[0].typ, TokenType::IntLiteral(n) if n == 42));
977        assert!(matches!(tokens[1].typ, TokenType::Number(n) if n == 3.15));
978
979        // Strings
980        let mut lexer = Lexer::new(r#""hello" "world\n""#);
981        let tokens = lexer.tokenize()?;
982        assert!(matches!(&tokens[0].typ, TokenType::String(s) if s == "hello"));
983        assert!(matches!(&tokens[1].typ, TokenType::String(s) if s == "world\n"));
984
985        // Booleans
986        let mut lexer = Lexer::new("true false");
987        let tokens = lexer.tokenize()?;
988        assert!(matches!(tokens[0].typ, TokenType::Bool(true)));
989        assert!(matches!(tokens[1].typ, TokenType::Bool(false)));
990        Ok(())
991    }
992
993    #[test]
994    fn test_identifiers_and_keywords() -> eyre::Result<()> {
995        let mut lexer = Lexer::new("my_var var if else for while int float na");
996        let tokens = lexer.tokenize()?;
997        assert!(matches!(&tokens[0].typ, TokenType::Ident(s) if s == "my_var"));
998        assert!(matches!(tokens[1].typ, TokenType::Var));
999        assert!(matches!(tokens[2].typ, TokenType::If));
1000        assert!(matches!(tokens[3].typ, TokenType::Else));
1001        assert!(matches!(tokens[4].typ, TokenType::For));
1002        assert!(matches!(tokens[5].typ, TokenType::While));
1003        assert!(matches!(tokens[6].typ, TokenType::Int));
1004        assert!(matches!(tokens[7].typ, TokenType::Float));
1005        assert!(matches!(tokens[8].typ, TokenType::Na));
1006        Ok(())
1007    }
1008
1009    #[test]
1010    fn test_type_is_a_keyword_only_from_v5() -> eyre::Result<()> {
1011        // v5 introduced user-defined types; before that `type` is just a name,
1012        // and v4 scripts do use it as one (e.g. `_id(type) =>`).
1013        let tokens = Lexer::with_version("type", PineVersion::V4).tokenize()?;
1014        assert!(matches!(&tokens[0].typ, TokenType::Ident(s) if s == "type"));
1015
1016        let tokens = Lexer::with_version("type", PineVersion::V5).tokenize()?;
1017        assert!(matches!(tokens[0].typ, TokenType::Type));
1018        Ok(())
1019    }
1020
1021    #[test]
1022    fn test_operators() -> eyre::Result<()> {
1023        let mut lexer = Lexer::new("+ - * / = == < >");
1024        let tokens = lexer.tokenize()?;
1025        assert!(matches!(tokens[0].typ, TokenType::Plus));
1026        assert!(matches!(tokens[1].typ, TokenType::Minus));
1027        assert!(matches!(tokens[2].typ, TokenType::Star));
1028        assert!(matches!(tokens[3].typ, TokenType::Slash));
1029        assert!(matches!(tokens[4].typ, TokenType::Assign));
1030        assert!(matches!(tokens[5].typ, TokenType::Equal));
1031        assert!(matches!(tokens[6].typ, TokenType::Less));
1032        assert!(matches!(tokens[7].typ, TokenType::Greater));
1033        Ok(())
1034    }
1035
1036    #[test]
1037    fn test_delimiters() -> eyre::Result<()> {
1038        let mut lexer = Lexer::new("( ) [ ] , . : ? \n");
1039        let tokens = lexer.tokenize()?;
1040        assert!(matches!(tokens[0].typ, TokenType::LParen));
1041        assert!(matches!(tokens[1].typ, TokenType::RParen));
1042        assert!(matches!(tokens[2].typ, TokenType::LBracket));
1043        assert!(matches!(tokens[3].typ, TokenType::RBracket));
1044        assert!(matches!(tokens[4].typ, TokenType::Comma));
1045        assert!(matches!(tokens[5].typ, TokenType::Dot));
1046        assert!(matches!(tokens[6].typ, TokenType::Colon));
1047        assert!(matches!(tokens[7].typ, TokenType::Question));
1048        assert!(matches!(tokens[8].typ, TokenType::Newline));
1049        Ok(())
1050    }
1051
1052    #[test]
1053    fn test_member_access() -> eyre::Result<()> {
1054        let mut lexer = Lexer::new("input.int ta.stoch");
1055        let tokens = lexer.tokenize()?;
1056        assert!(matches!(&tokens[0].typ, TokenType::Ident(s) if s == "input"));
1057        assert!(matches!(tokens[1].typ, TokenType::Dot));
1058        assert!(matches!(tokens[2].typ, TokenType::Int)); // 'int' is now a keyword
1059        assert!(matches!(&tokens[3].typ, TokenType::Ident(s) if s == "ta"));
1060        assert!(matches!(tokens[4].typ, TokenType::Dot));
1061        assert!(matches!(&tokens[5].typ, TokenType::Ident(s) if s == "stoch"));
1062        Ok(())
1063    }
1064
1065    #[test]
1066    fn test_comments() -> eyre::Result<()> {
1067        // A trailing comment is emitted as trivia between the code and Newline.
1068        let mut lexer = Lexer::new("42 // comment\n10");
1069        let tokens = lexer.tokenize()?;
1070        assert!(matches!(tokens[0].typ, TokenType::IntLiteral(n) if n == 42));
1071        assert!(matches!(&tokens[1].typ, TokenType::Comment(c) if c == " comment"));
1072        assert!(matches!(tokens[2].typ, TokenType::Newline));
1073        assert!(matches!(tokens[3].typ, TokenType::IntLiteral(n) if n == 10));
1074        Ok(())
1075    }
1076
1077    #[test]
1078    fn test_whole_line_comment_is_trivia() -> eyre::Result<()> {
1079        // A whole-line comment is captured without emitting Indent/Dedent.
1080        let mut lexer = Lexer::new("// header\n42");
1081        let tokens = lexer.tokenize()?;
1082        assert!(matches!(&tokens[0].typ, TokenType::Comment(c) if c == " header"));
1083        assert!(matches!(tokens[1].typ, TokenType::IntLiteral(n) if n == 42));
1084        assert!(!tokens
1085            .iter()
1086            .any(|t| matches!(t.typ, TokenType::Indent | TokenType::Dedent)));
1087        Ok(())
1088    }
1089
1090    #[test]
1091    fn test_errors() {
1092        // Unterminated string
1093        let mut lexer = Lexer::new(r#""hello"#);
1094        assert!(lexer.tokenize().is_err());
1095
1096        // Unexpected character
1097        let mut lexer = Lexer::new("@");
1098        assert!(lexer.tokenize().is_err());
1099    }
1100
1101    #[test]
1102    fn test_complex_expressions() -> eyre::Result<()> {
1103        // Variable declaration
1104        let mut lexer = Lexer::new("var x = 10");
1105        let tokens = lexer.tokenize()?;
1106        assert!(matches!(tokens[0].typ, TokenType::Var));
1107        assert!(matches!(&tokens[1].typ, TokenType::Ident(s) if s == "x"));
1108        assert!(matches!(tokens[2].typ, TokenType::Assign));
1109        assert!(matches!(tokens[3].typ, TokenType::IntLiteral(n) if n == 10));
1110
1111        // Array access
1112        let mut lexer = Lexer::new("close[1]");
1113        let tokens = lexer.tokenize()?;
1114        assert!(matches!(&tokens[0].typ, TokenType::Ident(s) if s == "close"));
1115        assert!(matches!(tokens[1].typ, TokenType::LBracket));
1116        assert!(matches!(tokens[2].typ, TokenType::IntLiteral(n) if n == 1));
1117        assert!(matches!(tokens[3].typ, TokenType::RBracket));
1118
1119        // Comparison
1120        let mut lexer = Lexer::new("x > 5");
1121        let tokens = lexer.tokenize()?;
1122        assert!(matches!(&tokens[0].typ, TokenType::Ident(s) if s == "x"));
1123        assert!(matches!(tokens[1].typ, TokenType::Greater));
1124        assert!(matches!(tokens[2].typ, TokenType::IntLiteral(n) if n == 5));
1125        Ok(())
1126    }
1127}
pine_lexer/lib.rs

pine_lexer/
lib.rs
