pine_lexer/
lib.rs

1use pine_core::PineVersion;
2use thiserror::Error;
3
4#[derive(Debug, Error)]
5pub enum LexerError {
6    #[error("Unterminated string at line {line}, column {column}")]
7    UnterminatedString { line: usize, column: usize },
8
9    #[error("Invalid hex color format '{value}' at line {line}, column {column}")]
10    InvalidHexColor {
11        value: String,
12        line: usize,
13        column: usize,
14    },
15
16    #[error("Unexpected character '{ch}' at line {line}, column {column}")]
17    UnexpectedCharacter {
18        ch: char,
19        line: usize,
20        column: usize,
21    },
22
23    #[error("Indentation error at line {line}")]
24    IndentationError { line: usize },
25
26    #[error("Invalid number '{value}' at line {line}, column {column}")]
27    InvalidNumber {
28        value: String,
29        line: usize,
30        column: usize,
31    },
32}
33
34// Token types
35#[derive(Debug, Clone, PartialEq)]
36pub enum TokenType {
37    IntLiteral(i64),
38    Number(f64),
39    String(String),
40    Bool(bool),
41    HexColor(String), // #RRGGBB or #RRGGBBAA
42
43    /// A `//` comment's text, verbatim and without the leading `//`. Emitted as
44    /// trivia for tools (the formatter); the parser filters it out.
45    Comment(String),
46    /// An empty source line, emitted as trivia so tools can preserve paragraph
47    /// breaks; the parser filters it out.
48    BlankLine,
49
50    // Identifiers and keywords
51    Ident(String),
52    Var,
53    Varip,
54    Const,
55    Type,
56    Enum,
57    Method,
58    Export,
59    Import,
60    If,
61    Else,
62    For,
63    While,
64    Break,
65    Continue,
66    To,
67    In,
68    Switch, // keywords
69    Int,
70    Float, // type keywords
71    Na,    // special value
72    And,
73    Or,
74    Not, // logical operators
75
76    // Operators
77    Plus,
78    Minus,
79    Star,
80    Slash,
81    Percent,
82    Equal,
83    NotEqual,
84    Less,
85    Greater,
86    LessEqual,
87    GreaterEqual,
88    Assign,
89    ColonAssign,
90    Arrow, // =, :=, =>
91    PlusAssign,
92    MinusAssign,
93    StarAssign,
94    SlashAssign, // +=, -=, *=, /=
95
96    // Delimiters
97    LParen,
98    RParen,
99    LBracket,
100    RBracket,
101    Comma,
102    Dot,
103    Colon,
104    Question,
105    Newline,
106    Indent,
107    Dedent,
108
109    Eof,
110}
111
112#[derive(Debug, Clone)]
113pub struct Token {
114    pub typ: TokenType,
115    pub lexeme: String,
116    pub line: usize,
117    pub column: usize,
118}
119
120pub struct Lexer {
121    input: Vec<char>,
122    current: usize,
123    line: usize,
124    column: usize,
125    indent_stack: Vec<usize>,   // Stack of indentation levels
126    pending_tokens: Vec<Token>, // Queue for Indent/Dedent tokens
127    paren_depth: usize,         // Open parentheses; while > 0, layout tokens are suppressed
128    version: PineVersion,       // Decides which words are keywords rather than identifiers
129}
130
131impl Lexer {
132    /// Lex as [`PineVersion::LATEST`]. Use [`Lexer::with_version`] when the
133    /// script's `//@version=` has already been read — some words are only
134    /// keywords in later versions.
135    pub fn new(input: &str) -> Self {
136        Self::with_version(input, PineVersion::LATEST)
137    }
138
139    pub fn with_version(input: &str, version: PineVersion) -> Self {
140        Self {
141            input: input.chars().collect(),
142            current: 0,
143            line: 1,
144            column: 1,
145            indent_stack: vec![0], // Start with base indentation level
146            pending_tokens: vec![],
147            paren_depth: 0,
148            version,
149        }
150    }
151
152    fn peek(&self) -> Option<char> {
153        self.input.get(self.current).copied()
154    }
155
156    fn advance(&mut self) -> Option<char> {
157        let ch = self.peek()?;
158        self.current += 1;
159        if ch == '\n' {
160            self.line += 1;
161            self.column = 1;
162        } else {
163            self.column += 1;
164        }
165        Some(ch)
166    }
167
168    fn skip_whitespace(&mut self) {
169        while let Some(ch) = self.peek() {
170            if ch == ' ' || ch == '\t' || ch == '\r' {
171                self.advance();
172            } else {
173                break;
174            }
175        }
176    }
177
178    fn scan_number(&mut self) -> Result<Token, LexerError> {
179        let start_line = self.line;
180        let start_col = self.column;
181        let mut num_str = String::new();
182
183        // Handle numbers starting with '.' like .5 or .088
184        if self.peek() == Some('.') {
185            num_str.push('.');
186            self.advance();
187        }
188
189        while let Some(ch) = self.peek() {
190            if ch.is_numeric() {
191                num_str.push(ch);
192                self.advance();
193            } else if ch == '.' && !num_str.contains('.') {
194                // Only consume '.' if we haven't seen one yet and it's followed by a digit
195                if let Some(next_ch) = self.input.get(self.current + 1) {
196                    if next_ch.is_numeric() {
197                        num_str.push(ch);
198                        self.advance();
199                    } else {
200                        break;
201                    }
202                } else {
203                    break;
204                }
205            } else {
206                break;
207            }
208        }
209
210        // No decimal point means an integer literal; Pine treats the two types
211        // differently. (This lexer does not read scientific notation, so a `.`
212        // is the only thing that makes a literal a float.)
213        let typ =
214            if num_str.contains('.') {
215                TokenType::Number(num_str.parse::<f64>().map_err(|_| {
216                    LexerError::InvalidNumber {
217                        value: num_str.clone(),
218                        line: start_line,
219                        column: start_col,
220                    }
221                })?)
222            } else {
223                TokenType::IntLiteral(num_str.parse::<i64>().map_err(|_| {
224                    LexerError::InvalidNumber {
225                        value: num_str.clone(),
226                        line: start_line,
227                        column: start_col,
228                    }
229                })?)
230            };
231        Ok(Token {
232            typ,
233            lexeme: num_str,
234            line: start_line,
235            column: start_col,
236        })
237    }
238
239    fn scan_identifier(&mut self) -> Token {
240        let start_line = self.line;
241        let start_col = self.column;
242        let mut ident = String::new();
243
244        while let Some(ch) = self.peek() {
245            if ch.is_alphanumeric() || ch == '_' {
246                ident.push(ch);
247                self.advance();
248            } else {
249                break;
250            }
251        }
252
253        // Check for keywords
254        let typ = match ident.as_str() {
255            "var" => TokenType::Var,
256            "varip" => TokenType::Varip,
257            "const" => TokenType::Const,
258            // `type` introduces a user-defined type from v5 on; before that it
259            // is an ordinary name, and scripts do use it as one.
260            "type" if self.version >= PineVersion::V5 => TokenType::Type,
261            "enum" => TokenType::Enum,
262            "method" => TokenType::Method,
263            "export" => TokenType::Export,
264            "import" => TokenType::Import,
265            "if" => TokenType::If,
266            "else" => TokenType::Else,
267            "true" => TokenType::Bool(true),
268            "false" => TokenType::Bool(false),
269            "for" => TokenType::For,
270            "while" => TokenType::While,
271            "break" => TokenType::Break,
272            "continue" => TokenType::Continue,
273            "to" => TokenType::To,
274            "in" => TokenType::In,
275            "switch" => TokenType::Switch,
276            "int" => TokenType::Int,
277            "float" => TokenType::Float,
278            "na" => TokenType::Na,
279            "and" => TokenType::And,
280            "or" => TokenType::Or,
281            "not" => TokenType::Not,
282            _ => TokenType::Ident(ident.clone()),
283        };
284
285        Token {
286            typ,
287            lexeme: ident,
288            line: start_line,
289            column: start_col,
290        }
291    }
292
293    fn scan_string(&mut self, quote_char: char) -> Result<Token, LexerError> {
294        let start_line = self.line;
295        let start_col = self.column;
296
297        self.advance(); // consume opening quote
298        let mut string = String::new();
299
300        while let Some(ch) = self.peek() {
301            if ch == quote_char {
302                self.advance();
303                return Ok(Token {
304                    typ: TokenType::String(string.clone()),
305                    lexeme: format!("{}{}{}", quote_char, string, quote_char),
306                    line: start_line,
307                    column: start_col,
308                });
309            } else if ch == '\\' {
310                self.advance();
311                if let Some(escaped) = self.advance() {
312                    string.push(match escaped {
313                        'n' => '\n',
314                        't' => '\t',
315                        '"' => '"',
316                        '\'' => '\'',
317                        '\\' => '\\',
318                        _ => escaped,
319                    });
320                }
321            } else {
322                string.push(ch);
323                self.advance();
324            }
325        }
326
327        Err(LexerError::UnterminatedString {
328            line: start_line,
329            column: start_col,
330        })
331    }
332
333    fn scan_hex_color(&mut self) -> Result<Token, LexerError> {
334        let start_line = self.line;
335        let start_col = self.column;
336
337        self.advance(); // consume '#'
338        let mut hex = String::from("#");
339
340        // Hex color format: #RRGGBB or #RRGGBBAA (6 or 8 hex digits)
341        while let Some(ch) = self.peek() {
342            if ch.is_ascii_hexdigit() {
343                hex.push(ch);
344                self.advance();
345            } else {
346                break;
347            }
348        }
349
350        // Validate length (should be 6 or 8 hex digits after #)
351        let hex_len = hex.len() - 1;
352        if hex_len != 6 && hex_len != 8 {
353            return Err(LexerError::InvalidHexColor {
354                value: hex,
355                line: start_line,
356                column: start_col,
357            });
358        }
359
360        Ok(Token {
361            typ: TokenType::HexColor(hex.clone()),
362            lexeme: hex,
363            line: start_line,
364            column: start_col,
365        })
366    }
367
368    fn next_token(&mut self) -> Result<Token, LexerError> {
369        self.skip_whitespace();
370
371        let ch = match self.peek() {
372            Some(c) => c,
373            None => {
374                return Ok(Token {
375                    typ: TokenType::Eof,
376                    lexeme: String::new(),
377                    line: self.line,
378                    column: self.column,
379                });
380            }
381        };
382
383        let line = self.line;
384        let col = self.column;
385
386        let token = match ch {
387            '+' => {
388                self.advance();
389                if self.peek() == Some('=') {
390                    self.advance();
391                    Token {
392                        typ: TokenType::PlusAssign,
393                        lexeme: "+=".to_string(),
394                        line,
395                        column: col,
396                    }
397                } else {
398                    Token {
399                        typ: TokenType::Plus,
400                        lexeme: "+".to_string(),
401                        line,
402                        column: col,
403                    }
404                }
405            }
406            '-' => {
407                self.advance();
408                if self.peek() == Some('=') {
409                    self.advance();
410                    Token {
411                        typ: TokenType::MinusAssign,
412                        lexeme: "-=".to_string(),
413                        line,
414                        column: col,
415                    }
416                } else {
417                    Token {
418                        typ: TokenType::Minus,
419                        lexeme: "-".to_string(),
420                        line,
421                        column: col,
422                    }
423                }
424            }
425            '*' => {
426                self.advance();
427                if self.peek() == Some('=') {
428                    self.advance();
429                    Token {
430                        typ: TokenType::StarAssign,
431                        lexeme: "*=".to_string(),
432                        line,
433                        column: col,
434                    }
435                } else {
436                    Token {
437                        typ: TokenType::Star,
438                        lexeme: "*".to_string(),
439                        line,
440                        column: col,
441                    }
442                }
443            }
444            '/' => {
445                self.advance();
446                if self.peek() == Some('/') {
447                    self.advance(); // consume the second '/'
448                    let mut text = String::new();
449                    while self.peek().is_some() && self.peek() != Some('\n') {
450                        text.push(self.advance().expect("peeked Some"));
451                    }
452                    Token {
453                        typ: TokenType::Comment(text.clone()),
454                        lexeme: format!("//{text}"),
455                        line,
456                        column: col,
457                    }
458                } else if self.peek() == Some('=') {
459                    self.advance();
460                    Token {
461                        typ: TokenType::SlashAssign,
462                        lexeme: "/=".to_string(),
463                        line,
464                        column: col,
465                    }
466                } else {
467                    Token {
468                        typ: TokenType::Slash,
469                        lexeme: "/".to_string(),
470                        line,
471                        column: col,
472                    }
473                }
474            }
475            '%' => {
476                self.advance();
477                Token {
478                    typ: TokenType::Percent,
479                    lexeme: "%".to_string(),
480                    line,
481                    column: col,
482                }
483            }
484            '=' => {
485                self.advance();
486                if self.peek() == Some('=') {
487                    self.advance();
488                    Token {
489                        typ: TokenType::Equal,
490                        lexeme: "==".to_string(),
491                        line,
492                        column: col,
493                    }
494                } else if self.peek() == Some('>') {
495                    self.advance();
496                    Token {
497                        typ: TokenType::Arrow,
498                        lexeme: "=>".to_string(),
499                        line,
500                        column: col,
501                    }
502                } else {
503                    Token {
504                        typ: TokenType::Assign,
505                        lexeme: "=".to_string(),
506                        line,
507                        column: col,
508                    }
509                }
510            }
511            '!' => {
512                self.advance();
513                if self.peek() == Some('=') {
514                    self.advance();
515                    Token {
516                        typ: TokenType::NotEqual,
517                        lexeme: "!=".to_string(),
518                        line,
519                        column: col,
520                    }
521                } else {
522                    return Err(LexerError::UnexpectedCharacter {
523                        ch: '!',
524                        line,
525                        column: col,
526                    });
527                }
528            }
529            '<' => {
530                self.advance();
531                if self.peek() == Some('=') {
532                    self.advance();
533                    Token {
534                        typ: TokenType::LessEqual,
535                        lexeme: "<=".to_string(),
536                        line,
537                        column: col,
538                    }
539                } else {
540                    Token {
541                        typ: TokenType::Less,
542                        lexeme: "<".to_string(),
543                        line,
544                        column: col,
545                    }
546                }
547            }
548            '>' => {
549                self.advance();
550                if self.peek() == Some('=') {
551                    self.advance();
552                    Token {
553                        typ: TokenType::GreaterEqual,
554                        lexeme: ">=".to_string(),
555                        line,
556                        column: col,
557                    }
558                } else {
559                    Token {
560                        typ: TokenType::Greater,
561                        lexeme: ">".to_string(),
562                        line,
563                        column: col,
564                    }
565                }
566            }
567            '(' => {
568                self.advance();
569                Token {
570                    typ: TokenType::LParen,
571                    lexeme: "(".to_string(),
572                    line,
573                    column: col,
574                }
575            }
576            ')' => {
577                self.advance();
578                Token {
579                    typ: TokenType::RParen,
580                    lexeme: ")".to_string(),
581                    line,
582                    column: col,
583                }
584            }
585            '[' => {
586                self.advance();
587                Token {
588                    typ: TokenType::LBracket,
589                    lexeme: "[".to_string(),
590                    line,
591                    column: col,
592                }
593            }
594            ']' => {
595                self.advance();
596                Token {
597                    typ: TokenType::RBracket,
598                    lexeme: "]".to_string(),
599                    line,
600                    column: col,
601                }
602            }
603            ',' => {
604                self.advance();
605                Token {
606                    typ: TokenType::Comma,
607                    lexeme: ",".to_string(),
608                    line,
609                    column: col,
610                }
611            }
612            '.' => {
613                // Check if this is a decimal number like .5 or .088
614                if let Some(next_ch) = self.input.get(self.current + 1) {
615                    if next_ch.is_numeric() {
616                        // This is a decimal number starting with .
617                        return self.scan_number();
618                    }
619                }
620                self.advance();
621                Token {
622                    typ: TokenType::Dot,
623                    lexeme: ".".to_string(),
624                    line,
625                    column: col,
626                }
627            }
628            ':' => {
629                self.advance();
630                if self.peek() == Some('=') {
631                    self.advance();
632                    Token {
633                        typ: TokenType::ColonAssign,
634                        lexeme: ":=".to_string(),
635                        line,
636                        column: col,
637                    }
638                } else {
639                    Token {
640                        typ: TokenType::Colon,
641                        lexeme: ":".to_string(),
642                        line,
643                        column: col,
644                    }
645                }
646            }
647            '?' => {
648                self.advance();
649                Token {
650                    typ: TokenType::Question,
651                    lexeme: "?".to_string(),
652                    line,
653                    column: col,
654                }
655            }
656            '\n' => {
657                self.advance();
658                Token {
659                    typ: TokenType::Newline,
660                    lexeme: "\\n".to_string(),
661                    line,
662                    column: col,
663                }
664            }
665            '"' => return self.scan_string('"'),
666            '\'' => return self.scan_string('\''),
667            '#' => return self.scan_hex_color(),
668            _ if ch.is_numeric() => return self.scan_number(),
669            _ if ch.is_alphabetic() || ch == '_' => self.scan_identifier(),
670            _ => {
671                return Err(LexerError::UnexpectedCharacter {
672                    ch,
673                    line,
674                    column: col,
675                })
676            }
677        };
678
679        Ok(token)
680    }
681
682    pub fn tokenize(&mut self) -> Result<Vec<Token>, LexerError> {
683        let mut tokens = vec![];
684        let mut at_line_start = true;
685
686        loop {
687            // Check if we have pending tokens (Indent/Dedent)
688            if !self.pending_tokens.is_empty() {
689                tokens.push(self.pending_tokens.remove(0));
690                continue;
691            }
692
693            // Handle indentation at the start of a line
694            if at_line_start {
695                at_line_start = false;
696
697                // Skip blank lines and comments
698                let saved_line = self.line;
699                let saved_col = self.column;
700
701                // Count leading spaces
702                let mut indent_level = 0;
703                while let Some(ch) = self.peek() {
704                    if ch == ' ' {
705                        indent_level += 1;
706                        self.advance();
707                    } else if ch == '\t' {
708                        indent_level += 4; // Treat tab as 4 spaces
709                        self.advance();
710                    } else {
711                        break;
712                    }
713                }
714
715                // Check if this is a blank line or comment
716                if let Some(ch) = self.peek() {
717                    if ch == '\n' || ch == '\r' {
718                        // Blank line - emit trivia and skip the newline.
719                        tokens.push(Token {
720                            typ: TokenType::BlankLine,
721                            lexeme: String::new(),
722                            line: self.line,
723                            column: self.column,
724                        });
725                        self.advance();
726                        at_line_start = true;
727                        continue;
728                    } else if ch == '/' && self.peek_ahead(1) == Some('/') {
729                        // A whole-line comment: capture it as trivia without
730                        // touching the indent stack (its indentation is layout).
731                        let comment_line = self.line;
732                        let comment_col = self.column;
733                        self.advance();
734                        self.advance();
735                        let mut text = String::new();
736                        while let Some(c) = self.peek() {
737                            if c == '\n' {
738                                break;
739                            }
740                            text.push(c);
741                            self.advance();
742                        }
743                        tokens.push(Token {
744                            typ: TokenType::Comment(text.clone()),
745                            lexeme: format!("//{text}"),
746                            line: comment_line,
747                            column: comment_col,
748                        });
749                        if self.peek() == Some('\n') {
750                            self.advance();
751                        }
752                        at_line_start = true;
753                        continue;
754                    }
755                } else {
756                    // EOF - emit dedents for all remaining levels
757                    let current_line = self.line;
758                    let current_col = self.column;
759                    while self.indent_stack.len() > 1 {
760                        self.indent_stack.pop();
761                        tokens.push(Token {
762                            typ: TokenType::Dedent,
763                            lexeme: String::new(),
764                            line: current_line,
765                            column: current_col,
766                        });
767                    }
768                    tokens.push(Token {
769                        typ: TokenType::Eof,
770                        lexeme: String::new(),
771                        line: current_line,
772                        column: current_col,
773                    });
774                    break;
775                }
776
777                if self.paren_depth > 0 {
778                    // Inside parentheses, Pine allows a wrapped line to use any
779                    // indentation, including a multiple of 4. Ignore this line's
780                    // indentation entirely: emit no Indent/Dedent and leave the
781                    // indent stack untouched. The Newline that would have ended
782                    // the previous line is suppressed where it is produced.
783                } else if indent_level % 4 != 0 {
784                    // Pine line-wrapping: a line indented by a non-multiple of
785                    // 4 spaces continues the previous logical line (Pine
786                    // reserves 4-space multiples for local blocks). Join it to
787                    // the previous line: drop the Newline that ended it and
788                    // leave the indent stack untouched.
789                    if matches!(
790                        tokens.last(),
791                        Some(Token {
792                            typ: TokenType::Newline,
793                            ..
794                        })
795                    ) {
796                        tokens.pop();
797                    }
798                } else {
799                    // Handle indent/dedent
800                    // SAFETY: indent_stack is initialized with vec![0] and we never pop the last element
801                    let current_indent = *self.indent_stack.last().unwrap();
802                    let line = saved_line;
803                    let col = saved_col;
804
805                    if indent_level > current_indent {
806                        // Indent
807                        self.indent_stack.push(indent_level);
808                        tokens.push(Token {
809                            typ: TokenType::Indent,
810                            lexeme: String::new(),
811                            line,
812                            column: col,
813                        });
814                    } else if indent_level < current_indent {
815                        // Dedent - possibly multiple levels
816                        // SAFETY: checked by len() > 1
817                        while self.indent_stack.len() > 1
818                            && *self.indent_stack.last().unwrap() > indent_level
819                        {
820                            self.indent_stack.pop();
821                            tokens.push(Token {
822                                typ: TokenType::Dedent,
823                                lexeme: String::new(),
824                                line,
825                                column: col,
826                            });
827                        }
828
829                        // Check for indentation error
830                        // SAFETY: indent_stack always has at least one element
831                        if *self.indent_stack.last().unwrap() != indent_level {
832                            return Err(LexerError::IndentationError { line });
833                        }
834                    }
835                }
836            }
837
838            // Get next token
839            let token = self.next_token()?;
840
841            // Track parenthesis nesting so layout tokens can be suppressed
842            // inside a parenthesised expression (Pine line-wrapping rule).
843            match token.typ {
844                TokenType::LParen => self.paren_depth += 1,
845                TokenType::RParen => self.paren_depth = self.paren_depth.saturating_sub(1),
846                _ => {}
847            }
848
849            // Check if this is a newline
850            if matches!(token.typ, TokenType::Newline) {
851                at_line_start = true;
852                // Inside parentheses a newline does not terminate the logical
853                // line, so drop it; the following line's indentation is ignored
854                // by the layout block above.
855                if self.paren_depth == 0 {
856                    tokens.push(token);
857                }
858            } else if matches!(token.typ, TokenType::Eof) {
859                // Emit dedents for all remaining levels
860                while self.indent_stack.len() > 1 {
861                    self.indent_stack.pop();
862                    tokens.push(Token {
863                        typ: TokenType::Dedent,
864                        lexeme: String::new(),
865                        line: token.line,
866                        column: token.column,
867                    });
868                }
869                tokens.push(token);
870                break;
871            } else {
872                tokens.push(token);
873            }
874        }
875
876        Ok(tokens)
877    }
878
879    fn peek_ahead(&self, offset: usize) -> Option<char> {
880        self.input.get(self.current + offset).copied()
881    }
882}
883
884#[cfg(test)]
885mod tests {
886    use super::*;
887
888    #[test]
889    fn test_line_wrapping_non_multiple_of_4_joins_lines() -> eyre::Result<()> {
890        // Pine line-wrapping rule: a line indented by a non-multiple of 4
891        // spaces continues the previous logical line (4-space multiples are
892        // reserved for local blocks). The wrapped lines must produce NO
893        // Newline before the continuation and NO Indent/Dedent tokens.
894        let mut lexer = Lexer::new("a = x < 2\n         and y\nb = 1");
895        let tokens = lexer.tokenize()?;
896        assert!(
897            !tokens
898                .iter()
899                .any(|t| matches!(t.typ, TokenType::Indent | TokenType::Dedent)),
900            "wrapped continuation must not emit Indent/Dedent: {:?}",
901            tokens.iter().map(|t| &t.typ).collect::<Vec<_>>()
902        );
903        // `a = x < 2 and y` must be one logical line: the only Newline comes
904        // after `y` (plus optionally after `b = 1`).
905        let and_pos = tokens
906            .iter()
907            .position(|t| matches!(t.typ, TokenType::And))
908            .expect("And token present");
909        assert!(
910            !tokens[..and_pos]
911                .iter()
912                .any(|t| matches!(t.typ, TokenType::Newline)),
913            "no Newline may precede the continuation's `and`: {:?}",
914            tokens.iter().map(|t| &t.typ).collect::<Vec<_>>()
915        );
916        Ok(())
917    }
918
919    #[test]
920    fn test_block_indent_multiple_of_4_still_indents() -> eyre::Result<()> {
921        let mut lexer = Lexer::new("if cond\n    x = 1\ny = 2");
922        let tokens = lexer.tokenize()?;
923        assert!(
924            tokens.iter().any(|t| matches!(t.typ, TokenType::Indent)),
925            "4-space block body must still emit Indent"
926        );
927        assert!(
928            tokens.iter().any(|t| matches!(t.typ, TokenType::Dedent)),
929            "return to column 0 must still emit Dedent"
930        );
931        Ok(())
932    }
933
934    #[test]
935    fn test_parens_suppress_layout_at_any_indent() -> eyre::Result<()> {
936        // Inside parentheses a wrapped line may use any indentation, including a
937        // multiple of 4. The whole call is one logical line: no Newline, Indent
938        // or Dedent appears between `(` and `)`.
939        let mut lexer = Lexer::new("plot(\n    a,\n        b\n)");
940        let tokens = lexer.tokenize()?;
941        assert!(matches!(&tokens[0].typ, TokenType::Ident(s) if s == "plot"));
942        assert!(matches!(tokens[1].typ, TokenType::LParen));
943        assert!(matches!(&tokens[2].typ, TokenType::Ident(s) if s == "a"));
944        assert!(matches!(tokens[3].typ, TokenType::Comma));
945        assert!(matches!(&tokens[4].typ, TokenType::Ident(s) if s == "b"));
946        assert!(matches!(tokens[5].typ, TokenType::RParen));
947        assert!(matches!(tokens[6].typ, TokenType::Eof));
948        Ok(())
949    }
950
951    #[test]
952    fn test_newline_after_closing_paren_terminates() -> eyre::Result<()> {
953        // The Newline after the closing paren still terminates the statement,
954        // so a following statement stays separate.
955        let mut lexer = Lexer::new("x = f(\n    a\n)\ny = 1");
956        let tokens = lexer.tokenize()?;
957        assert!(matches!(tokens[5].typ, TokenType::RParen));
958        assert!(matches!(tokens[6].typ, TokenType::Newline));
959        assert!(matches!(&tokens[7].typ, TokenType::Ident(s) if s == "y"));
960        Ok(())
961    }
962
963    #[test]
964    fn test_literals() -> eyre::Result<()> {
965        // Numbers
966        let mut lexer = Lexer::new("42 3.15");
967        let tokens = lexer.tokenize()?;
968        assert!(matches!(tokens[0].typ, TokenType::IntLiteral(n) if n == 42));
969        assert!(matches!(tokens[1].typ, TokenType::Number(n) if n == 3.15));
970
971        // Strings
972        let mut lexer = Lexer::new(r#""hello" "world\n""#);
973        let tokens = lexer.tokenize()?;
974        assert!(matches!(&tokens[0].typ, TokenType::String(s) if s == "hello"));
975        assert!(matches!(&tokens[1].typ, TokenType::String(s) if s == "world\n"));
976
977        // Booleans
978        let mut lexer = Lexer::new("true false");
979        let tokens = lexer.tokenize()?;
980        assert!(matches!(tokens[0].typ, TokenType::Bool(true)));
981        assert!(matches!(tokens[1].typ, TokenType::Bool(false)));
982        Ok(())
983    }
984
985    #[test]
986    fn test_identifiers_and_keywords() -> eyre::Result<()> {
987        let mut lexer = Lexer::new("my_var var if else for while int float na");
988        let tokens = lexer.tokenize()?;
989        assert!(matches!(&tokens[0].typ, TokenType::Ident(s) if s == "my_var"));
990        assert!(matches!(tokens[1].typ, TokenType::Var));
991        assert!(matches!(tokens[2].typ, TokenType::If));
992        assert!(matches!(tokens[3].typ, TokenType::Else));
993        assert!(matches!(tokens[4].typ, TokenType::For));
994        assert!(matches!(tokens[5].typ, TokenType::While));
995        assert!(matches!(tokens[6].typ, TokenType::Int));
996        assert!(matches!(tokens[7].typ, TokenType::Float));
997        assert!(matches!(tokens[8].typ, TokenType::Na));
998        Ok(())
999    }
1000
1001    #[test]
1002    fn test_type_is_a_keyword_only_from_v5() -> eyre::Result<()> {
1003        // v5 introduced user-defined types; before that `type` is just a name,
1004        // and v4 scripts do use it as one (e.g. `_id(type) =>`).
1005        let tokens = Lexer::with_version("type", PineVersion::V4).tokenize()?;
1006        assert!(matches!(&tokens[0].typ, TokenType::Ident(s) if s == "type"));
1007
1008        let tokens = Lexer::with_version("type", PineVersion::V5).tokenize()?;
1009        assert!(matches!(tokens[0].typ, TokenType::Type));
1010        Ok(())
1011    }
1012
1013    #[test]
1014    fn test_operators() -> eyre::Result<()> {
1015        let mut lexer = Lexer::new("+ - * / = == < >");
1016        let tokens = lexer.tokenize()?;
1017        assert!(matches!(tokens[0].typ, TokenType::Plus));
1018        assert!(matches!(tokens[1].typ, TokenType::Minus));
1019        assert!(matches!(tokens[2].typ, TokenType::Star));
1020        assert!(matches!(tokens[3].typ, TokenType::Slash));
1021        assert!(matches!(tokens[4].typ, TokenType::Assign));
1022        assert!(matches!(tokens[5].typ, TokenType::Equal));
1023        assert!(matches!(tokens[6].typ, TokenType::Less));
1024        assert!(matches!(tokens[7].typ, TokenType::Greater));
1025        Ok(())
1026    }
1027
1028    #[test]
1029    fn test_delimiters() -> eyre::Result<()> {
1030        let mut lexer = Lexer::new("( ) [ ] , . : ? \n");
1031        let tokens = lexer.tokenize()?;
1032        assert!(matches!(tokens[0].typ, TokenType::LParen));
1033        assert!(matches!(tokens[1].typ, TokenType::RParen));
1034        assert!(matches!(tokens[2].typ, TokenType::LBracket));
1035        assert!(matches!(tokens[3].typ, TokenType::RBracket));
1036        assert!(matches!(tokens[4].typ, TokenType::Comma));
1037        assert!(matches!(tokens[5].typ, TokenType::Dot));
1038        assert!(matches!(tokens[6].typ, TokenType::Colon));
1039        assert!(matches!(tokens[7].typ, TokenType::Question));
1040        assert!(matches!(tokens[8].typ, TokenType::Newline));
1041        Ok(())
1042    }
1043
1044    #[test]
1045    fn test_member_access() -> eyre::Result<()> {
1046        let mut lexer = Lexer::new("input.int ta.stoch");
1047        let tokens = lexer.tokenize()?;
1048        assert!(matches!(&tokens[0].typ, TokenType::Ident(s) if s == "input"));
1049        assert!(matches!(tokens[1].typ, TokenType::Dot));
1050        assert!(matches!(tokens[2].typ, TokenType::Int)); // 'int' is now a keyword
1051        assert!(matches!(&tokens[3].typ, TokenType::Ident(s) if s == "ta"));
1052        assert!(matches!(tokens[4].typ, TokenType::Dot));
1053        assert!(matches!(&tokens[5].typ, TokenType::Ident(s) if s == "stoch"));
1054        Ok(())
1055    }
1056
1057    #[test]
1058    fn test_comments() -> eyre::Result<()> {
1059        // A trailing comment is emitted as trivia between the code and Newline.
1060        let mut lexer = Lexer::new("42 // comment\n10");
1061        let tokens = lexer.tokenize()?;
1062        assert!(matches!(tokens[0].typ, TokenType::IntLiteral(n) if n == 42));
1063        assert!(matches!(&tokens[1].typ, TokenType::Comment(c) if c == " comment"));
1064        assert!(matches!(tokens[2].typ, TokenType::Newline));
1065        assert!(matches!(tokens[3].typ, TokenType::IntLiteral(n) if n == 10));
1066        Ok(())
1067    }
1068
1069    #[test]
1070    fn test_whole_line_comment_is_trivia() -> eyre::Result<()> {
1071        // A whole-line comment is captured without emitting Indent/Dedent.
1072        let mut lexer = Lexer::new("// header\n42");
1073        let tokens = lexer.tokenize()?;
1074        assert!(matches!(&tokens[0].typ, TokenType::Comment(c) if c == " header"));
1075        assert!(matches!(tokens[1].typ, TokenType::IntLiteral(n) if n == 42));
1076        assert!(!tokens
1077            .iter()
1078            .any(|t| matches!(t.typ, TokenType::Indent | TokenType::Dedent)));
1079        Ok(())
1080    }
1081
1082    #[test]
1083    fn test_errors() {
1084        // Unterminated string
1085        let mut lexer = Lexer::new(r#""hello"#);
1086        assert!(lexer.tokenize().is_err());
1087
1088        // Unexpected character
1089        let mut lexer = Lexer::new("@");
1090        assert!(lexer.tokenize().is_err());
1091    }
1092
1093    #[test]
1094    fn test_complex_expressions() -> eyre::Result<()> {
1095        // Variable declaration
1096        let mut lexer = Lexer::new("var x = 10");
1097        let tokens = lexer.tokenize()?;
1098        assert!(matches!(tokens[0].typ, TokenType::Var));
1099        assert!(matches!(&tokens[1].typ, TokenType::Ident(s) if s == "x"));
1100        assert!(matches!(tokens[2].typ, TokenType::Assign));
1101        assert!(matches!(tokens[3].typ, TokenType::IntLiteral(n) if n == 10));
1102
1103        // Array access
1104        let mut lexer = Lexer::new("close[1]");
1105        let tokens = lexer.tokenize()?;
1106        assert!(matches!(&tokens[0].typ, TokenType::Ident(s) if s == "close"));
1107        assert!(matches!(tokens[1].typ, TokenType::LBracket));
1108        assert!(matches!(tokens[2].typ, TokenType::IntLiteral(n) if n == 1));
1109        assert!(matches!(tokens[3].typ, TokenType::RBracket));
1110
1111        // Comparison
1112        let mut lexer = Lexer::new("x > 5");
1113        let tokens = lexer.tokenize()?;
1114        assert!(matches!(&tokens[0].typ, TokenType::Ident(s) if s == "x"));
1115        assert!(matches!(tokens[1].typ, TokenType::Greater));
1116        assert!(matches!(tokens[2].typ, TokenType::IntLiteral(n) if n == 5));
1117        Ok(())
1118    }
1119}
pine_lexer/lib.rs

pine_lexer/
lib.rs
