pine_lexer/
lib.rs

1use pine_core::PineVersion;
2use thiserror::Error;
3
4#[derive(Debug, Error)]
5pub enum LexerError {
6    #[error("Unterminated string at line {line}, column {column}")]
7    UnterminatedString { line: usize, column: usize },
8
9    #[error("Invalid hex color format '{value}' at line {line}, column {column}")]
10    InvalidHexColor {
11        value: String,
12        line: usize,
13        column: usize,
14    },
15
16    #[error("Unexpected character '{ch}' at line {line}, column {column}")]
17    UnexpectedCharacter {
18        ch: char,
19        line: usize,
20        column: usize,
21    },
22
23    #[error("Indentation error at line {line}")]
24    IndentationError { line: usize },
25
26    #[error("Invalid number '{value}' at line {line}, column {column}")]
27    InvalidNumber {
28        value: String,
29        line: usize,
30        column: usize,
31    },
32}
33
34// Token types
35#[derive(Debug, Clone, PartialEq)]
36pub enum TokenType {
37    IntLiteral(i64),
38    Number(f64),
39    String(String),
40    Bool(bool),
41    HexColor(String), // #RRGGBB or #RRGGBBAA
42
43    // Identifiers and keywords
44    Ident(String),
45    Var,
46    Varip,
47    Const,
48    Type,
49    Enum,
50    Method,
51    Export,
52    Import,
53    If,
54    Else,
55    For,
56    While,
57    Break,
58    Continue,
59    To,
60    In,
61    Switch, // keywords
62    Int,
63    Float, // type keywords
64    Na,    // special value
65    And,
66    Or,
67    Not, // logical operators
68
69    // Operators
70    Plus,
71    Minus,
72    Star,
73    Slash,
74    Percent,
75    Equal,
76    NotEqual,
77    Less,
78    Greater,
79    LessEqual,
80    GreaterEqual,
81    Assign,
82    ColonAssign,
83    Arrow, // =, :=, =>
84    PlusAssign,
85    MinusAssign,
86    StarAssign,
87    SlashAssign, // +=, -=, *=, /=
88
89    // Delimiters
90    LParen,
91    RParen,
92    LBracket,
93    RBracket,
94    Comma,
95    Dot,
96    Colon,
97    Question,
98    Newline,
99    Indent,
100    Dedent,
101
102    Eof,
103}
104
105#[derive(Debug, Clone)]
106pub struct Token {
107    pub typ: TokenType,
108    pub lexeme: String,
109    pub line: usize,
110    pub column: usize,
111}
112
113pub struct Lexer {
114    input: Vec<char>,
115    current: usize,
116    line: usize,
117    column: usize,
118    indent_stack: Vec<usize>,   // Stack of indentation levels
119    pending_tokens: Vec<Token>, // Queue for Indent/Dedent tokens
120    paren_depth: usize,         // Open parentheses; while > 0, layout tokens are suppressed
121    version: PineVersion,       // Decides which words are keywords rather than identifiers
122}
123
124impl Lexer {
125    /// Lex as [`PineVersion::LATEST`]. Use [`Lexer::with_version`] when the
126    /// script's `//@version=` has already been read — some words are only
127    /// keywords in later versions.
128    pub fn new(input: &str) -> Self {
129        Self::with_version(input, PineVersion::LATEST)
130    }
131
132    pub fn with_version(input: &str, version: PineVersion) -> Self {
133        Self {
134            input: input.chars().collect(),
135            current: 0,
136            line: 1,
137            column: 1,
138            indent_stack: vec![0], // Start with base indentation level
139            pending_tokens: vec![],
140            paren_depth: 0,
141            version,
142        }
143    }
144
145    fn peek(&self) -> Option<char> {
146        self.input.get(self.current).copied()
147    }
148
149    fn advance(&mut self) -> Option<char> {
150        let ch = self.peek()?;
151        self.current += 1;
152        if ch == '\n' {
153            self.line += 1;
154            self.column = 1;
155        } else {
156            self.column += 1;
157        }
158        Some(ch)
159    }
160
161    fn skip_whitespace(&mut self) {
162        while let Some(ch) = self.peek() {
163            if ch == ' ' || ch == '\t' || ch == '\r' {
164                self.advance();
165            } else {
166                break;
167            }
168        }
169    }
170
171    fn scan_number(&mut self) -> Result<Token, LexerError> {
172        let start_line = self.line;
173        let start_col = self.column;
174        let mut num_str = String::new();
175
176        // Handle numbers starting with '.' like .5 or .088
177        if self.peek() == Some('.') {
178            num_str.push('.');
179            self.advance();
180        }
181
182        while let Some(ch) = self.peek() {
183            if ch.is_numeric() {
184                num_str.push(ch);
185                self.advance();
186            } else if ch == '.' && !num_str.contains('.') {
187                // Only consume '.' if we haven't seen one yet and it's followed by a digit
188                if let Some(next_ch) = self.input.get(self.current + 1) {
189                    if next_ch.is_numeric() {
190                        num_str.push(ch);
191                        self.advance();
192                    } else {
193                        break;
194                    }
195                } else {
196                    break;
197                }
198            } else {
199                break;
200            }
201        }
202
203        // No decimal point means an integer literal; Pine treats the two types
204        // differently. (This lexer does not read scientific notation, so a `.`
205        // is the only thing that makes a literal a float.)
206        let typ =
207            if num_str.contains('.') {
208                TokenType::Number(num_str.parse::<f64>().map_err(|_| {
209                    LexerError::InvalidNumber {
210                        value: num_str.clone(),
211                        line: start_line,
212                        column: start_col,
213                    }
214                })?)
215            } else {
216                TokenType::IntLiteral(num_str.parse::<i64>().map_err(|_| {
217                    LexerError::InvalidNumber {
218                        value: num_str.clone(),
219                        line: start_line,
220                        column: start_col,
221                    }
222                })?)
223            };
224        Ok(Token {
225            typ,
226            lexeme: num_str,
227            line: start_line,
228            column: start_col,
229        })
230    }
231
232    fn scan_identifier(&mut self) -> Token {
233        let start_line = self.line;
234        let start_col = self.column;
235        let mut ident = String::new();
236
237        while let Some(ch) = self.peek() {
238            if ch.is_alphanumeric() || ch == '_' {
239                ident.push(ch);
240                self.advance();
241            } else {
242                break;
243            }
244        }
245
246        // Check for keywords
247        let typ = match ident.as_str() {
248            "var" => TokenType::Var,
249            "varip" => TokenType::Varip,
250            "const" => TokenType::Const,
251            // `type` introduces a user-defined type from v5 on; before that it
252            // is an ordinary name, and scripts do use it as one.
253            "type" if self.version >= PineVersion::V5 => TokenType::Type,
254            "enum" => TokenType::Enum,
255            "method" => TokenType::Method,
256            "export" => TokenType::Export,
257            "import" => TokenType::Import,
258            "if" => TokenType::If,
259            "else" => TokenType::Else,
260            "true" => TokenType::Bool(true),
261            "false" => TokenType::Bool(false),
262            "for" => TokenType::For,
263            "while" => TokenType::While,
264            "break" => TokenType::Break,
265            "continue" => TokenType::Continue,
266            "to" => TokenType::To,
267            "in" => TokenType::In,
268            "switch" => TokenType::Switch,
269            "int" => TokenType::Int,
270            "float" => TokenType::Float,
271            "na" => TokenType::Na,
272            "and" => TokenType::And,
273            "or" => TokenType::Or,
274            "not" => TokenType::Not,
275            _ => TokenType::Ident(ident.clone()),
276        };
277
278        Token {
279            typ,
280            lexeme: ident,
281            line: start_line,
282            column: start_col,
283        }
284    }
285
286    fn scan_string(&mut self, quote_char: char) -> Result<Token, LexerError> {
287        let start_line = self.line;
288        let start_col = self.column;
289
290        self.advance(); // consume opening quote
291        let mut string = String::new();
292
293        while let Some(ch) = self.peek() {
294            if ch == quote_char {
295                self.advance();
296                return Ok(Token {
297                    typ: TokenType::String(string.clone()),
298                    lexeme: format!("{}{}{}", quote_char, string, quote_char),
299                    line: start_line,
300                    column: start_col,
301                });
302            } else if ch == '\\' {
303                self.advance();
304                if let Some(escaped) = self.advance() {
305                    string.push(match escaped {
306                        'n' => '\n',
307                        't' => '\t',
308                        '"' => '"',
309                        '\'' => '\'',
310                        '\\' => '\\',
311                        _ => escaped,
312                    });
313                }
314            } else {
315                string.push(ch);
316                self.advance();
317            }
318        }
319
320        Err(LexerError::UnterminatedString {
321            line: start_line,
322            column: start_col,
323        })
324    }
325
326    fn scan_hex_color(&mut self) -> Result<Token, LexerError> {
327        let start_line = self.line;
328        let start_col = self.column;
329
330        self.advance(); // consume '#'
331        let mut hex = String::from("#");
332
333        // Hex color format: #RRGGBB or #RRGGBBAA (6 or 8 hex digits)
334        while let Some(ch) = self.peek() {
335            if ch.is_ascii_hexdigit() {
336                hex.push(ch);
337                self.advance();
338            } else {
339                break;
340            }
341        }
342
343        // Validate length (should be 6 or 8 hex digits after #)
344        let hex_len = hex.len() - 1;
345        if hex_len != 6 && hex_len != 8 {
346            return Err(LexerError::InvalidHexColor {
347                value: hex,
348                line: start_line,
349                column: start_col,
350            });
351        }
352
353        Ok(Token {
354            typ: TokenType::HexColor(hex.clone()),
355            lexeme: hex,
356            line: start_line,
357            column: start_col,
358        })
359    }
360
361    fn next_token(&mut self) -> Result<Token, LexerError> {
362        self.skip_whitespace();
363
364        let ch = match self.peek() {
365            Some(c) => c,
366            None => {
367                return Ok(Token {
368                    typ: TokenType::Eof,
369                    lexeme: String::new(),
370                    line: self.line,
371                    column: self.column,
372                });
373            }
374        };
375
376        let line = self.line;
377        let col = self.column;
378
379        let token = match ch {
380            '+' => {
381                self.advance();
382                if self.peek() == Some('=') {
383                    self.advance();
384                    Token {
385                        typ: TokenType::PlusAssign,
386                        lexeme: "+=".to_string(),
387                        line,
388                        column: col,
389                    }
390                } else {
391                    Token {
392                        typ: TokenType::Plus,
393                        lexeme: "+".to_string(),
394                        line,
395                        column: col,
396                    }
397                }
398            }
399            '-' => {
400                self.advance();
401                if self.peek() == Some('=') {
402                    self.advance();
403                    Token {
404                        typ: TokenType::MinusAssign,
405                        lexeme: "-=".to_string(),
406                        line,
407                        column: col,
408                    }
409                } else {
410                    Token {
411                        typ: TokenType::Minus,
412                        lexeme: "-".to_string(),
413                        line,
414                        column: col,
415                    }
416                }
417            }
418            '*' => {
419                self.advance();
420                if self.peek() == Some('=') {
421                    self.advance();
422                    Token {
423                        typ: TokenType::StarAssign,
424                        lexeme: "*=".to_string(),
425                        line,
426                        column: col,
427                    }
428                } else {
429                    Token {
430                        typ: TokenType::Star,
431                        lexeme: "*".to_string(),
432                        line,
433                        column: col,
434                    }
435                }
436            }
437            '/' => {
438                self.advance();
439                if self.peek() == Some('/') {
440                    // Comment - skip to end of line
441                    while self.peek().is_some() && self.peek() != Some('\n') {
442                        self.advance();
443                    }
444                    return self.next_token();
445                } else if self.peek() == Some('=') {
446                    self.advance();
447                    Token {
448                        typ: TokenType::SlashAssign,
449                        lexeme: "/=".to_string(),
450                        line,
451                        column: col,
452                    }
453                } else {
454                    Token {
455                        typ: TokenType::Slash,
456                        lexeme: "/".to_string(),
457                        line,
458                        column: col,
459                    }
460                }
461            }
462            '%' => {
463                self.advance();
464                Token {
465                    typ: TokenType::Percent,
466                    lexeme: "%".to_string(),
467                    line,
468                    column: col,
469                }
470            }
471            '=' => {
472                self.advance();
473                if self.peek() == Some('=') {
474                    self.advance();
475                    Token {
476                        typ: TokenType::Equal,
477                        lexeme: "==".to_string(),
478                        line,
479                        column: col,
480                    }
481                } else if self.peek() == Some('>') {
482                    self.advance();
483                    Token {
484                        typ: TokenType::Arrow,
485                        lexeme: "=>".to_string(),
486                        line,
487                        column: col,
488                    }
489                } else {
490                    Token {
491                        typ: TokenType::Assign,
492                        lexeme: "=".to_string(),
493                        line,
494                        column: col,
495                    }
496                }
497            }
498            '!' => {
499                self.advance();
500                if self.peek() == Some('=') {
501                    self.advance();
502                    Token {
503                        typ: TokenType::NotEqual,
504                        lexeme: "!=".to_string(),
505                        line,
506                        column: col,
507                    }
508                } else {
509                    return Err(LexerError::UnexpectedCharacter {
510                        ch: '!',
511                        line,
512                        column: col,
513                    });
514                }
515            }
516            '<' => {
517                self.advance();
518                if self.peek() == Some('=') {
519                    self.advance();
520                    Token {
521                        typ: TokenType::LessEqual,
522                        lexeme: "<=".to_string(),
523                        line,
524                        column: col,
525                    }
526                } else {
527                    Token {
528                        typ: TokenType::Less,
529                        lexeme: "<".to_string(),
530                        line,
531                        column: col,
532                    }
533                }
534            }
535            '>' => {
536                self.advance();
537                if self.peek() == Some('=') {
538                    self.advance();
539                    Token {
540                        typ: TokenType::GreaterEqual,
541                        lexeme: ">=".to_string(),
542                        line,
543                        column: col,
544                    }
545                } else {
546                    Token {
547                        typ: TokenType::Greater,
548                        lexeme: ">".to_string(),
549                        line,
550                        column: col,
551                    }
552                }
553            }
554            '(' => {
555                self.advance();
556                Token {
557                    typ: TokenType::LParen,
558                    lexeme: "(".to_string(),
559                    line,
560                    column: col,
561                }
562            }
563            ')' => {
564                self.advance();
565                Token {
566                    typ: TokenType::RParen,
567                    lexeme: ")".to_string(),
568                    line,
569                    column: col,
570                }
571            }
572            '[' => {
573                self.advance();
574                Token {
575                    typ: TokenType::LBracket,
576                    lexeme: "[".to_string(),
577                    line,
578                    column: col,
579                }
580            }
581            ']' => {
582                self.advance();
583                Token {
584                    typ: TokenType::RBracket,
585                    lexeme: "]".to_string(),
586                    line,
587                    column: col,
588                }
589            }
590            ',' => {
591                self.advance();
592                Token {
593                    typ: TokenType::Comma,
594                    lexeme: ",".to_string(),
595                    line,
596                    column: col,
597                }
598            }
599            '.' => {
600                // Check if this is a decimal number like .5 or .088
601                if let Some(next_ch) = self.input.get(self.current + 1) {
602                    if next_ch.is_numeric() {
603                        // This is a decimal number starting with .
604                        return self.scan_number();
605                    }
606                }
607                self.advance();
608                Token {
609                    typ: TokenType::Dot,
610                    lexeme: ".".to_string(),
611                    line,
612                    column: col,
613                }
614            }
615            ':' => {
616                self.advance();
617                if self.peek() == Some('=') {
618                    self.advance();
619                    Token {
620                        typ: TokenType::ColonAssign,
621                        lexeme: ":=".to_string(),
622                        line,
623                        column: col,
624                    }
625                } else {
626                    Token {
627                        typ: TokenType::Colon,
628                        lexeme: ":".to_string(),
629                        line,
630                        column: col,
631                    }
632                }
633            }
634            '?' => {
635                self.advance();
636                Token {
637                    typ: TokenType::Question,
638                    lexeme: "?".to_string(),
639                    line,
640                    column: col,
641                }
642            }
643            '\n' => {
644                self.advance();
645                Token {
646                    typ: TokenType::Newline,
647                    lexeme: "\\n".to_string(),
648                    line,
649                    column: col,
650                }
651            }
652            '"' => return self.scan_string('"'),
653            '\'' => return self.scan_string('\''),
654            '#' => return self.scan_hex_color(),
655            _ if ch.is_numeric() => return self.scan_number(),
656            _ if ch.is_alphabetic() || ch == '_' => self.scan_identifier(),
657            _ => {
658                return Err(LexerError::UnexpectedCharacter {
659                    ch,
660                    line,
661                    column: col,
662                })
663            }
664        };
665
666        Ok(token)
667    }
668
669    pub fn tokenize(&mut self) -> Result<Vec<Token>, LexerError> {
670        let mut tokens = vec![];
671        let mut at_line_start = true;
672
673        loop {
674            // Check if we have pending tokens (Indent/Dedent)
675            if !self.pending_tokens.is_empty() {
676                tokens.push(self.pending_tokens.remove(0));
677                continue;
678            }
679
680            // Handle indentation at the start of a line
681            if at_line_start {
682                at_line_start = false;
683
684                // Skip blank lines and comments
685                let saved_line = self.line;
686                let saved_col = self.column;
687
688                // Count leading spaces
689                let mut indent_level = 0;
690                while let Some(ch) = self.peek() {
691                    if ch == ' ' {
692                        indent_level += 1;
693                        self.advance();
694                    } else if ch == '\t' {
695                        indent_level += 4; // Treat tab as 4 spaces
696                        self.advance();
697                    } else {
698                        break;
699                    }
700                }
701
702                // Check if this is a blank line or comment
703                if let Some(ch) = self.peek() {
704                    if ch == '\n' || ch == '\r' {
705                        // Blank line - skip the newline and continue
706                        self.advance();
707                        at_line_start = true;
708                        continue;
709                    } else if ch == '/' && self.peek_ahead(1) == Some('/') {
710                        // Comment line - skip to end of line
711                        while let Some(c) = self.peek() {
712                            if c == '\n' {
713                                break;
714                            }
715                            self.advance();
716                        }
717                        if self.peek() == Some('\n') {
718                            self.advance();
719                        }
720                        at_line_start = true;
721                        continue;
722                    }
723                } else {
724                    // EOF - emit dedents for all remaining levels
725                    let current_line = self.line;
726                    let current_col = self.column;
727                    while self.indent_stack.len() > 1 {
728                        self.indent_stack.pop();
729                        tokens.push(Token {
730                            typ: TokenType::Dedent,
731                            lexeme: String::new(),
732                            line: current_line,
733                            column: current_col,
734                        });
735                    }
736                    tokens.push(Token {
737                        typ: TokenType::Eof,
738                        lexeme: String::new(),
739                        line: current_line,
740                        column: current_col,
741                    });
742                    break;
743                }
744
745                if self.paren_depth > 0 {
746                    // Inside parentheses, Pine allows a wrapped line to use any
747                    // indentation, including a multiple of 4. Ignore this line's
748                    // indentation entirely: emit no Indent/Dedent and leave the
749                    // indent stack untouched. The Newline that would have ended
750                    // the previous line is suppressed where it is produced.
751                } else if indent_level % 4 != 0 {
752                    // Pine line-wrapping: a line indented by a non-multiple of
753                    // 4 spaces continues the previous logical line (Pine
754                    // reserves 4-space multiples for local blocks). Join it to
755                    // the previous line: drop the Newline that ended it and
756                    // leave the indent stack untouched.
757                    if matches!(
758                        tokens.last(),
759                        Some(Token {
760                            typ: TokenType::Newline,
761                            ..
762                        })
763                    ) {
764                        tokens.pop();
765                    }
766                } else {
767                    // Handle indent/dedent
768                    // SAFETY: indent_stack is initialized with vec![0] and we never pop the last element
769                    let current_indent = *self.indent_stack.last().unwrap();
770                    let line = saved_line;
771                    let col = saved_col;
772
773                    if indent_level > current_indent {
774                        // Indent
775                        self.indent_stack.push(indent_level);
776                        tokens.push(Token {
777                            typ: TokenType::Indent,
778                            lexeme: String::new(),
779                            line,
780                            column: col,
781                        });
782                    } else if indent_level < current_indent {
783                        // Dedent - possibly multiple levels
784                        // SAFETY: checked by len() > 1
785                        while self.indent_stack.len() > 1
786                            && *self.indent_stack.last().unwrap() > indent_level
787                        {
788                            self.indent_stack.pop();
789                            tokens.push(Token {
790                                typ: TokenType::Dedent,
791                                lexeme: String::new(),
792                                line,
793                                column: col,
794                            });
795                        }
796
797                        // Check for indentation error
798                        // SAFETY: indent_stack always has at least one element
799                        if *self.indent_stack.last().unwrap() != indent_level {
800                            return Err(LexerError::IndentationError { line });
801                        }
802                    }
803                }
804            }
805
806            // Get next token
807            let token = self.next_token()?;
808
809            // Track parenthesis nesting so layout tokens can be suppressed
810            // inside a parenthesised expression (Pine line-wrapping rule).
811            match token.typ {
812                TokenType::LParen => self.paren_depth += 1,
813                TokenType::RParen => self.paren_depth = self.paren_depth.saturating_sub(1),
814                _ => {}
815            }
816
817            // Check if this is a newline
818            if matches!(token.typ, TokenType::Newline) {
819                at_line_start = true;
820                // Inside parentheses a newline does not terminate the logical
821                // line, so drop it; the following line's indentation is ignored
822                // by the layout block above.
823                if self.paren_depth == 0 {
824                    tokens.push(token);
825                }
826            } else if matches!(token.typ, TokenType::Eof) {
827                // Emit dedents for all remaining levels
828                while self.indent_stack.len() > 1 {
829                    self.indent_stack.pop();
830                    tokens.push(Token {
831                        typ: TokenType::Dedent,
832                        lexeme: String::new(),
833                        line: token.line,
834                        column: token.column,
835                    });
836                }
837                tokens.push(token);
838                break;
839            } else {
840                tokens.push(token);
841            }
842        }
843
844        Ok(tokens)
845    }
846
847    fn peek_ahead(&self, offset: usize) -> Option<char> {
848        self.input.get(self.current + offset).copied()
849    }
850}
851
852#[cfg(test)]
853mod tests {
854    use super::*;
855
856    #[test]
857    fn test_line_wrapping_non_multiple_of_4_joins_lines() -> eyre::Result<()> {
858        // Pine line-wrapping rule: a line indented by a non-multiple of 4
859        // spaces continues the previous logical line (4-space multiples are
860        // reserved for local blocks). The wrapped lines must produce NO
861        // Newline before the continuation and NO Indent/Dedent tokens.
862        let mut lexer = Lexer::new("a = x < 2\n         and y\nb = 1");
863        let tokens = lexer.tokenize()?;
864        assert!(
865            !tokens
866                .iter()
867                .any(|t| matches!(t.typ, TokenType::Indent | TokenType::Dedent)),
868            "wrapped continuation must not emit Indent/Dedent: {:?}",
869            tokens.iter().map(|t| &t.typ).collect::<Vec<_>>()
870        );
871        // `a = x < 2 and y` must be one logical line: the only Newline comes
872        // after `y` (plus optionally after `b = 1`).
873        let and_pos = tokens
874            .iter()
875            .position(|t| matches!(t.typ, TokenType::And))
876            .expect("And token present");
877        assert!(
878            !tokens[..and_pos]
879                .iter()
880                .any(|t| matches!(t.typ, TokenType::Newline)),
881            "no Newline may precede the continuation's `and`: {:?}",
882            tokens.iter().map(|t| &t.typ).collect::<Vec<_>>()
883        );
884        Ok(())
885    }
886
887    #[test]
888    fn test_block_indent_multiple_of_4_still_indents() -> eyre::Result<()> {
889        let mut lexer = Lexer::new("if cond\n    x = 1\ny = 2");
890        let tokens = lexer.tokenize()?;
891        assert!(
892            tokens.iter().any(|t| matches!(t.typ, TokenType::Indent)),
893            "4-space block body must still emit Indent"
894        );
895        assert!(
896            tokens.iter().any(|t| matches!(t.typ, TokenType::Dedent)),
897            "return to column 0 must still emit Dedent"
898        );
899        Ok(())
900    }
901
902    #[test]
903    fn test_parens_suppress_layout_at_any_indent() -> eyre::Result<()> {
904        // Inside parentheses a wrapped line may use any indentation, including a
905        // multiple of 4. The whole call is one logical line: no Newline, Indent
906        // or Dedent appears between `(` and `)`.
907        let mut lexer = Lexer::new("plot(\n    a,\n        b\n)");
908        let tokens = lexer.tokenize()?;
909        assert!(matches!(&tokens[0].typ, TokenType::Ident(s) if s == "plot"));
910        assert!(matches!(tokens[1].typ, TokenType::LParen));
911        assert!(matches!(&tokens[2].typ, TokenType::Ident(s) if s == "a"));
912        assert!(matches!(tokens[3].typ, TokenType::Comma));
913        assert!(matches!(&tokens[4].typ, TokenType::Ident(s) if s == "b"));
914        assert!(matches!(tokens[5].typ, TokenType::RParen));
915        assert!(matches!(tokens[6].typ, TokenType::Eof));
916        Ok(())
917    }
918
919    #[test]
920    fn test_newline_after_closing_paren_terminates() -> eyre::Result<()> {
921        // The Newline after the closing paren still terminates the statement,
922        // so a following statement stays separate.
923        let mut lexer = Lexer::new("x = f(\n    a\n)\ny = 1");
924        let tokens = lexer.tokenize()?;
925        assert!(matches!(tokens[5].typ, TokenType::RParen));
926        assert!(matches!(tokens[6].typ, TokenType::Newline));
927        assert!(matches!(&tokens[7].typ, TokenType::Ident(s) if s == "y"));
928        Ok(())
929    }
930
931    #[test]
932    fn test_literals() -> eyre::Result<()> {
933        // Numbers
934        let mut lexer = Lexer::new("42 3.15");
935        let tokens = lexer.tokenize()?;
936        assert!(matches!(tokens[0].typ, TokenType::IntLiteral(n) if n == 42));
937        assert!(matches!(tokens[1].typ, TokenType::Number(n) if n == 3.15));
938
939        // Strings
940        let mut lexer = Lexer::new(r#""hello" "world\n""#);
941        let tokens = lexer.tokenize()?;
942        assert!(matches!(&tokens[0].typ, TokenType::String(s) if s == "hello"));
943        assert!(matches!(&tokens[1].typ, TokenType::String(s) if s == "world\n"));
944
945        // Booleans
946        let mut lexer = Lexer::new("true false");
947        let tokens = lexer.tokenize()?;
948        assert!(matches!(tokens[0].typ, TokenType::Bool(true)));
949        assert!(matches!(tokens[1].typ, TokenType::Bool(false)));
950        Ok(())
951    }
952
953    #[test]
954    fn test_identifiers_and_keywords() -> eyre::Result<()> {
955        let mut lexer = Lexer::new("my_var var if else for while int float na");
956        let tokens = lexer.tokenize()?;
957        assert!(matches!(&tokens[0].typ, TokenType::Ident(s) if s == "my_var"));
958        assert!(matches!(tokens[1].typ, TokenType::Var));
959        assert!(matches!(tokens[2].typ, TokenType::If));
960        assert!(matches!(tokens[3].typ, TokenType::Else));
961        assert!(matches!(tokens[4].typ, TokenType::For));
962        assert!(matches!(tokens[5].typ, TokenType::While));
963        assert!(matches!(tokens[6].typ, TokenType::Int));
964        assert!(matches!(tokens[7].typ, TokenType::Float));
965        assert!(matches!(tokens[8].typ, TokenType::Na));
966        Ok(())
967    }
968
969    #[test]
970    fn test_type_is_a_keyword_only_from_v5() -> eyre::Result<()> {
971        // v5 introduced user-defined types; before that `type` is just a name,
972        // and v4 scripts do use it as one (e.g. `_id(type) =>`).
973        let tokens = Lexer::with_version("type", PineVersion::V4).tokenize()?;
974        assert!(matches!(&tokens[0].typ, TokenType::Ident(s) if s == "type"));
975
976        let tokens = Lexer::with_version("type", PineVersion::V5).tokenize()?;
977        assert!(matches!(tokens[0].typ, TokenType::Type));
978        Ok(())
979    }
980
981    #[test]
982    fn test_operators() -> eyre::Result<()> {
983        let mut lexer = Lexer::new("+ - * / = == < >");
984        let tokens = lexer.tokenize()?;
985        assert!(matches!(tokens[0].typ, TokenType::Plus));
986        assert!(matches!(tokens[1].typ, TokenType::Minus));
987        assert!(matches!(tokens[2].typ, TokenType::Star));
988        assert!(matches!(tokens[3].typ, TokenType::Slash));
989        assert!(matches!(tokens[4].typ, TokenType::Assign));
990        assert!(matches!(tokens[5].typ, TokenType::Equal));
991        assert!(matches!(tokens[6].typ, TokenType::Less));
992        assert!(matches!(tokens[7].typ, TokenType::Greater));
993        Ok(())
994    }
995
996    #[test]
997    fn test_delimiters() -> eyre::Result<()> {
998        let mut lexer = Lexer::new("( ) [ ] , . : ? \n");
999        let tokens = lexer.tokenize()?;
1000        assert!(matches!(tokens[0].typ, TokenType::LParen));
1001        assert!(matches!(tokens[1].typ, TokenType::RParen));
1002        assert!(matches!(tokens[2].typ, TokenType::LBracket));
1003        assert!(matches!(tokens[3].typ, TokenType::RBracket));
1004        assert!(matches!(tokens[4].typ, TokenType::Comma));
1005        assert!(matches!(tokens[5].typ, TokenType::Dot));
1006        assert!(matches!(tokens[6].typ, TokenType::Colon));
1007        assert!(matches!(tokens[7].typ, TokenType::Question));
1008        assert!(matches!(tokens[8].typ, TokenType::Newline));
1009        Ok(())
1010    }
1011
1012    #[test]
1013    fn test_member_access() -> eyre::Result<()> {
1014        let mut lexer = Lexer::new("input.int ta.stoch");
1015        let tokens = lexer.tokenize()?;
1016        assert!(matches!(&tokens[0].typ, TokenType::Ident(s) if s == "input"));
1017        assert!(matches!(tokens[1].typ, TokenType::Dot));
1018        assert!(matches!(tokens[2].typ, TokenType::Int)); // 'int' is now a keyword
1019        assert!(matches!(&tokens[3].typ, TokenType::Ident(s) if s == "ta"));
1020        assert!(matches!(tokens[4].typ, TokenType::Dot));
1021        assert!(matches!(&tokens[5].typ, TokenType::Ident(s) if s == "stoch"));
1022        Ok(())
1023    }
1024
1025    #[test]
1026    fn test_comments() -> eyre::Result<()> {
1027        let mut lexer = Lexer::new("42 // comment\n10");
1028        let tokens = lexer.tokenize()?;
1029        assert!(matches!(tokens[0].typ, TokenType::IntLiteral(n) if n == 42));
1030        assert!(matches!(tokens[1].typ, TokenType::Newline));
1031        assert!(matches!(tokens[2].typ, TokenType::IntLiteral(n) if n == 10));
1032        Ok(())
1033    }
1034
1035    #[test]
1036    fn test_errors() {
1037        // Unterminated string
1038        let mut lexer = Lexer::new(r#""hello"#);
1039        assert!(lexer.tokenize().is_err());
1040
1041        // Unexpected character
1042        let mut lexer = Lexer::new("@");
1043        assert!(lexer.tokenize().is_err());
1044    }
1045
1046    #[test]
1047    fn test_complex_expressions() -> eyre::Result<()> {
1048        // Variable declaration
1049        let mut lexer = Lexer::new("var x = 10");
1050        let tokens = lexer.tokenize()?;
1051        assert!(matches!(tokens[0].typ, TokenType::Var));
1052        assert!(matches!(&tokens[1].typ, TokenType::Ident(s) if s == "x"));
1053        assert!(matches!(tokens[2].typ, TokenType::Assign));
1054        assert!(matches!(tokens[3].typ, TokenType::IntLiteral(n) if n == 10));
1055
1056        // Array access
1057        let mut lexer = Lexer::new("close[1]");
1058        let tokens = lexer.tokenize()?;
1059        assert!(matches!(&tokens[0].typ, TokenType::Ident(s) if s == "close"));
1060        assert!(matches!(tokens[1].typ, TokenType::LBracket));
1061        assert!(matches!(tokens[2].typ, TokenType::IntLiteral(n) if n == 1));
1062        assert!(matches!(tokens[3].typ, TokenType::RBracket));
1063
1064        // Comparison
1065        let mut lexer = Lexer::new("x > 5");
1066        let tokens = lexer.tokenize()?;
1067        assert!(matches!(&tokens[0].typ, TokenType::Ident(s) if s == "x"));
1068        assert!(matches!(tokens[1].typ, TokenType::Greater));
1069        assert!(matches!(tokens[2].typ, TokenType::IntLiteral(n) if n == 5));
1070        Ok(())
1071    }
1072}
pine_lexer/lib.rs

pine_lexer/
lib.rs
