dazzle_core/scheme/
parser.rs

1//! Scheme parser (lexer + S-expression parser)
2//!
3//! Ported from OpenJade's `SchemeParser.cxx` (~2,500 lines).
4//!
5//! ## Key Features
6//!
7//! - **Whitespace-agnostic**: Unlike Steel, this parser handles all valid R4RS whitespace
8//! - **Line number tracking**: Error messages report line:column, not byte spans
9//! - **Full R4RS syntax**:
10//!   - Numbers: integers, reals, hex (#x), octal (#o), binary (#b)
11//!   - Strings: escapes (\n, \t, \", \\, \xNN)
12//!   - Symbols and keywords
13//!   - Quote/quasiquote/unquote/unquote-splicing
14//!   - Comments: line (;) and block (#| ... |#)
15//!   - Vectors: #(...)
16//!   - Booleans: #t, #f
17//!   - Characters: #\a, #\space, #\newline
18//!
19//! ## Architecture
20//!
21//! 1. **Lexer** (`Tokenizer`): Character stream → Token stream
22//! 2. **Parser** (`Parser`): Token stream → Value (S-expressions)
23//!
24//! ## OpenJade Correspondence
25//!
26//! | Dazzle        | OpenJade             | Purpose                |
27//! |---------------|----------------------|------------------------|
28//! | `Token`       | `SchemeParser::tok_` | Token types            |
29//! | `Tokenizer`   | `SchemeParser`       | Lexical analysis       |
30//! | `Parser`      | `SchemeParser::get*` | Syntax analysis        |
31//!
32//! ## Error Handling
33//!
34//! Parse errors include:
35//! - Line and column numbers (not byte offsets!)
36//! - Descriptive messages
37//! - Context (what was expected)
38
39use crate::scheme::value::Value;
40use std::fmt;
41
42// =============================================================================
43// Token Types
44// =============================================================================
45
46/// Token type (corresponds to OpenJade's token enum)
47#[derive(Debug, Clone, PartialEq)]
48pub enum Token {
49    // Literals
50    Integer(i64),
51    Real(f64),
52    String(String),
53    Char(char),
54    Symbol(String),
55    Keyword(String),
56    Bool(bool),
57
58    // Delimiters
59    LeftParen,   // (
60    RightParen,  // )
61    LeftBracket, // [  (optional R5RS)
62    RightBracket, // ] (optional R5RS)
63    Dot,         // .
64
65    // Quotation
66    Quote,            // '
67    Quasiquote,       // `
68    Unquote,          // ,
69    UnquoteSplicing,  // ,@
70
71    // Vector
72    VectorStart, // #(
73
74    // End of input
75    Eof,
76}
77
78impl fmt::Display for Token {
79    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
80        match self {
81            Token::Integer(n) => write!(f, "{}", n),
82            Token::Real(n) => write!(f, "{}", n),
83            Token::String(s) => write!(f, "\"{}\"", s),
84            Token::Char(ch) => write!(f, "#\\{}", ch),
85            Token::Symbol(s) => write!(f, "{}", s),
86            Token::Keyword(s) => write!(f, "#:{}", s),
87            Token::Bool(b) => write!(f, "{}", if *b { "#t" } else { "#f" }),
88            Token::LeftParen => write!(f, "("),
89            Token::RightParen => write!(f, ")"),
90            Token::LeftBracket => write!(f, "["),
91            Token::RightBracket => write!(f, "]"),
92            Token::Dot => write!(f, "."),
93            Token::Quote => write!(f, "'"),
94            Token::Quasiquote => write!(f, "`"),
95            Token::Unquote => write!(f, ","),
96            Token::UnquoteSplicing => write!(f, ",@"),
97            Token::VectorStart => write!(f, "#("),
98            Token::Eof => write!(f, "<EOF>"),
99        }
100    }
101}
102
103// =============================================================================
104// Source Position (for error reporting)
105// =============================================================================
106
107/// Source code position (line and column)
108///
109/// **Important**: Line and column numbers start at 1 (human-readable).
110#[derive(Debug, Clone, Copy, PartialEq, Eq)]
111pub struct Position {
112    pub line: usize,
113    pub column: usize,
114}
115
116impl Position {
117    pub fn new() -> Self {
118        Position { line: 1, column: 1 }
119    }
120}
121
122impl Default for Position {
123    fn default() -> Self {
124        Self::new()
125    }
126}
127
128impl fmt::Display for Position {
129    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
130        write!(f, "{}:{}", self.line, self.column)
131    }
132}
133
134// =============================================================================
135// Parse Error
136// =============================================================================
137
138/// Parse error with line:column position
139#[derive(Debug, Clone)]
140pub struct ParseError {
141    pub message: String,
142    pub position: Position,
143}
144
145impl ParseError {
146    pub fn new(message: String, position: Position) -> Self {
147        ParseError { message, position }
148    }
149}
150
151impl fmt::Display for ParseError {
152    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
153        write!(f, "Parse error at {}: {}", self.position, self.message)
154    }
155}
156
157impl std::error::Error for ParseError {}
158
159pub type ParseResult<T> = Result<T, ParseError>;
160
161// =============================================================================
162// Tokenizer (Lexer)
163// =============================================================================
164
165/// Tokenizer for Scheme source code
166///
167/// Corresponds to OpenJade's `SchemeParser` lexical analysis methods.
168///
169/// ## Whitespace Handling
170///
171/// **Critical difference from Steel**: This tokenizer is fully whitespace-agnostic.
172/// Whitespace can appear anywhere between tokens without affecting parsing.
173///
174/// Example that breaks Steel but works here:
175/// ```scheme
176/// (let ((x 1)
177///       (y 2))  ; Multi-line let bindings
178///   (+ x y))
179/// ```
180pub struct Tokenizer {
181    /// Input source code
182    input: Vec<char>,
183
184    /// Current position in input
185    pos: usize,
186
187    /// Current line (1-based)
188    line: usize,
189
190    /// Current column (1-based)
191    column: usize,
192
193    /// Peeked token (for lookahead)
194    peeked: Option<Token>,
195}
196
197impl Tokenizer {
198    /// Create a new tokenizer from source code
199    pub fn new(input: &str) -> Self {
200        Tokenizer {
201            input: input.chars().collect(),
202            pos: 0,
203            line: 1,
204            column: 1,
205            peeked: None,
206        }
207    }
208
209    /// Get current position (for error reporting)
210    pub fn position(&self) -> Position {
211        Position {
212            line: self.line,
213            column: self.column,
214        }
215    }
216
217    /// Peek at current character without consuming
218    fn peek_char(&self) -> Option<char> {
219        if self.pos < self.input.len() {
220            Some(self.input[self.pos])
221        } else {
222            None
223        }
224    }
225
226    /// Peek at character at offset from current position
227    fn peek_char_at(&self, offset: usize) -> Option<char> {
228        let index = self.pos + offset;
229        if index < self.input.len() {
230            Some(self.input[index])
231        } else {
232            None
233        }
234    }
235
236    /// Consume and return current character
237    fn next_char(&mut self) -> Option<char> {
238        if self.pos < self.input.len() {
239            let ch = self.input[self.pos];
240            self.pos += 1;
241
242            // Update line/column tracking
243            if ch == '\n' {
244                self.line += 1;
245                self.column = 1;
246            } else {
247                self.column += 1;
248            }
249
250            Some(ch)
251        } else {
252            None
253        }
254    }
255
256    /// Skip whitespace and comments
257    ///
258    /// **Key feature**: Handles both line comments (;) and block comments (#| ... |#)
259    fn skip_whitespace(&mut self) {
260        loop {
261            match self.peek_char() {
262                // Whitespace
263                Some(ch) if ch.is_whitespace() => {
264                    self.next_char();
265                }
266
267                // Line comment: ; to end of line
268                Some(';') => {
269                    self.next_char();
270                    while let Some(ch) = self.peek_char() {
271                        self.next_char();
272                        if ch == '\n' {
273                            break;
274                        }
275                    }
276                }
277
278                // Block comment: #| ... |#
279                Some('#') if self.peek_char_at(1) == Some('|') => {
280                    self.next_char(); // #
281                    self.next_char(); // |
282
283                    // Find closing |#
284                    let mut depth = 1;
285                    while depth > 0 {
286                        match self.next_char() {
287                            Some('|') if self.peek_char() == Some('#') => {
288                                self.next_char(); // #
289                                depth -= 1;
290                            }
291                            Some('#') if self.peek_char() == Some('|') => {
292                                self.next_char(); // |
293                                depth += 1; // Nested block comment
294                            }
295                            Some(_) => {} // Continue
296                            None => break, // EOF in comment (error, but tolerate)
297                        }
298                    }
299                }
300
301                // Not whitespace or comment
302                _ => break,
303            }
304        }
305    }
306
307    /// Check if character is a delimiter (ends a token)
308    fn is_delimiter(ch: char) -> bool {
309        ch.is_whitespace()
310            || matches!(
311                ch,
312                '(' | ')' | '[' | ']' | '"' | ';' | ',' | '`' | '\''
313            )
314    }
315
316    /// Parse an integer or real number
317    fn parse_number(&mut self, start_pos: Position) -> ParseResult<Token> {
318        let mut num_str = String::new();
319
320        // Collect digits and special characters
321        while let Some(ch) = self.peek_char() {
322            if ch.is_ascii_digit() || matches!(ch, '.' | 'e' | 'E' | '+' | '-') {
323                num_str.push(ch);
324                self.next_char();
325            } else if Self::is_delimiter(ch) {
326                break;
327            } else {
328                // Invalid character in number
329                return Err(ParseError::new(
330                    format!("Invalid character in number: {}", ch),
331                    start_pos,
332                ));
333            }
334        }
335
336        // Try parsing as integer first
337        if let Ok(n) = num_str.parse::<i64>() {
338            return Ok(Token::Integer(n));
339        }
340
341        // Try parsing as float
342        if let Ok(n) = num_str.parse::<f64>() {
343            return Ok(Token::Real(n));
344        }
345
346        Err(ParseError::new(
347            format!("Invalid number: {}", num_str),
348            start_pos,
349        ))
350    }
351
352    /// Parse a hexadecimal number (#x prefix)
353    fn parse_hex_number(&mut self, start_pos: Position) -> ParseResult<Token> {
354        let mut num_str = String::new();
355
356        while let Some(ch) = self.peek_char() {
357            if ch.is_ascii_hexdigit() {
358                num_str.push(ch);
359                self.next_char();
360            } else if Self::is_delimiter(ch) {
361                break;
362            } else {
363                return Err(ParseError::new(
364                    format!("Invalid character in hex number: {}", ch),
365                    start_pos,
366                ));
367            }
368        }
369
370        if num_str.is_empty() {
371            return Err(ParseError::new("Empty hex number".to_string(), start_pos));
372        }
373
374        i64::from_str_radix(&num_str, 16)
375            .map(Token::Integer)
376            .map_err(|_| ParseError::new(format!("Invalid hex number: {}", num_str), start_pos))
377    }
378
379    /// Parse an octal number (#o prefix)
380    fn parse_octal_number(&mut self, start_pos: Position) -> ParseResult<Token> {
381        let mut num_str = String::new();
382
383        while let Some(ch) = self.peek_char() {
384            if ch.is_digit(8) {
385                num_str.push(ch);
386                self.next_char();
387            } else if Self::is_delimiter(ch) {
388                break;
389            } else {
390                return Err(ParseError::new(
391                    format!("Invalid character in octal number: {}", ch),
392                    start_pos,
393                ));
394            }
395        }
396
397        if num_str.is_empty() {
398            return Err(ParseError::new("Empty octal number".to_string(), start_pos));
399        }
400
401        i64::from_str_radix(&num_str, 8)
402            .map(Token::Integer)
403            .map_err(|_| ParseError::new(format!("Invalid octal number: {}", num_str), start_pos))
404    }
405
406    /// Parse a binary number (#b prefix)
407    fn parse_binary_number(&mut self, start_pos: Position) -> ParseResult<Token> {
408        let mut num_str = String::new();
409
410        while let Some(ch) = self.peek_char() {
411            if matches!(ch, '0' | '1') {
412                num_str.push(ch);
413                self.next_char();
414            } else if Self::is_delimiter(ch) {
415                break;
416            } else {
417                return Err(ParseError::new(
418                    format!("Invalid character in binary number: {}", ch),
419                    start_pos,
420                ));
421            }
422        }
423
424        if num_str.is_empty() {
425            return Err(ParseError::new("Empty binary number".to_string(), start_pos));
426        }
427
428        i64::from_str_radix(&num_str, 2)
429            .map(Token::Integer)
430            .map_err(|_| ParseError::new(format!("Invalid binary number: {}", num_str), start_pos))
431    }
432
433    /// Parse a symbol or keyword
434    fn parse_symbol(&mut self) -> String {
435        let mut sym = String::new();
436
437        while let Some(ch) = self.peek_char() {
438            if Self::is_delimiter(ch) {
439                break;
440            }
441            sym.push(ch);
442            self.next_char();
443        }
444
445        sym
446    }
447
448    /// Parse a string literal
449    fn parse_string(&mut self, start_pos: Position) -> ParseResult<String> {
450        self.next_char(); // Consume opening "
451
452        let mut result = String::new();
453
454        loop {
455            match self.next_char() {
456                Some('"') => {
457                    // Closing quote
458                    return Ok(result);
459                }
460                Some('\\') => {
461                    // Escape sequence
462                    match self.next_char() {
463                        Some('n') => result.push('\n'),
464                        Some('t') => result.push('\t'),
465                        Some('r') => result.push('\r'),
466                        Some('\\') => result.push('\\'),
467                        Some('"') => result.push('"'),
468                        Some(ch) => result.push(ch), // Unknown escape, keep literal
469                        None => {
470                            return Err(ParseError::new(
471                                "Unexpected EOF in string escape".to_string(),
472                                start_pos,
473                            ))
474                        }
475                    }
476                }
477                Some(ch) => {
478                    result.push(ch);
479                }
480                None => {
481                    return Err(ParseError::new(
482                        "Unexpected EOF in string".to_string(),
483                        start_pos,
484                    ))
485                }
486            }
487        }
488    }
489
490    /// Parse a character literal (#\a, #\space, #\newline)
491    /// Called after # has been consumed
492    fn parse_char(&mut self, start_pos: Position) -> ParseResult<char> {
493        // Expect backslash
494        if self.next_char() != Some('\\') {
495            return Err(ParseError::new(
496                "Expected \\ after # in character literal".to_string(),
497                start_pos,
498            ));
499        }
500
501        // Read character name
502        let mut name = String::new();
503        while let Some(ch) = self.peek_char() {
504            if Self::is_delimiter(ch) {
505                break;
506            }
507            name.push(ch);
508            self.next_char();
509        }
510
511        if name.is_empty() {
512            return Err(ParseError::new(
513                "Empty character literal".to_string(),
514                start_pos,
515            ));
516        }
517
518        // Named characters
519        match name.as_str() {
520            "space" => Ok(' '),
521            "newline" => Ok('\n'),
522            "tab" => Ok('\t'),
523            "return" => Ok('\r'),
524            s if s.len() == 1 => Ok(s.chars().next().unwrap()),
525            _ => Err(ParseError::new(
526                format!("Invalid character literal: #\\{}", name),
527                start_pos,
528            )),
529        }
530    }
531
532    /// Get the next token
533    pub fn next_token(&mut self) -> ParseResult<Token> {
534        // Check if we have a peeked token
535        if let Some(tok) = self.peeked.take() {
536            return Ok(tok);
537        }
538
539        // Skip whitespace and comments
540        self.skip_whitespace();
541
542        let start_pos = self.position();
543
544        match self.peek_char() {
545            None => Ok(Token::Eof),
546
547            Some('(') => {
548                self.next_char();
549                Ok(Token::LeftParen)
550            }
551
552            Some(')') => {
553                self.next_char();
554                Ok(Token::RightParen)
555            }
556
557            Some('[') => {
558                self.next_char();
559                Ok(Token::LeftBracket)
560            }
561
562            Some(']') => {
563                self.next_char();
564                Ok(Token::RightBracket)
565            }
566
567            Some('\'') => {
568                self.next_char();
569                Ok(Token::Quote)
570            }
571
572            Some('`') => {
573                self.next_char();
574                Ok(Token::Quasiquote)
575            }
576
577            Some(',') => {
578                self.next_char();
579                // Check for ,@
580                if self.peek_char() == Some('@') {
581                    self.next_char();
582                    Ok(Token::UnquoteSplicing)
583                } else {
584                    Ok(Token::Unquote)
585                }
586            }
587
588            Some('"') => {
589                let s = self.parse_string(start_pos)?;
590                Ok(Token::String(s))
591            }
592
593            Some('#') => {
594                self.next_char(); // Consume #
595                match self.peek_char() {
596                    Some('t') => {
597                        self.next_char();
598                        Ok(Token::Bool(true))
599                    }
600                    Some('f') => {
601                        self.next_char();
602                        Ok(Token::Bool(false))
603                    }
604                    Some('(') => {
605                        self.next_char();
606                        Ok(Token::VectorStart)
607                    }
608                    Some('\\') => {
609                        let ch = self.parse_char(start_pos)?;
610                        Ok(Token::Char(ch))
611                    }
612                    Some(':') => {
613                        self.next_char(); // Consume :
614                        let name = self.parse_symbol();
615                        Ok(Token::Keyword(name))
616                    }
617                    Some('x') | Some('X') => {
618                        self.next_char(); // Consume x
619                        self.parse_hex_number(start_pos)
620                    }
621                    Some('o') | Some('O') => {
622                        self.next_char(); // Consume o
623                        self.parse_octal_number(start_pos)
624                    }
625                    Some('b') | Some('B') => {
626                        self.next_char(); // Consume b
627                        self.parse_binary_number(start_pos)
628                    }
629                    _ => Err(ParseError::new(
630                        format!("Invalid # syntax: #{:?}", self.peek_char()),
631                        start_pos,
632                    )),
633                }
634            }
635
636            Some(ch) if ch.is_ascii_digit() => self.parse_number(start_pos),
637
638            Some('+') | Some('-') => {
639                // Could be number or symbol
640                if let Some(next) = self.peek_char_at(1) {
641                    if next.is_ascii_digit() {
642                        self.parse_number(start_pos)
643                    } else {
644                        let sym = self.parse_symbol();
645                        Ok(Token::Symbol(sym))
646                    }
647                } else {
648                    let sym = self.parse_symbol();
649                    Ok(Token::Symbol(sym))
650                }
651            }
652
653            Some('.') => {
654                // Could be dot or number starting with .
655                if let Some(next) = self.peek_char_at(1) {
656                    if next.is_ascii_digit() {
657                        self.parse_number(start_pos)
658                    } else {
659                        self.next_char();
660                        Ok(Token::Dot)
661                    }
662                } else {
663                    self.next_char();
664                    Ok(Token::Dot)
665                }
666            }
667
668            Some(_) => {
669                // Symbol
670                let sym = self.parse_symbol();
671                Ok(Token::Symbol(sym))
672            }
673        }
674    }
675
676    /// Peek at the next token without consuming it
677    pub fn peek_token(&mut self) -> ParseResult<&Token> {
678        if self.peeked.is_none() {
679            let tok = self.next_token()?;
680            self.peeked = Some(tok);
681        }
682        Ok(self.peeked.as_ref().unwrap())
683    }
684}
685
686// =============================================================================
687// Parser (S-expression builder)
688// =============================================================================
689
690/// Parser for building Scheme values from tokens
691///
692/// Corresponds to OpenJade's `SchemeParser::get*` methods.
693///
694/// ## Usage
695///
696/// ```ignore
697/// let parser = Parser::new("(+ 1 2)");
698/// let expr = parser.parse().unwrap();
699/// ```
700pub struct Parser {
701    tokenizer: Tokenizer,
702}
703
704impl Parser {
705    /// Create a new parser from source code
706    pub fn new(input: &str) -> Self {
707        Parser {
708            tokenizer: Tokenizer::new(input),
709        }
710    }
711
712    /// Parse a single S-expression
713    ///
714    /// Returns `Ok(Value)` on success, `Err(ParseError)` on failure.
715    pub fn parse(&mut self) -> ParseResult<Value> {
716        self.parse_expr()
717    }
718
719    /// Parse all S-expressions in input
720    ///
721    /// Returns a list of all top-level expressions.
722    pub fn parse_all(&mut self) -> ParseResult<Vec<Value>> {
723        let mut exprs = Vec::new();
724
725        loop {
726            let tok = self.tokenizer.peek_token()?;
727            if *tok == Token::Eof {
728                break;
729            }
730            exprs.push(self.parse_expr()?);
731        }
732
733        Ok(exprs)
734    }
735
736    /// Parse a single expression
737    fn parse_expr(&mut self) -> ParseResult<Value> {
738        let start_pos = self.tokenizer.position();
739        let tok = self.tokenizer.next_token()?;
740
741        match tok {
742            // Literals
743            Token::Integer(n) => Ok(Value::integer(n)),
744            Token::Real(n) => Ok(Value::real(n)),
745            Token::String(s) => Ok(Value::string(s)),
746            Token::Char(ch) => Ok(Value::char(ch)),
747            Token::Bool(b) => Ok(Value::bool(b)),
748            Token::Symbol(s) => Ok(Value::symbol(&s)),
749            Token::Keyword(s) => Ok(Value::keyword(&s)),
750
751            // Lists
752            Token::LeftParen | Token::LeftBracket => self.parse_list(start_pos),
753
754            // Vectors
755            Token::VectorStart => self.parse_vector(start_pos),
756
757            // Quote
758            Token::Quote => {
759                let quoted = self.parse_expr()?;
760                Ok(Value::cons(Value::symbol("quote"), Value::cons(quoted, Value::Nil)))
761            }
762
763            // Quasiquote
764            Token::Quasiquote => {
765                let quoted = self.parse_expr()?;
766                Ok(Value::cons(
767                    Value::symbol("quasiquote"),
768                    Value::cons(quoted, Value::Nil),
769                ))
770            }
771
772            // Unquote
773            Token::Unquote => {
774                let quoted = self.parse_expr()?;
775                Ok(Value::cons(
776                    Value::symbol("unquote"),
777                    Value::cons(quoted, Value::Nil),
778                ))
779            }
780
781            // Unquote-splicing
782            Token::UnquoteSplicing => {
783                let quoted = self.parse_expr()?;
784                Ok(Value::cons(
785                    Value::symbol("unquote-splicing"),
786                    Value::cons(quoted, Value::Nil),
787                ))
788            }
789
790            // Unexpected tokens
791            Token::RightParen | Token::RightBracket => Err(ParseError::new(
792                format!("Unexpected closing delimiter: {}", tok),
793                start_pos,
794            )),
795
796            Token::Dot => Err(ParseError::new(
797                "Unexpected dot outside of list".to_string(),
798                start_pos,
799            )),
800
801            Token::Eof => Err(ParseError::new(
802                "Unexpected end of input".to_string(),
803                start_pos,
804            )),
805        }
806    }
807
808    /// Parse a list (after opening paren consumed)
809    fn parse_list(&mut self, start_pos: Position) -> ParseResult<Value> {
810        let mut elements = Vec::new();
811        let mut dotted_tail = None;
812
813        loop {
814            let tok = self.tokenizer.peek_token()?;
815
816            match tok {
817                Token::RightParen | Token::RightBracket => {
818                    self.tokenizer.next_token()?; // Consume closing paren
819                    break;
820                }
821
822                Token::Dot => {
823                    self.tokenizer.next_token()?; // Consume dot
824
825                    // Parse the tail
826                    dotted_tail = Some(self.parse_expr()?);
827
828                    // Expect closing paren
829                    let tok = self.tokenizer.next_token()?;
830                    if !matches!(tok, Token::RightParen | Token::RightBracket) {
831                        return Err(ParseError::new(
832                            format!("Expected ) after dotted tail, got {}", tok),
833                            start_pos,
834                        ));
835                    }
836                    break;
837                }
838
839                Token::Eof => {
840                    return Err(ParseError::new(
841                        "Unexpected EOF in list".to_string(),
842                        start_pos,
843                    ))
844                }
845
846                _ => {
847                    elements.push(self.parse_expr()?);
848                }
849            }
850        }
851
852        // Build the list from right to left
853        let mut result = dotted_tail.unwrap_or(Value::Nil);
854        for elem in elements.into_iter().rev() {
855            result = Value::cons(elem, result);
856        }
857
858        Ok(result)
859    }
860
861    /// Parse a vector (after #( consumed)
862    fn parse_vector(&mut self, start_pos: Position) -> ParseResult<Value> {
863        let mut elements = Vec::new();
864
865        loop {
866            let tok = self.tokenizer.peek_token()?;
867
868            match tok {
869                Token::RightParen => {
870                    self.tokenizer.next_token()?; // Consume )
871                    break;
872                }
873
874                Token::Eof => {
875                    return Err(ParseError::new(
876                        "Unexpected EOF in vector".to_string(),
877                        start_pos,
878                    ))
879                }
880
881                _ => {
882                    elements.push(self.parse_expr()?);
883                }
884            }
885        }
886
887        Ok(Value::vector(elements))
888    }
889}
890
891// =============================================================================
892// Tests
893// =============================================================================
894
895#[cfg(test)]
896mod tests {
897    use super::*;
898
899    #[test]
900    fn test_tokenize_simple() {
901        let mut tok = Tokenizer::new("(+ 1 2)");
902        assert_eq!(tok.next_token().unwrap(), Token::LeftParen);
903        assert_eq!(tok.next_token().unwrap(), Token::Symbol("+".to_string()));
904        assert_eq!(tok.next_token().unwrap(), Token::Integer(1));
905        assert_eq!(tok.next_token().unwrap(), Token::Integer(2));
906        assert_eq!(tok.next_token().unwrap(), Token::RightParen);
907        assert_eq!(tok.next_token().unwrap(), Token::Eof);
908    }
909
910    #[test]
911    fn test_tokenize_whitespace_agnostic() {
912        // This is the critical test that Steel fails!
913        let input = r#"(let ((x 1)
914                            (y 2))
915                         (+ x y))"#;
916        let mut tok = Tokenizer::new(input);
917
918        assert_eq!(tok.next_token().unwrap(), Token::LeftParen);
919        assert_eq!(tok.next_token().unwrap(), Token::Symbol("let".to_string()));
920        assert_eq!(tok.next_token().unwrap(), Token::LeftParen);
921        assert_eq!(tok.next_token().unwrap(), Token::LeftParen);
922        assert_eq!(tok.next_token().unwrap(), Token::Symbol("x".to_string()));
923        assert_eq!(tok.next_token().unwrap(), Token::Integer(1));
924        assert_eq!(tok.next_token().unwrap(), Token::RightParen);
925        // Multi-line whitespace handled correctly!
926        assert_eq!(tok.next_token().unwrap(), Token::LeftParen);
927        assert_eq!(tok.next_token().unwrap(), Token::Symbol("y".to_string()));
928        assert_eq!(tok.next_token().unwrap(), Token::Integer(2));
929    }
930
931    #[test]
932    fn test_tokenize_strings() {
933        let mut tok = Tokenizer::new(r#""hello world""#);
934        assert_eq!(
935            tok.next_token().unwrap(),
936            Token::String("hello world".to_string())
937        );
938
939        let mut tok = Tokenizer::new(r#""with\nnewline""#);
940        assert_eq!(
941            tok.next_token().unwrap(),
942            Token::String("with\nnewline".to_string())
943        );
944    }
945
946    #[test]
947    fn test_tokenize_comments() {
948        let mut tok = Tokenizer::new("(+ 1 ; comment\n 2)");
949        assert_eq!(tok.next_token().unwrap(), Token::LeftParen);
950        assert_eq!(tok.next_token().unwrap(), Token::Symbol("+".to_string()));
951        assert_eq!(tok.next_token().unwrap(), Token::Integer(1));
952        // Comment skipped!
953        assert_eq!(tok.next_token().unwrap(), Token::Integer(2));
954        assert_eq!(tok.next_token().unwrap(), Token::RightParen);
955    }
956
957    #[test]
958    fn test_tokenize_block_comments() {
959        let mut tok = Tokenizer::new("(+ 1 #| block comment |# 2)");
960        assert_eq!(tok.next_token().unwrap(), Token::LeftParen);
961        assert_eq!(tok.next_token().unwrap(), Token::Symbol("+".to_string()));
962        assert_eq!(tok.next_token().unwrap(), Token::Integer(1));
963        // Block comment skipped!
964        assert_eq!(tok.next_token().unwrap(), Token::Integer(2));
965        assert_eq!(tok.next_token().unwrap(), Token::RightParen);
966    }
967
968    #[test]
969    fn test_tokenize_booleans() {
970        let mut tok = Tokenizer::new("#t #f");
971        assert_eq!(tok.next_token().unwrap(), Token::Bool(true));
972        assert_eq!(tok.next_token().unwrap(), Token::Bool(false));
973    }
974
975    #[test]
976    fn test_tokenize_characters() {
977        let mut tok = Tokenizer::new(r#"#\a #\space #\newline"#);
978        assert_eq!(tok.next_token().unwrap(), Token::Char('a'));
979        assert_eq!(tok.next_token().unwrap(), Token::Char(' '));
980        assert_eq!(tok.next_token().unwrap(), Token::Char('\n'));
981    }
982
983    #[test]
984    fn test_tokenize_hex_numbers() {
985        // Lowercase #x
986        let mut tok = Tokenizer::new("#xff");
987        assert_eq!(tok.next_token().unwrap(), Token::Integer(255));
988
989        // Uppercase #X
990        let mut tok = Tokenizer::new("#X10");
991        assert_eq!(tok.next_token().unwrap(), Token::Integer(16));
992
993        // Mixed case
994        let mut tok = Tokenizer::new("#xDEADBEEF");
995        assert_eq!(tok.next_token().unwrap(), Token::Integer(0xDEADBEEF));
996
997        // Zero
998        let mut tok = Tokenizer::new("#x0");
999        assert_eq!(tok.next_token().unwrap(), Token::Integer(0));
1000    }
1001
1002    #[test]
1003    fn test_tokenize_octal_numbers() {
1004        // Lowercase #o
1005        let mut tok = Tokenizer::new("#o77");
1006        assert_eq!(tok.next_token().unwrap(), Token::Integer(63));
1007
1008        // Uppercase #O
1009        let mut tok = Tokenizer::new("#O10");
1010        assert_eq!(tok.next_token().unwrap(), Token::Integer(8));
1011
1012        // Zero
1013        let mut tok = Tokenizer::new("#o0");
1014        assert_eq!(tok.next_token().unwrap(), Token::Integer(0));
1015
1016        // Max valid octal digits
1017        let mut tok = Tokenizer::new("#o777");
1018        assert_eq!(tok.next_token().unwrap(), Token::Integer(511));
1019    }
1020
1021    #[test]
1022    fn test_tokenize_binary_numbers() {
1023        // Lowercase #b
1024        let mut tok = Tokenizer::new("#b1010");
1025        assert_eq!(tok.next_token().unwrap(), Token::Integer(10));
1026
1027        // Uppercase #B
1028        let mut tok = Tokenizer::new("#B1111");
1029        assert_eq!(tok.next_token().unwrap(), Token::Integer(15));
1030
1031        // Zero
1032        let mut tok = Tokenizer::new("#b0");
1033        assert_eq!(tok.next_token().unwrap(), Token::Integer(0));
1034
1035        // All ones
1036        let mut tok = Tokenizer::new("#b11111111");
1037        assert_eq!(tok.next_token().unwrap(), Token::Integer(255));
1038    }
1039
1040    #[test]
1041    fn test_tokenize_quote() {
1042        let mut tok = Tokenizer::new("'(1 2)");
1043        assert_eq!(tok.next_token().unwrap(), Token::Quote);
1044        assert_eq!(tok.next_token().unwrap(), Token::LeftParen);
1045        assert_eq!(tok.next_token().unwrap(), Token::Integer(1));
1046        assert_eq!(tok.next_token().unwrap(), Token::Integer(2));
1047        assert_eq!(tok.next_token().unwrap(), Token::RightParen);
1048    }
1049
1050    #[test]
1051    fn test_error_positions() {
1052        let mut tok = Tokenizer::new("(+ 1\n  \"unclosed string");
1053        tok.next_token().unwrap(); // (
1054        tok.next_token().unwrap(); // +
1055        tok.next_token().unwrap(); // 1
1056
1057        let err = tok.next_token().unwrap_err();
1058        assert_eq!(err.position.line, 2); // Error on line 2
1059        assert!(err.message.contains("EOF in string"));
1060    }
1061
1062    // =================================================================
1063    // Parser Tests
1064    // =================================================================
1065
1066    #[test]
1067    fn test_parse_integer() {
1068        let mut parser = Parser::new("42");
1069        let val = parser.parse().unwrap();
1070        assert!(val.is_integer());
1071        if let Value::Integer(n) = val {
1072            assert_eq!(n, 42);
1073        }
1074    }
1075
1076    #[test]
1077    fn test_parse_simple_list() {
1078        let mut parser = Parser::new("(+ 1 2)");
1079        let val = parser.parse().unwrap();
1080        assert!(val.is_list());
1081
1082        // Check structure: (+ 1 2)
1083        if let Value::Pair(ref p) = val {
1084            let pair = p.borrow();
1085            assert!(pair.car.is_symbol());
1086        }
1087    }
1088
1089    #[test]
1090    fn test_parse_nested_list() {
1091        let mut parser = Parser::new("(+ (* 2 3) 4)");
1092        let val = parser.parse().unwrap();
1093        assert!(val.is_list());
1094    }
1095
1096    #[test]
1097    fn test_parse_quoted() {
1098        let mut parser = Parser::new("'(1 2 3)");
1099        let val = parser.parse().unwrap();
1100
1101        // Should be (quote (1 2 3))
1102        if let Value::Pair(ref p) = val {
1103            let pair = p.borrow();
1104            if let Value::Symbol(s) = &pair.car {
1105                assert_eq!(&**s, "quote");
1106            } else {
1107                panic!("Expected symbol 'quote'");
1108            }
1109        } else {
1110            panic!("Expected pair");
1111        }
1112    }
1113
1114    #[test]
1115    fn test_parse_vector() {
1116        let mut parser = Parser::new("#(1 2 3)");
1117        let val = parser.parse().unwrap();
1118        assert!(val.is_vector());
1119
1120        if let Value::Vector(ref v) = val {
1121            let vec = v.borrow();
1122            assert_eq!(vec.len(), 3);
1123        }
1124    }
1125
1126    #[test]
1127    fn test_parse_dotted_list() {
1128        let mut parser = Parser::new("(1 . 2)");
1129        let val = parser.parse().unwrap();
1130
1131        if let Value::Pair(ref p) = val {
1132            let pair = p.borrow();
1133            assert!(matches!(pair.car, Value::Integer(1)));
1134            assert!(matches!(pair.cdr, Value::Integer(2)));
1135        } else {
1136            panic!("Expected pair");
1137        }
1138    }
1139
1140    #[test]
1141    fn test_parse_string() {
1142        let mut parser = Parser::new(r#""hello world""#);
1143        let val = parser.parse().unwrap();
1144        assert!(val.is_string());
1145    }
1146
1147    #[test]
1148    fn test_parse_bool() {
1149        let mut parser = Parser::new("#t");
1150        let val = parser.parse().unwrap();
1151        assert!(val.is_bool());
1152        assert!(val.is_true());
1153
1154        let mut parser = Parser::new("#f");
1155        let val = parser.parse().unwrap();
1156        assert!(val.is_bool());
1157        assert!(!val.is_true());
1158    }
1159
1160    #[test]
1161    fn test_parse_multiline_let() {
1162        // THE CRITICAL TEST: Multi-line let bindings (breaks Steel!)
1163        let input = r#"
1164            (let ((x 1)
1165                  (y 2))
1166              (+ x y))
1167        "#;
1168
1169        let mut parser = Parser::new(input);
1170        let val = parser.parse().unwrap();
1171        assert!(val.is_list());
1172
1173        // Should parse successfully despite multi-line formatting
1174        // This is what Steel cannot handle!
1175    }
1176
1177    #[test]
1178    fn test_parse_all() {
1179        let input = "(define x 1) (define y 2) (+ x y)";
1180        let mut parser = Parser::new(input);
1181        let exprs = parser.parse_all().unwrap();
1182        assert_eq!(exprs.len(), 3);
1183    }
1184
1185    #[test]
1186    fn test_parse_empty_list() {
1187        let mut parser = Parser::new("()");
1188        let val = parser.parse().unwrap();
1189        assert!(val.is_nil());
1190    }
1191
1192    #[test]
1193    fn test_parse_keyword() {
1194        let mut parser = Parser::new("#:foo");
1195        let val = parser.parse().unwrap();
1196        if let Value::Keyword(ref k) = val {
1197            assert_eq!(&**k, "foo");
1198        } else {
1199            panic!("Expected keyword");
1200        }
1201    }
1202}
1203
dazzle_core/scheme/parser.rs

dazzle_core/scheme/
parser.rs
