dazzle_core/scheme/
parser.rs

1//! Scheme parser (lexer + S-expression parser)
2//!
3//! Ported from OpenJade's `SchemeParser.cxx` (~2,500 lines).
4//!
5//! ## Key Features
6//!
7//! - **Whitespace-agnostic**: Unlike Steel, this parser handles all valid R4RS whitespace
8//! - **Line number tracking**: Error messages report line:column, not byte spans
9//! - **Full R4RS syntax**:
10//!   - Numbers: integers, reals, hex (#x), octal (#o), binary (#b)
11//!   - Strings: escapes (\n, \t, \", \\, \xNN)
12//!   - Symbols and keywords
13//!   - Quote/quasiquote/unquote/unquote-splicing
14//!   - Comments: line (;) and block (#| ... |#)
15//!   - Vectors: #(...)
16//!   - Booleans: #t, #f
17//!   - Characters: #\a, #\space, #\newline
18//!
19//! ## Architecture
20//!
21//! 1. **Lexer** (`Tokenizer`): Character stream → Token stream
22//! 2. **Parser** (`Parser`): Token stream → Value (S-expressions)
23//!
24//! ## OpenJade Correspondence
25//!
26//! | Dazzle        | OpenJade             | Purpose                |
27//! |---------------|----------------------|------------------------|
28//! | `Token`       | `SchemeParser::tok_` | Token types            |
29//! | `Tokenizer`   | `SchemeParser`       | Lexical analysis       |
30//! | `Parser`      | `SchemeParser::get*` | Syntax analysis        |
31//!
32//! ## Error Handling
33//!
34//! Parse errors include:
35//! - Line and column numbers (not byte offsets!)
36//! - Descriptive messages
37//! - Context (what was expected)
38
39use crate::scheme::value::Value;
40use std::fmt;
41
42// =============================================================================
43// Token Types
44// =============================================================================
45
46/// Token type (corresponds to OpenJade's token enum)
47#[derive(Debug, Clone, PartialEq)]
48pub enum Token {
49    // Literals
50    Integer(i64),
51    Real(f64),
52    String(String),
53    Char(char),
54    Symbol(String),
55    Keyword(String),
56    Bool(bool),
57
58    // Delimiters
59    LeftParen,   // (
60    RightParen,  // )
61    LeftBracket, // [  (optional R5RS)
62    RightBracket, // ] (optional R5RS)
63    Dot,         // .
64
65    // Quotation
66    Quote,            // '
67    Quasiquote,       // `
68    Unquote,          // ,
69    UnquoteSplicing,  // ,@
70
71    // Vector
72    VectorStart, // #(
73
74    // End of input
75    Eof,
76}
77
78impl fmt::Display for Token {
79    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
80        match self {
81            Token::Integer(n) => write!(f, "{}", n),
82            Token::Real(n) => write!(f, "{}", n),
83            Token::String(s) => write!(f, "\"{}\"", s),
84            Token::Char(ch) => write!(f, "#\\{}", ch),
85            Token::Symbol(s) => write!(f, "{}", s),
86            Token::Keyword(s) => write!(f, "#:{}", s),
87            Token::Bool(b) => write!(f, "{}", if *b { "#t" } else { "#f" }),
88            Token::LeftParen => write!(f, "("),
89            Token::RightParen => write!(f, ")"),
90            Token::LeftBracket => write!(f, "["),
91            Token::RightBracket => write!(f, "]"),
92            Token::Dot => write!(f, "."),
93            Token::Quote => write!(f, "'"),
94            Token::Quasiquote => write!(f, "`"),
95            Token::Unquote => write!(f, ","),
96            Token::UnquoteSplicing => write!(f, ",@"),
97            Token::VectorStart => write!(f, "#("),
98            Token::Eof => write!(f, "<EOF>"),
99        }
100    }
101}
102
103// =============================================================================
104// Source Position (for error reporting)
105// =============================================================================
106
107/// Source code position (line and column)
108///
109/// **Important**: Line and column numbers start at 1 (human-readable).
110#[derive(Debug, Clone, Copy, PartialEq, Eq)]
111pub struct Position {
112    pub line: usize,
113    pub column: usize,
114}
115
116impl Position {
117    pub fn new() -> Self {
118        Position { line: 1, column: 1 }
119    }
120}
121
122impl Default for Position {
123    fn default() -> Self {
124        Self::new()
125    }
126}
127
128impl fmt::Display for Position {
129    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
130        write!(f, "{}:{}", self.line, self.column)
131    }
132}
133
134// =============================================================================
135// Parse Error
136// =============================================================================
137
138/// Parse error with line:column position
139#[derive(Debug, Clone)]
140pub struct ParseError {
141    pub message: String,
142    pub position: Position,
143}
144
145impl ParseError {
146    pub fn new(message: String, position: Position) -> Self {
147        ParseError { message, position }
148    }
149}
150
151impl fmt::Display for ParseError {
152    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
153        write!(f, "Parse error at {}: {}", self.position, self.message)
154    }
155}
156
157impl std::error::Error for ParseError {}
158
159pub type ParseResult<T> = Result<T, ParseError>;
160
161// =============================================================================
162// Tokenizer (Lexer)
163// =============================================================================
164
165/// Tokenizer for Scheme source code
166///
167/// Corresponds to OpenJade's `SchemeParser` lexical analysis methods.
168///
169/// ## Whitespace Handling
170///
171/// **Critical difference from Steel**: This tokenizer is fully whitespace-agnostic.
172/// Whitespace can appear anywhere between tokens without affecting parsing.
173///
174/// Example that breaks Steel but works here:
175/// ```scheme
176/// (let ((x 1)
177///       (y 2))  ; Multi-line let bindings
178///   (+ x y))
179/// ```
180pub struct Tokenizer {
181    /// Input source code
182    input: Vec<char>,
183
184    /// Current position in input
185    pos: usize,
186
187    /// Current line (1-based)
188    line: usize,
189
190    /// Current column (1-based)
191    column: usize,
192
193    /// Peeked token (for lookahead)
194    peeked: Option<Token>,
195}
196
197impl Tokenizer {
198    /// Create a new tokenizer from source code
199    pub fn new(input: &str) -> Self {
200        Tokenizer {
201            input: input.chars().collect(),
202            pos: 0,
203            line: 1,
204            column: 1,
205            peeked: None,
206        }
207    }
208
209    /// Get current position (for error reporting)
210    pub fn position(&self) -> Position {
211        Position {
212            line: self.line,
213            column: self.column,
214        }
215    }
216
217    /// Peek at current character without consuming
218    fn peek_char(&self) -> Option<char> {
219        if self.pos < self.input.len() {
220            Some(self.input[self.pos])
221        } else {
222            None
223        }
224    }
225
226    /// Peek at character at offset from current position
227    fn peek_char_at(&self, offset: usize) -> Option<char> {
228        let index = self.pos + offset;
229        if index < self.input.len() {
230            Some(self.input[index])
231        } else {
232            None
233        }
234    }
235
236    /// Consume and return current character
237    fn next_char(&mut self) -> Option<char> {
238        if self.pos < self.input.len() {
239            let ch = self.input[self.pos];
240            self.pos += 1;
241
242            // Update line/column tracking
243            if ch == '\n' {
244                self.line += 1;
245                self.column = 1;
246            } else {
247                self.column += 1;
248            }
249
250            Some(ch)
251        } else {
252            None
253        }
254    }
255
256    /// Skip whitespace and comments
257    ///
258    /// **Key feature**: Handles both line comments (;) and block comments (#| ... |#)
259    fn skip_whitespace(&mut self) {
260        loop {
261            match self.peek_char() {
262                // Whitespace
263                Some(ch) if ch.is_whitespace() => {
264                    self.next_char();
265                }
266
267                // Line comment: ; to end of line
268                Some(';') => {
269                    self.next_char();
270                    while let Some(ch) = self.peek_char() {
271                        self.next_char();
272                        if ch == '\n' {
273                            break;
274                        }
275                    }
276                }
277
278                // Block comment: #| ... |#
279                Some('#') if self.peek_char_at(1) == Some('|') => {
280                    self.next_char(); // #
281                    self.next_char(); // |
282
283                    // Find closing |#
284                    let mut depth = 1;
285                    while depth > 0 {
286                        match self.next_char() {
287                            Some('|') if self.peek_char() == Some('#') => {
288                                self.next_char(); // #
289                                depth -= 1;
290                            }
291                            Some('#') if self.peek_char() == Some('|') => {
292                                self.next_char(); // |
293                                depth += 1; // Nested block comment
294                            }
295                            Some(_) => {} // Continue
296                            None => break, // EOF in comment (error, but tolerate)
297                        }
298                    }
299                }
300
301                // Not whitespace or comment
302                _ => break,
303            }
304        }
305    }
306
307    /// Check if character is a delimiter (ends a token)
308    fn is_delimiter(ch: char) -> bool {
309        ch.is_whitespace()
310            || matches!(
311                ch,
312                '(' | ')' | '[' | ']' | '"' | ';' | ',' | '`' | '\''
313            )
314    }
315
316    /// Parse an integer or real number
317    fn parse_number(&mut self, start_pos: Position) -> ParseResult<Token> {
318        let mut num_str = String::new();
319
320        // Collect digits and special characters
321        while let Some(ch) = self.peek_char() {
322            if ch.is_ascii_digit() || matches!(ch, '.' | 'e' | 'E' | '+' | '-') {
323                num_str.push(ch);
324                self.next_char();
325            } else if Self::is_delimiter(ch) {
326                break;
327            } else {
328                // Invalid character in number
329                return Err(ParseError::new(
330                    format!("Invalid character in number: {}", ch),
331                    start_pos,
332                ));
333            }
334        }
335
336        // Try parsing as integer first
337        if let Ok(n) = num_str.parse::<i64>() {
338            return Ok(Token::Integer(n));
339        }
340
341        // Try parsing as float
342        if let Ok(n) = num_str.parse::<f64>() {
343            return Ok(Token::Real(n));
344        }
345
346        Err(ParseError::new(
347            format!("Invalid number: {}", num_str),
348            start_pos,
349        ))
350    }
351
352    /// Parse a hexadecimal number (#x prefix)
353    fn parse_hex_number(&mut self, start_pos: Position) -> ParseResult<Token> {
354        let mut num_str = String::new();
355
356        while let Some(ch) = self.peek_char() {
357            if ch.is_ascii_hexdigit() {
358                num_str.push(ch);
359                self.next_char();
360            } else if Self::is_delimiter(ch) {
361                break;
362            } else {
363                return Err(ParseError::new(
364                    format!("Invalid character in hex number: {}", ch),
365                    start_pos,
366                ));
367            }
368        }
369
370        if num_str.is_empty() {
371            return Err(ParseError::new("Empty hex number".to_string(), start_pos));
372        }
373
374        i64::from_str_radix(&num_str, 16)
375            .map(Token::Integer)
376            .map_err(|_| ParseError::new(format!("Invalid hex number: {}", num_str), start_pos))
377    }
378
379    /// Parse an octal number (#o prefix)
380    fn parse_octal_number(&mut self, start_pos: Position) -> ParseResult<Token> {
381        let mut num_str = String::new();
382
383        while let Some(ch) = self.peek_char() {
384            if ch.is_digit(8) {
385                num_str.push(ch);
386                self.next_char();
387            } else if Self::is_delimiter(ch) {
388                break;
389            } else {
390                return Err(ParseError::new(
391                    format!("Invalid character in octal number: {}", ch),
392                    start_pos,
393                ));
394            }
395        }
396
397        if num_str.is_empty() {
398            return Err(ParseError::new("Empty octal number".to_string(), start_pos));
399        }
400
401        i64::from_str_radix(&num_str, 8)
402            .map(Token::Integer)
403            .map_err(|_| ParseError::new(format!("Invalid octal number: {}", num_str), start_pos))
404    }
405
406    /// Parse a binary number (#b prefix)
407    fn parse_binary_number(&mut self, start_pos: Position) -> ParseResult<Token> {
408        let mut num_str = String::new();
409
410        while let Some(ch) = self.peek_char() {
411            if matches!(ch, '0' | '1') {
412                num_str.push(ch);
413                self.next_char();
414            } else if Self::is_delimiter(ch) {
415                break;
416            } else {
417                return Err(ParseError::new(
418                    format!("Invalid character in binary number: {}", ch),
419                    start_pos,
420                ));
421            }
422        }
423
424        if num_str.is_empty() {
425            return Err(ParseError::new("Empty binary number".to_string(), start_pos));
426        }
427
428        i64::from_str_radix(&num_str, 2)
429            .map(Token::Integer)
430            .map_err(|_| ParseError::new(format!("Invalid binary number: {}", num_str), start_pos))
431    }
432
433    /// Parse a symbol or keyword
434    fn parse_symbol(&mut self) -> String {
435        let mut sym = String::new();
436
437        while let Some(ch) = self.peek_char() {
438            if Self::is_delimiter(ch) {
439                break;
440            }
441            sym.push(ch);
442            self.next_char();
443        }
444
445        sym
446    }
447
448    /// Parse a string literal
449    fn parse_string(&mut self, start_pos: Position) -> ParseResult<String> {
450        self.next_char(); // Consume opening "
451
452        let mut result = String::new();
453
454        loop {
455            match self.next_char() {
456                Some('"') => {
457                    // Closing quote
458                    return Ok(result);
459                }
460                Some('\\') => {
461                    // Escape sequence
462                    match self.next_char() {
463                        Some('n') => result.push('\n'),
464                        Some('t') => result.push('\t'),
465                        Some('r') => result.push('\r'),
466                        Some('\\') => result.push('\\'),
467                        Some('"') => result.push('"'),
468                        Some(ch) => result.push(ch), // Unknown escape, keep literal
469                        None => {
470                            return Err(ParseError::new(
471                                "Unexpected EOF in string escape".to_string(),
472                                start_pos,
473                            ))
474                        }
475                    }
476                }
477                Some(ch) => {
478                    result.push(ch);
479                }
480                None => {
481                    return Err(ParseError::new(
482                        "Unexpected EOF in string".to_string(),
483                        start_pos,
484                    ))
485                }
486            }
487        }
488    }
489
490    /// Parse a character literal (#\a, #\space, #\newline)
491    /// Called after # has been consumed
492    fn parse_char(&mut self, start_pos: Position) -> ParseResult<char> {
493        // Expect backslash
494        if self.next_char() != Some('\\') {
495            return Err(ParseError::new(
496                "Expected \\ after # in character literal".to_string(),
497                start_pos,
498            ));
499        }
500
501        // Read character name
502        let mut name = String::new();
503        while let Some(ch) = self.peek_char() {
504            if Self::is_delimiter(ch) {
505                break;
506            }
507            name.push(ch);
508            self.next_char();
509        }
510
511        if name.is_empty() {
512            return Err(ParseError::new(
513                "Empty character literal".to_string(),
514                start_pos,
515            ));
516        }
517
518        // Named characters
519        match name.as_str() {
520            "space" => Ok(' '),
521            "newline" => Ok('\n'),
522            "tab" => Ok('\t'),
523            "return" => Ok('\r'),
524            s if s.len() == 1 => Ok(s.chars().next().unwrap()),
525            _ => Err(ParseError::new(
526                format!("Invalid character literal: #\\{}", name),
527                start_pos,
528            )),
529        }
530    }
531
532    /// Get the next token
533    pub fn next_token(&mut self) -> ParseResult<Token> {
534        // Check if we have a peeked token
535        if let Some(tok) = self.peeked.take() {
536            return Ok(tok);
537        }
538
539        // Skip whitespace and comments
540        self.skip_whitespace();
541
542        let start_pos = self.position();
543
544        match self.peek_char() {
545            None => Ok(Token::Eof),
546
547            Some('(') => {
548                self.next_char();
549                Ok(Token::LeftParen)
550            }
551
552            Some(')') => {
553                self.next_char();
554                Ok(Token::RightParen)
555            }
556
557            Some('[') => {
558                self.next_char();
559                Ok(Token::LeftBracket)
560            }
561
562            Some(']') => {
563                self.next_char();
564                Ok(Token::RightBracket)
565            }
566
567            Some('\'') => {
568                self.next_char();
569                Ok(Token::Quote)
570            }
571
572            Some('`') => {
573                self.next_char();
574                Ok(Token::Quasiquote)
575            }
576
577            Some(',') => {
578                self.next_char();
579                // Check for ,@
580                if self.peek_char() == Some('@') {
581                    self.next_char();
582                    Ok(Token::UnquoteSplicing)
583                } else {
584                    Ok(Token::Unquote)
585                }
586            }
587
588            Some('"') => {
589                let s = self.parse_string(start_pos)?;
590                Ok(Token::String(s))
591            }
592
593            Some('#') => {
594                self.next_char(); // Consume #
595                match self.peek_char() {
596                    Some('t') => {
597                        self.next_char();
598                        Ok(Token::Bool(true))
599                    }
600                    Some('f') => {
601                        self.next_char();
602                        Ok(Token::Bool(false))
603                    }
604                    Some('(') => {
605                        self.next_char();
606                        Ok(Token::VectorStart)
607                    }
608                    Some('\\') => {
609                        let ch = self.parse_char(start_pos)?;
610                        Ok(Token::Char(ch))
611                    }
612                    Some(':') => {
613                        self.next_char(); // Consume :
614                        let name = self.parse_symbol();
615                        Ok(Token::Keyword(name))
616                    }
617                    Some('x') | Some('X') => {
618                        self.next_char(); // Consume x
619                        self.parse_hex_number(start_pos)
620                    }
621                    Some('o') | Some('O') => {
622                        self.next_char(); // Consume o
623                        self.parse_octal_number(start_pos)
624                    }
625                    Some('b') | Some('B') => {
626                        self.next_char(); // Consume b
627                        self.parse_binary_number(start_pos)
628                    }
629                    _ => Err(ParseError::new(
630                        format!("Invalid # syntax: #{:?}", self.peek_char()),
631                        start_pos,
632                    )),
633                }
634            }
635
636            Some(ch) if ch.is_ascii_digit() => self.parse_number(start_pos),
637
638            Some('+') | Some('-') => {
639                // Could be number or symbol
640                if let Some(next) = self.peek_char_at(1) {
641                    if next.is_ascii_digit() {
642                        self.parse_number(start_pos)
643                    } else {
644                        let sym = self.parse_symbol();
645                        Ok(Token::Symbol(sym))
646                    }
647                } else {
648                    let sym = self.parse_symbol();
649                    Ok(Token::Symbol(sym))
650                }
651            }
652
653            Some('.') => {
654                // Could be dot or number starting with .
655                if let Some(next) = self.peek_char_at(1) {
656                    if next.is_ascii_digit() {
657                        self.parse_number(start_pos)
658                    } else {
659                        self.next_char();
660                        Ok(Token::Dot)
661                    }
662                } else {
663                    self.next_char();
664                    Ok(Token::Dot)
665                }
666            }
667
668            Some(_) => {
669                // Symbol or DSSSL keyword (trailing colon)
670                let sym = self.parse_symbol();
671
672                // DSSSL uses trailing colon for keywords: name:
673                if sym.ends_with(':') {
674                    let keyword_name = sym[..sym.len()-1].to_string();
675                    Ok(Token::Keyword(keyword_name))
676                } else {
677                    Ok(Token::Symbol(sym))
678                }
679            }
680        }
681    }
682
683    /// Peek at the next token without consuming it
684    pub fn peek_token(&mut self) -> ParseResult<&Token> {
685        if self.peeked.is_none() {
686            let tok = self.next_token()?;
687            self.peeked = Some(tok);
688        }
689        Ok(self.peeked.as_ref().unwrap())
690    }
691}
692
693// =============================================================================
694// Parser (S-expression builder)
695// =============================================================================
696
697/// Parser for building Scheme values from tokens
698///
699/// Corresponds to OpenJade's `SchemeParser::get*` methods.
700///
701/// ## Usage
702///
703/// ```ignore
704/// let parser = Parser::new("(+ 1 2)");
705/// let expr = parser.parse().unwrap();
706/// ```
707pub struct Parser {
708    tokenizer: Tokenizer,
709}
710
711impl Parser {
712    /// Create a new parser from source code
713    pub fn new(input: &str) -> Self {
714        Parser {
715            tokenizer: Tokenizer::new(input),
716        }
717    }
718
719    /// Parse a single S-expression
720    ///
721    /// Returns `Ok(Value)` on success, `Err(ParseError)` on failure.
722    pub fn parse(&mut self) -> ParseResult<Value> {
723        self.parse_expr()
724    }
725
726    /// Parse all S-expressions in input
727    ///
728    /// Returns a list of all top-level expressions.
729    pub fn parse_all(&mut self) -> ParseResult<Vec<Value>> {
730        let mut exprs = Vec::new();
731
732        loop {
733            let tok = self.tokenizer.peek_token()?;
734            if *tok == Token::Eof {
735                break;
736            }
737            exprs.push(self.parse_expr()?);
738        }
739
740        Ok(exprs)
741    }
742
743    /// Parse a single expression
744    fn parse_expr(&mut self) -> ParseResult<Value> {
745        let start_pos = self.tokenizer.position();
746        let tok = self.tokenizer.next_token()?;
747
748        match tok {
749            // Literals
750            Token::Integer(n) => Ok(Value::integer(n)),
751            Token::Real(n) => Ok(Value::real(n)),
752            Token::String(s) => Ok(Value::string(s)),
753            Token::Char(ch) => Ok(Value::char(ch)),
754            Token::Bool(b) => Ok(Value::bool(b)),
755            Token::Symbol(s) => Ok(Value::symbol(&s)),
756            Token::Keyword(s) => Ok(Value::keyword(&s)),
757
758            // Lists
759            Token::LeftParen | Token::LeftBracket => self.parse_list(start_pos),
760
761            // Vectors
762            Token::VectorStart => self.parse_vector(start_pos),
763
764            // Quote
765            Token::Quote => {
766                let quoted = self.parse_expr()?;
767                Ok(Value::cons(Value::symbol("quote"), Value::cons(quoted, Value::Nil)))
768            }
769
770            // Quasiquote
771            Token::Quasiquote => {
772                let quoted = self.parse_expr()?;
773                Ok(Value::cons(
774                    Value::symbol("quasiquote"),
775                    Value::cons(quoted, Value::Nil),
776                ))
777            }
778
779            // Unquote
780            Token::Unquote => {
781                let quoted = self.parse_expr()?;
782                Ok(Value::cons(
783                    Value::symbol("unquote"),
784                    Value::cons(quoted, Value::Nil),
785                ))
786            }
787
788            // Unquote-splicing
789            Token::UnquoteSplicing => {
790                let quoted = self.parse_expr()?;
791                Ok(Value::cons(
792                    Value::symbol("unquote-splicing"),
793                    Value::cons(quoted, Value::Nil),
794                ))
795            }
796
797            // Unexpected tokens
798            Token::RightParen | Token::RightBracket => Err(ParseError::new(
799                format!("Unexpected closing delimiter: {}", tok),
800                start_pos,
801            )),
802
803            Token::Dot => Err(ParseError::new(
804                "Unexpected dot outside of list".to_string(),
805                start_pos,
806            )),
807
808            Token::Eof => Err(ParseError::new(
809                "Unexpected end of input".to_string(),
810                start_pos,
811            )),
812        }
813    }
814
815    /// Parse a list (after opening paren consumed)
816    fn parse_list(&mut self, start_pos: Position) -> ParseResult<Value> {
817        let mut elements = Vec::new();
818        let mut dotted_tail = None;
819
820        loop {
821            let tok = self.tokenizer.peek_token()?;
822
823            match tok {
824                Token::RightParen | Token::RightBracket => {
825                    self.tokenizer.next_token()?; // Consume closing paren
826                    break;
827                }
828
829                Token::Dot => {
830                    self.tokenizer.next_token()?; // Consume dot
831
832                    // Parse the tail
833                    dotted_tail = Some(self.parse_expr()?);
834
835                    // Expect closing paren
836                    let tok = self.tokenizer.next_token()?;
837                    if !matches!(tok, Token::RightParen | Token::RightBracket) {
838                        return Err(ParseError::new(
839                            format!("Expected ) after dotted tail, got {}", tok),
840                            start_pos,
841                        ));
842                    }
843                    break;
844                }
845
846                Token::Eof => {
847                    return Err(ParseError::new(
848                        "Unexpected EOF in list".to_string(),
849                        start_pos,
850                    ))
851                }
852
853                _ => {
854                    elements.push(self.parse_expr()?);
855                }
856            }
857        }
858
859        // Build the list from right to left
860        let mut result = dotted_tail.unwrap_or(Value::Nil);
861        for elem in elements.into_iter().rev() {
862            result = Value::cons(elem, result);
863        }
864
865        Ok(result)
866    }
867
868    /// Parse a vector (after #( consumed)
869    fn parse_vector(&mut self, start_pos: Position) -> ParseResult<Value> {
870        let mut elements = Vec::new();
871
872        loop {
873            let tok = self.tokenizer.peek_token()?;
874
875            match tok {
876                Token::RightParen => {
877                    self.tokenizer.next_token()?; // Consume )
878                    break;
879                }
880
881                Token::Eof => {
882                    return Err(ParseError::new(
883                        "Unexpected EOF in vector".to_string(),
884                        start_pos,
885                    ))
886                }
887
888                _ => {
889                    elements.push(self.parse_expr()?);
890                }
891            }
892        }
893
894        Ok(Value::vector(elements))
895    }
896}
897
898// =============================================================================
899// Tests
900// =============================================================================
901
902#[cfg(test)]
903mod tests {
904    use super::*;
905
906    #[test]
907    fn test_tokenize_simple() {
908        let mut tok = Tokenizer::new("(+ 1 2)");
909        assert_eq!(tok.next_token().unwrap(), Token::LeftParen);
910        assert_eq!(tok.next_token().unwrap(), Token::Symbol("+".to_string()));
911        assert_eq!(tok.next_token().unwrap(), Token::Integer(1));
912        assert_eq!(tok.next_token().unwrap(), Token::Integer(2));
913        assert_eq!(tok.next_token().unwrap(), Token::RightParen);
914        assert_eq!(tok.next_token().unwrap(), Token::Eof);
915    }
916
917    #[test]
918    fn test_tokenize_whitespace_agnostic() {
919        // This is the critical test that Steel fails!
920        let input = r#"(let ((x 1)
921                            (y 2))
922                         (+ x y))"#;
923        let mut tok = Tokenizer::new(input);
924
925        assert_eq!(tok.next_token().unwrap(), Token::LeftParen);
926        assert_eq!(tok.next_token().unwrap(), Token::Symbol("let".to_string()));
927        assert_eq!(tok.next_token().unwrap(), Token::LeftParen);
928        assert_eq!(tok.next_token().unwrap(), Token::LeftParen);
929        assert_eq!(tok.next_token().unwrap(), Token::Symbol("x".to_string()));
930        assert_eq!(tok.next_token().unwrap(), Token::Integer(1));
931        assert_eq!(tok.next_token().unwrap(), Token::RightParen);
932        // Multi-line whitespace handled correctly!
933        assert_eq!(tok.next_token().unwrap(), Token::LeftParen);
934        assert_eq!(tok.next_token().unwrap(), Token::Symbol("y".to_string()));
935        assert_eq!(tok.next_token().unwrap(), Token::Integer(2));
936    }
937
938    #[test]
939    fn test_tokenize_strings() {
940        let mut tok = Tokenizer::new(r#""hello world""#);
941        assert_eq!(
942            tok.next_token().unwrap(),
943            Token::String("hello world".to_string())
944        );
945
946        let mut tok = Tokenizer::new(r#""with\nnewline""#);
947        assert_eq!(
948            tok.next_token().unwrap(),
949            Token::String("with\nnewline".to_string())
950        );
951    }
952
953    #[test]
954    fn test_tokenize_comments() {
955        let mut tok = Tokenizer::new("(+ 1 ; comment\n 2)");
956        assert_eq!(tok.next_token().unwrap(), Token::LeftParen);
957        assert_eq!(tok.next_token().unwrap(), Token::Symbol("+".to_string()));
958        assert_eq!(tok.next_token().unwrap(), Token::Integer(1));
959        // Comment skipped!
960        assert_eq!(tok.next_token().unwrap(), Token::Integer(2));
961        assert_eq!(tok.next_token().unwrap(), Token::RightParen);
962    }
963
964    #[test]
965    fn test_tokenize_block_comments() {
966        let mut tok = Tokenizer::new("(+ 1 #| block comment |# 2)");
967        assert_eq!(tok.next_token().unwrap(), Token::LeftParen);
968        assert_eq!(tok.next_token().unwrap(), Token::Symbol("+".to_string()));
969        assert_eq!(tok.next_token().unwrap(), Token::Integer(1));
970        // Block comment skipped!
971        assert_eq!(tok.next_token().unwrap(), Token::Integer(2));
972        assert_eq!(tok.next_token().unwrap(), Token::RightParen);
973    }
974
975    #[test]
976    fn test_tokenize_booleans() {
977        let mut tok = Tokenizer::new("#t #f");
978        assert_eq!(tok.next_token().unwrap(), Token::Bool(true));
979        assert_eq!(tok.next_token().unwrap(), Token::Bool(false));
980    }
981
982    #[test]
983    fn test_tokenize_characters() {
984        let mut tok = Tokenizer::new(r#"#\a #\space #\newline"#);
985        assert_eq!(tok.next_token().unwrap(), Token::Char('a'));
986        assert_eq!(tok.next_token().unwrap(), Token::Char(' '));
987        assert_eq!(tok.next_token().unwrap(), Token::Char('\n'));
988    }
989
990    #[test]
991    fn test_tokenize_hex_numbers() {
992        // Lowercase #x
993        let mut tok = Tokenizer::new("#xff");
994        assert_eq!(tok.next_token().unwrap(), Token::Integer(255));
995
996        // Uppercase #X
997        let mut tok = Tokenizer::new("#X10");
998        assert_eq!(tok.next_token().unwrap(), Token::Integer(16));
999
1000        // Mixed case
1001        let mut tok = Tokenizer::new("#xDEADBEEF");
1002        assert_eq!(tok.next_token().unwrap(), Token::Integer(0xDEADBEEF));
1003
1004        // Zero
1005        let mut tok = Tokenizer::new("#x0");
1006        assert_eq!(tok.next_token().unwrap(), Token::Integer(0));
1007    }
1008
1009    #[test]
1010    fn test_tokenize_octal_numbers() {
1011        // Lowercase #o
1012        let mut tok = Tokenizer::new("#o77");
1013        assert_eq!(tok.next_token().unwrap(), Token::Integer(63));
1014
1015        // Uppercase #O
1016        let mut tok = Tokenizer::new("#O10");
1017        assert_eq!(tok.next_token().unwrap(), Token::Integer(8));
1018
1019        // Zero
1020        let mut tok = Tokenizer::new("#o0");
1021        assert_eq!(tok.next_token().unwrap(), Token::Integer(0));
1022
1023        // Max valid octal digits
1024        let mut tok = Tokenizer::new("#o777");
1025        assert_eq!(tok.next_token().unwrap(), Token::Integer(511));
1026    }
1027
1028    #[test]
1029    fn test_tokenize_binary_numbers() {
1030        // Lowercase #b
1031        let mut tok = Tokenizer::new("#b1010");
1032        assert_eq!(tok.next_token().unwrap(), Token::Integer(10));
1033
1034        // Uppercase #B
1035        let mut tok = Tokenizer::new("#B1111");
1036        assert_eq!(tok.next_token().unwrap(), Token::Integer(15));
1037
1038        // Zero
1039        let mut tok = Tokenizer::new("#b0");
1040        assert_eq!(tok.next_token().unwrap(), Token::Integer(0));
1041
1042        // All ones
1043        let mut tok = Tokenizer::new("#b11111111");
1044        assert_eq!(tok.next_token().unwrap(), Token::Integer(255));
1045    }
1046
1047    #[test]
1048    fn test_tokenize_quote() {
1049        let mut tok = Tokenizer::new("'(1 2)");
1050        assert_eq!(tok.next_token().unwrap(), Token::Quote);
1051        assert_eq!(tok.next_token().unwrap(), Token::LeftParen);
1052        assert_eq!(tok.next_token().unwrap(), Token::Integer(1));
1053        assert_eq!(tok.next_token().unwrap(), Token::Integer(2));
1054        assert_eq!(tok.next_token().unwrap(), Token::RightParen);
1055    }
1056
1057    #[test]
1058    fn test_error_positions() {
1059        let mut tok = Tokenizer::new("(+ 1\n  \"unclosed string");
1060        tok.next_token().unwrap(); // (
1061        tok.next_token().unwrap(); // +
1062        tok.next_token().unwrap(); // 1
1063
1064        let err = tok.next_token().unwrap_err();
1065        assert_eq!(err.position.line, 2); // Error on line 2
1066        assert!(err.message.contains("EOF in string"));
1067    }
1068
1069    // =================================================================
1070    // Parser Tests
1071    // =================================================================
1072
1073    #[test]
1074    fn test_parse_integer() {
1075        let mut parser = Parser::new("42");
1076        let val = parser.parse().unwrap();
1077        assert!(val.is_integer());
1078        if let Value::Integer(n) = val {
1079            assert_eq!(n, 42);
1080        }
1081    }
1082
1083    #[test]
1084    fn test_parse_simple_list() {
1085        let mut parser = Parser::new("(+ 1 2)");
1086        let val = parser.parse().unwrap();
1087        assert!(val.is_list());
1088
1089        // Check structure: (+ 1 2)
1090        if let Value::Pair(ref p) = val {
1091            let pair = p.borrow();
1092            assert!(pair.car.is_symbol());
1093        }
1094    }
1095
1096    #[test]
1097    fn test_parse_nested_list() {
1098        let mut parser = Parser::new("(+ (* 2 3) 4)");
1099        let val = parser.parse().unwrap();
1100        assert!(val.is_list());
1101    }
1102
1103    #[test]
1104    fn test_parse_quoted() {
1105        let mut parser = Parser::new("'(1 2 3)");
1106        let val = parser.parse().unwrap();
1107
1108        // Should be (quote (1 2 3))
1109        if let Value::Pair(ref p) = val {
1110            let pair = p.borrow();
1111            if let Value::Symbol(s) = &pair.car {
1112                assert_eq!(&**s, "quote");
1113            } else {
1114                panic!("Expected symbol 'quote'");
1115            }
1116        } else {
1117            panic!("Expected pair");
1118        }
1119    }
1120
1121    #[test]
1122    fn test_parse_vector() {
1123        let mut parser = Parser::new("#(1 2 3)");
1124        let val = parser.parse().unwrap();
1125        assert!(val.is_vector());
1126
1127        if let Value::Vector(ref v) = val {
1128            let vec = v.borrow();
1129            assert_eq!(vec.len(), 3);
1130        }
1131    }
1132
1133    #[test]
1134    fn test_parse_dotted_list() {
1135        let mut parser = Parser::new("(1 . 2)");
1136        let val = parser.parse().unwrap();
1137
1138        if let Value::Pair(ref p) = val {
1139            let pair = p.borrow();
1140            assert!(matches!(pair.car, Value::Integer(1)));
1141            assert!(matches!(pair.cdr, Value::Integer(2)));
1142        } else {
1143            panic!("Expected pair");
1144        }
1145    }
1146
1147    #[test]
1148    fn test_parse_string() {
1149        let mut parser = Parser::new(r#""hello world""#);
1150        let val = parser.parse().unwrap();
1151        assert!(val.is_string());
1152    }
1153
1154    #[test]
1155    fn test_parse_bool() {
1156        let mut parser = Parser::new("#t");
1157        let val = parser.parse().unwrap();
1158        assert!(val.is_bool());
1159        assert!(val.is_true());
1160
1161        let mut parser = Parser::new("#f");
1162        let val = parser.parse().unwrap();
1163        assert!(val.is_bool());
1164        assert!(!val.is_true());
1165    }
1166
1167    #[test]
1168    fn test_parse_multiline_let() {
1169        // THE CRITICAL TEST: Multi-line let bindings (breaks Steel!)
1170        let input = r#"
1171            (let ((x 1)
1172                  (y 2))
1173              (+ x y))
1174        "#;
1175
1176        let mut parser = Parser::new(input);
1177        let val = parser.parse().unwrap();
1178        assert!(val.is_list());
1179
1180        // Should parse successfully despite multi-line formatting
1181        // This is what Steel cannot handle!
1182    }
1183
1184    #[test]
1185    fn test_parse_all() {
1186        let input = "(define x 1) (define y 2) (+ x y)";
1187        let mut parser = Parser::new(input);
1188        let exprs = parser.parse_all().unwrap();
1189        assert_eq!(exprs.len(), 3);
1190    }
1191
1192    #[test]
1193    fn test_parse_empty_list() {
1194        let mut parser = Parser::new("()");
1195        let val = parser.parse().unwrap();
1196        assert!(val.is_nil());
1197    }
1198
1199    #[test]
1200    fn test_parse_keyword() {
1201        let mut parser = Parser::new("#:foo");
1202        let val = parser.parse().unwrap();
1203        if let Value::Keyword(ref k) = val {
1204            assert_eq!(&**k, "foo");
1205        } else {
1206            panic!("Expected keyword");
1207        }
1208    }
1209}
1210
dazzle_core/scheme/parser.rs

dazzle_core/scheme/
parser.rs
