dazzle_core/scheme/
parser.rs

1//! Scheme parser (lexer + S-expression parser)
2//!
3//! Ported from OpenJade's `SchemeParser.cxx` (~2,500 lines).
4//!
5//! ## Key Features
6//!
7//! - **Whitespace-agnostic**: Unlike Steel, this parser handles all valid R4RS whitespace
8//! - **Line number tracking**: Error messages report line:column, not byte spans
9//! - **Full R4RS syntax**:
10//!   - Numbers: integers, reals, hex (#x), octal (#o), binary (#b)
11//!   - Strings: escapes (\n, \t, \", \\, \xNN)
12//!   - Symbols and keywords
13//!   - Quote/quasiquote/unquote/unquote-splicing
14//!   - Comments: line (;) and block (#| ... |#)
15//!   - Vectors: #(...)
16//!   - Booleans: #t, #f
17//!   - Characters: #\a, #\space, #\newline
18//!
19//! ## Architecture
20//!
21//! 1. **Lexer** (`Tokenizer`): Character stream → Token stream
22//! 2. **Parser** (`Parser`): Token stream → Value (S-expressions)
23//!
24//! ## OpenJade Correspondence
25//!
26//! | Dazzle        | OpenJade             | Purpose                |
27//! |---------------|----------------------|------------------------|
28//! | `Token`       | `SchemeParser::tok_` | Token types            |
29//! | `Tokenizer`   | `SchemeParser`       | Lexical analysis       |
30//! | `Parser`      | `SchemeParser::get*` | Syntax analysis        |
31//!
32//! ## Error Handling
33//!
34//! Parse errors include:
35//! - Line and column numbers (not byte offsets!)
36//! - Descriptive messages
37//! - Context (what was expected)
38
39use crate::scheme::value::Value;
40use std::fmt;
41
42// =============================================================================
43// Token Types
44// =============================================================================
45
46/// Token type (corresponds to OpenJade's token enum)
47#[derive(Debug, Clone, PartialEq)]
48pub enum Token {
49    // Literals
50    Integer(i64),
51    Real(f64),
52    String(String),
53    Char(char),
54    Symbol(String),
55    Keyword(String),
56    Bool(bool),
57
58    // Delimiters
59    LeftParen,   // (
60    RightParen,  // )
61    LeftBracket, // [  (optional R5RS)
62    RightBracket, // ] (optional R5RS)
63    Dot,         // .
64
65    // Quotation
66    Quote,            // '
67    Quasiquote,       // `
68    Unquote,          // ,
69    UnquoteSplicing,  // ,@
70
71    // Vector
72    VectorStart, // #(
73
74    // End of input
75    Eof,
76}
77
78impl fmt::Display for Token {
79    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
80        match self {
81            Token::Integer(n) => write!(f, "{}", n),
82            Token::Real(n) => write!(f, "{}", n),
83            Token::String(s) => write!(f, "\"{}\"", s),
84            Token::Char(ch) => write!(f, "#\\{}", ch),
85            Token::Symbol(s) => write!(f, "{}", s),
86            Token::Keyword(s) => write!(f, "#:{}", s),
87            Token::Bool(b) => write!(f, "{}", if *b { "#t" } else { "#f" }),
88            Token::LeftParen => write!(f, "("),
89            Token::RightParen => write!(f, ")"),
90            Token::LeftBracket => write!(f, "["),
91            Token::RightBracket => write!(f, "]"),
92            Token::Dot => write!(f, "."),
93            Token::Quote => write!(f, "'"),
94            Token::Quasiquote => write!(f, "`"),
95            Token::Unquote => write!(f, ","),
96            Token::UnquoteSplicing => write!(f, ",@"),
97            Token::VectorStart => write!(f, "#("),
98            Token::Eof => write!(f, "<EOF>"),
99        }
100    }
101}
102
103// =============================================================================
104// Source Position (for error reporting)
105// =============================================================================
106
107/// Source code position (line and column)
108///
109/// **Important**: Line and column numbers start at 1 (human-readable).
110#[derive(Debug, Clone, Copy, PartialEq, Eq)]
111pub struct Position {
112    pub line: usize,
113    pub column: usize,
114}
115
116impl Position {
117    pub fn new() -> Self {
118        Position { line: 1, column: 1 }
119    }
120}
121
122impl Default for Position {
123    fn default() -> Self {
124        Self::new()
125    }
126}
127
128impl fmt::Display for Position {
129    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
130        write!(f, "{}:{}", self.line, self.column)
131    }
132}
133
134// =============================================================================
135// Parse Error
136// =============================================================================
137
138/// Parse error with line:column position
139#[derive(Debug, Clone)]
140pub struct ParseError {
141    pub message: String,
142    pub position: Position,
143}
144
145impl ParseError {
146    pub fn new(message: String, position: Position) -> Self {
147        ParseError { message, position }
148    }
149}
150
151impl fmt::Display for ParseError {
152    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
153        write!(f, "Parse error at {}: {}", self.position, self.message)
154    }
155}
156
157impl std::error::Error for ParseError {}
158
159pub type ParseResult<T> = Result<T, ParseError>;
160
161// =============================================================================
162// Tokenizer (Lexer)
163// =============================================================================
164
165/// Tokenizer for Scheme source code
166///
167/// Corresponds to OpenJade's `SchemeParser` lexical analysis methods.
168///
169/// ## Whitespace Handling
170///
171/// **Critical difference from Steel**: This tokenizer is fully whitespace-agnostic.
172/// Whitespace can appear anywhere between tokens without affecting parsing.
173///
174/// Example that breaks Steel but works here:
175/// ```scheme
176/// (let ((x 1)
177///       (y 2))  ; Multi-line let bindings
178///   (+ x y))
179/// ```
180pub struct Tokenizer {
181    /// Input source code
182    input: Vec<char>,
183
184    /// Current position in input
185    pos: usize,
186
187    /// Current line (1-based)
188    line: usize,
189
190    /// Current column (1-based)
191    column: usize,
192
193    /// Peeked token (for lookahead)
194    peeked: Option<Token>,
195}
196
197impl Tokenizer {
198    /// Create a new tokenizer from source code
199    pub fn new(input: &str) -> Self {
200        Tokenizer {
201            input: input.chars().collect(),
202            pos: 0,
203            line: 1,
204            column: 1,
205            peeked: None,
206        }
207    }
208
209    /// Get current position (for error reporting)
210    pub fn position(&self) -> Position {
211        Position {
212            line: self.line,
213            column: self.column,
214        }
215    }
216
217    /// Peek at current character without consuming
218    fn peek_char(&self) -> Option<char> {
219        if self.pos < self.input.len() {
220            Some(self.input[self.pos])
221        } else {
222            None
223        }
224    }
225
226    /// Peek at character at offset from current position
227    fn peek_char_at(&self, offset: usize) -> Option<char> {
228        let index = self.pos + offset;
229        if index < self.input.len() {
230            Some(self.input[index])
231        } else {
232            None
233        }
234    }
235
236    /// Consume and return current character
237    fn next_char(&mut self) -> Option<char> {
238        if self.pos < self.input.len() {
239            let ch = self.input[self.pos];
240            self.pos += 1;
241
242            // Update line/column tracking
243            if ch == '\n' {
244                self.line += 1;
245                self.column = 1;
246            } else {
247                self.column += 1;
248            }
249
250            Some(ch)
251        } else {
252            None
253        }
254    }
255
256    /// Skip whitespace and comments
257    ///
258    /// **Key feature**: Handles both line comments (;) and block comments (#| ... |#)
259    fn skip_whitespace(&mut self) {
260        loop {
261            match self.peek_char() {
262                // Whitespace
263                Some(ch) if ch.is_whitespace() => {
264                    self.next_char();
265                }
266
267                // Line comment: ; to end of line
268                Some(';') => {
269                    self.next_char();
270                    while let Some(ch) = self.peek_char() {
271                        self.next_char();
272                        if ch == '\n' {
273                            break;
274                        }
275                    }
276                }
277
278                // Block comment: #| ... |#
279                Some('#') if self.peek_char_at(1) == Some('|') => {
280                    self.next_char(); // #
281                    self.next_char(); // |
282
283                    // Find closing |#
284                    let mut depth = 1;
285                    while depth > 0 {
286                        match self.next_char() {
287                            Some('|') if self.peek_char() == Some('#') => {
288                                self.next_char(); // #
289                                depth -= 1;
290                            }
291                            Some('#') if self.peek_char() == Some('|') => {
292                                self.next_char(); // |
293                                depth += 1; // Nested block comment
294                            }
295                            Some(_) => {} // Continue
296                            None => break, // EOF in comment (error, but tolerate)
297                        }
298                    }
299                }
300
301                // Not whitespace or comment
302                _ => break,
303            }
304        }
305    }
306
307    /// Check if character is a delimiter (ends a token)
308    fn is_delimiter(ch: char) -> bool {
309        ch.is_whitespace()
310            || matches!(
311                ch,
312                '(' | ')' | '[' | ']' | '"' | ';' | ',' | '`' | '\''
313            )
314    }
315
316    /// Parse an integer or real number
317    fn parse_number(&mut self, start_pos: Position) -> ParseResult<Token> {
318        let mut num_str = String::new();
319
320        // Collect digits and special characters
321        while let Some(ch) = self.peek_char() {
322            if ch.is_ascii_digit() || matches!(ch, '.' | 'e' | 'E' | '+' | '-') {
323                num_str.push(ch);
324                self.next_char();
325            } else if Self::is_delimiter(ch) {
326                break;
327            } else {
328                // Invalid character in number
329                return Err(ParseError::new(
330                    format!("Invalid character in number: {}", ch),
331                    start_pos,
332                ));
333            }
334        }
335
336        // Try parsing as integer first
337        if let Ok(n) = num_str.parse::<i64>() {
338            return Ok(Token::Integer(n));
339        }
340
341        // Try parsing as float
342        if let Ok(n) = num_str.parse::<f64>() {
343            return Ok(Token::Real(n));
344        }
345
346        Err(ParseError::new(
347            format!("Invalid number: {}", num_str),
348            start_pos,
349        ))
350    }
351
352    /// Parse a hexadecimal number (#x prefix)
353    fn parse_hex_number(&mut self, start_pos: Position) -> ParseResult<Token> {
354        let mut num_str = String::new();
355
356        while let Some(ch) = self.peek_char() {
357            if ch.is_ascii_hexdigit() {
358                num_str.push(ch);
359                self.next_char();
360            } else if Self::is_delimiter(ch) {
361                break;
362            } else {
363                return Err(ParseError::new(
364                    format!("Invalid character in hex number: {}", ch),
365                    start_pos,
366                ));
367            }
368        }
369
370        if num_str.is_empty() {
371            return Err(ParseError::new("Empty hex number".to_string(), start_pos));
372        }
373
374        i64::from_str_radix(&num_str, 16)
375            .map(Token::Integer)
376            .map_err(|_| ParseError::new(format!("Invalid hex number: {}", num_str), start_pos))
377    }
378
379    /// Parse an octal number (#o prefix)
380    fn parse_octal_number(&mut self, start_pos: Position) -> ParseResult<Token> {
381        let mut num_str = String::new();
382
383        while let Some(ch) = self.peek_char() {
384            if ch.is_digit(8) {
385                num_str.push(ch);
386                self.next_char();
387            } else if Self::is_delimiter(ch) {
388                break;
389            } else {
390                return Err(ParseError::new(
391                    format!("Invalid character in octal number: {}", ch),
392                    start_pos,
393                ));
394            }
395        }
396
397        if num_str.is_empty() {
398            return Err(ParseError::new("Empty octal number".to_string(), start_pos));
399        }
400
401        i64::from_str_radix(&num_str, 8)
402            .map(Token::Integer)
403            .map_err(|_| ParseError::new(format!("Invalid octal number: {}", num_str), start_pos))
404    }
405
406    /// Parse a binary number (#b prefix)
407    fn parse_binary_number(&mut self, start_pos: Position) -> ParseResult<Token> {
408        let mut num_str = String::new();
409
410        while let Some(ch) = self.peek_char() {
411            if matches!(ch, '0' | '1') {
412                num_str.push(ch);
413                self.next_char();
414            } else if Self::is_delimiter(ch) {
415                break;
416            } else {
417                return Err(ParseError::new(
418                    format!("Invalid character in binary number: {}", ch),
419                    start_pos,
420                ));
421            }
422        }
423
424        if num_str.is_empty() {
425            return Err(ParseError::new("Empty binary number".to_string(), start_pos));
426        }
427
428        i64::from_str_radix(&num_str, 2)
429            .map(Token::Integer)
430            .map_err(|_| ParseError::new(format!("Invalid binary number: {}", num_str), start_pos))
431    }
432
433    /// Parse a symbol or keyword
434    fn parse_symbol(&mut self) -> String {
435        let mut sym = String::new();
436
437        while let Some(ch) = self.peek_char() {
438            if Self::is_delimiter(ch) {
439                break;
440            }
441            sym.push(ch);
442            self.next_char();
443        }
444
445        sym
446    }
447
448    /// Parse a string literal
449    fn parse_string(&mut self, start_pos: Position) -> ParseResult<String> {
450        self.next_char(); // Consume opening "
451
452        let mut result = String::new();
453
454        loop {
455            match self.next_char() {
456                Some('"') => {
457                    // Closing quote
458                    // Normalize CRLF to LF (to match OpenJade behavior)
459                    // OpenJade always outputs Unix line endings regardless of template line endings
460                    let normalized = result.replace("\r\n", "\n");
461                    return Ok(normalized);
462                }
463                Some('\\') => {
464                    // Escape sequence
465                    match self.next_char() {
466                        Some('n') => result.push('\n'),
467                        Some('t') => result.push('\t'),
468                        Some('r') => result.push('\r'),
469                        Some('\\') => result.push('\\'),
470                        Some('"') => result.push('"'),
471                        Some(ch) => result.push(ch), // Unknown escape, keep literal
472                        None => {
473                            return Err(ParseError::new(
474                                "Unexpected EOF in string escape".to_string(),
475                                start_pos,
476                            ))
477                        }
478                    }
479                }
480                Some(ch) => {
481                    result.push(ch);
482                }
483                None => {
484                    return Err(ParseError::new(
485                        "Unexpected EOF in string".to_string(),
486                        start_pos,
487                    ))
488                }
489            }
490        }
491    }
492
493    /// Parse a character literal (#\a, #\space, #\newline)
494    /// Called after # has been consumed
495    fn parse_char(&mut self, start_pos: Position) -> ParseResult<char> {
496        // Expect backslash
497        if self.next_char() != Some('\\') {
498            return Err(ParseError::new(
499                "Expected \\ after # in character literal".to_string(),
500                start_pos,
501            ));
502        }
503
504        // Read character name
505        let mut name = String::new();
506        while let Some(ch) = self.peek_char() {
507            if Self::is_delimiter(ch) {
508                break;
509            }
510            name.push(ch);
511            self.next_char();
512        }
513
514        if name.is_empty() {
515            return Err(ParseError::new(
516                "Empty character literal".to_string(),
517                start_pos,
518            ));
519        }
520
521        // Named characters
522        match name.as_str() {
523            "space" => Ok(' '),
524            "newline" => Ok('\n'),
525            "tab" => Ok('\t'),
526            "return" => Ok('\r'),
527            // OpenJade Unicode character literal: #\U-XXXX (e.g., #\U-00E4 for ä)
528            // Format: U-XXXX where XXXX is hexadecimal Unicode code point
529            s if s.starts_with("U-") => {
530                let hex_str = &s[2..]; // Skip "U-"
531                u32::from_str_radix(hex_str, 16)
532                    .ok()
533                    .and_then(std::char::from_u32)
534                    .ok_or_else(|| ParseError::new(
535                        format!("Invalid Unicode character literal: #\\{}", name),
536                        start_pos,
537                    ))
538            }
539            // Accept any single Unicode character (handles UTF-8 multi-byte sequences)
540            // OpenJade accepts UTF-8 characters in character literals for define-language
541            s if s.chars().count() == 1 => Ok(s.chars().next().unwrap()),
542            _ => Err(ParseError::new(
543                format!("Invalid character literal: #\\{}", name),
544                start_pos,
545            )),
546        }
547    }
548
549    /// Parse a CDATA string literal: <![CDATA[...]]>
550    /// This is an OpenJade extension for multi-line string literals
551    /// The content between <![CDATA[ and ]]> is returned as a string token
552    fn parse_cdata_string(&mut self, start_pos: Position) -> ParseResult<Token> {
553        // Skip "<![CDATA["
554        for _ in 0..9 {
555            self.next_char();
556        }
557
558        let mut content = String::new();
559
560        // Read until we find "]]>"
561        loop {
562            match self.peek_char() {
563                None => {
564                    return Err(ParseError::new(
565                        "Unclosed CDATA section: missing ]]>".to_string(),
566                        start_pos,
567                    ));
568                }
569                Some(']') => {
570                    // Check if this is the closing ]]>
571                    if self.pos + 2 < self.input.len()
572                        && self.input[self.pos] == ']'
573                        && self.input[self.pos + 1] == ']'
574                        && self.input[self.pos + 2] == '>'
575                    {
576                        // Skip ]]>
577                        self.next_char(); // ]
578                        self.next_char(); // ]
579                        self.next_char(); // >
580                        break;
581                    } else {
582                        content.push(']');
583                        self.next_char();
584                    }
585                }
586                Some(ch) => {
587                    content.push(ch);
588                    self.next_char();
589                }
590            }
591        }
592
593        Ok(Token::String(content))
594    }
595
596    /// Get the next token
597    pub fn next_token(&mut self) -> ParseResult<Token> {
598        // Check if we have a peeked token
599        if let Some(tok) = self.peeked.take() {
600            return Ok(tok);
601        }
602
603        // Skip whitespace and comments
604        self.skip_whitespace();
605
606        let start_pos = self.position();
607
608        match self.peek_char() {
609            None => Ok(Token::Eof),
610
611            Some('(') => {
612                self.next_char();
613                Ok(Token::LeftParen)
614            }
615
616            Some(')') => {
617                self.next_char();
618                Ok(Token::RightParen)
619            }
620
621            Some('[') => {
622                self.next_char();
623                Ok(Token::LeftBracket)
624            }
625
626            Some(']') => {
627                self.next_char();
628                Ok(Token::RightBracket)
629            }
630
631            Some('\'') => {
632                self.next_char();
633                Ok(Token::Quote)
634            }
635
636            Some('`') => {
637                self.next_char();
638                Ok(Token::Quasiquote)
639            }
640
641            Some(',') => {
642                self.next_char();
643                // Check for ,@
644                if self.peek_char() == Some('@') {
645                    self.next_char();
646                    Ok(Token::UnquoteSplicing)
647                } else {
648                    Ok(Token::Unquote)
649                }
650            }
651
652            Some('"') => {
653                let s = self.parse_string(start_pos)?;
654                Ok(Token::String(s))
655            }
656
657            Some('#') => {
658                self.next_char(); // Consume #
659                match self.peek_char() {
660                    Some('t') => {
661                        self.next_char();
662                        Ok(Token::Bool(true))
663                    }
664                    Some('f') => {
665                        self.next_char();
666                        Ok(Token::Bool(false))
667                    }
668                    Some('(') => {
669                        self.next_char();
670                        Ok(Token::VectorStart)
671                    }
672                    Some('\\') => {
673                        let ch = self.parse_char(start_pos)?;
674                        Ok(Token::Char(ch))
675                    }
676                    Some(':') => {
677                        self.next_char(); // Consume :
678                        let name = self.parse_symbol();
679                        Ok(Token::Keyword(name))
680                    }
681                    Some('x') | Some('X') => {
682                        self.next_char(); // Consume x
683                        self.parse_hex_number(start_pos)
684                    }
685                    Some('o') | Some('O') => {
686                        self.next_char(); // Consume o
687                        self.parse_octal_number(start_pos)
688                    }
689                    Some('b') | Some('B') => {
690                        self.next_char(); // Consume b
691                        self.parse_binary_number(start_pos)
692                    }
693                    _ => Err(ParseError::new(
694                        format!("Invalid # syntax: #{:?}", self.peek_char()),
695                        start_pos,
696                    )),
697                }
698            }
699
700            Some(ch) if ch.is_ascii_digit() => self.parse_number(start_pos),
701
702            Some('+') | Some('-') => {
703                // Could be number or symbol
704                if let Some(next) = self.peek_char_at(1) {
705                    if next.is_ascii_digit() {
706                        self.parse_number(start_pos)
707                    } else {
708                        let sym = self.parse_symbol();
709                        Ok(Token::Symbol(sym))
710                    }
711                } else {
712                    let sym = self.parse_symbol();
713                    Ok(Token::Symbol(sym))
714                }
715            }
716
717            Some('.') => {
718                // Could be dot or number starting with .
719                if let Some(next) = self.peek_char_at(1) {
720                    if next.is_ascii_digit() {
721                        self.parse_number(start_pos)
722                    } else {
723                        self.next_char();
724                        Ok(Token::Dot)
725                    }
726                } else {
727                    self.next_char();
728                    Ok(Token::Dot)
729                }
730            }
731
732            Some('<') => {
733                // Check for CDATA section: <![CDATA[...]]>
734                // This is an OpenJade extension for multi-line string literals
735                let cdata_prefix = ['<', '!', '[', 'C', 'D', 'A', 'T', 'A', '['];
736                let is_cdata = self.pos + cdata_prefix.len() <= self.input.len()
737                    && self.input[self.pos..self.pos + cdata_prefix.len()] == cdata_prefix;
738
739                if is_cdata {
740                    self.parse_cdata_string(start_pos)
741                } else {
742                    // Regular < symbol
743                    let sym = self.parse_symbol();
744                    Ok(Token::Symbol(sym))
745                }
746            }
747
748            Some(_) => {
749                // Symbol or DSSSL keyword (trailing colon)
750                let sym = self.parse_symbol();
751
752                // DSSSL uses trailing colon for keywords: name:
753                if sym.ends_with(':') {
754                    let keyword_name = sym[..sym.len()-1].to_string();
755                    Ok(Token::Keyword(keyword_name))
756                } else {
757                    Ok(Token::Symbol(sym))
758                }
759            }
760        }
761    }
762
763    /// Peek at the next token without consuming it
764    pub fn peek_token(&mut self) -> ParseResult<&Token> {
765        if self.peeked.is_none() {
766            let tok = self.next_token()?;
767            self.peeked = Some(tok);
768        }
769        Ok(self.peeked.as_ref().unwrap())
770    }
771}
772
773// =============================================================================
774// Parser (S-expression builder)
775// =============================================================================
776
777/// Parser for building Scheme values from tokens
778///
779/// Corresponds to OpenJade's `SchemeParser::get*` methods.
780///
781/// ## Usage
782///
783/// ```ignore
784/// let parser = Parser::new("(+ 1 2)");
785/// let expr = parser.parse().unwrap();
786/// ```
787pub struct Parser {
788    tokenizer: Tokenizer,
789}
790
791impl Parser {
792    /// Create a new parser from source code
793    pub fn new(input: &str) -> Self {
794        Parser {
795            tokenizer: Tokenizer::new(input),
796        }
797    }
798
799    /// Parse a single S-expression
800    ///
801    /// Returns `Ok(Value)` on success, `Err(ParseError)` on failure.
802    pub fn parse(&mut self) -> ParseResult<Value> {
803        self.parse_expr()
804    }
805
806    /// Parse all S-expressions in input
807    ///
808    /// Returns a list of all top-level expressions.
809    pub fn parse_all(&mut self) -> ParseResult<Vec<Value>> {
810        let mut exprs = Vec::new();
811
812        loop {
813            let tok = self.tokenizer.peek_token()?;
814            if *tok == Token::Eof {
815                break;
816            }
817            exprs.push(self.parse_expr()?);
818        }
819
820        Ok(exprs)
821    }
822
823    /// Parse a single expression
824    fn parse_expr(&mut self) -> ParseResult<Value> {
825        let start_pos = self.tokenizer.position();
826        let tok = self.tokenizer.next_token()?;
827
828        match tok {
829            // Literals
830            Token::Integer(n) => Ok(Value::integer(n)),
831            Token::Real(n) => Ok(Value::real(n)),
832            Token::String(s) => Ok(Value::string(s)),
833            Token::Char(ch) => Ok(Value::char(ch)),
834            Token::Bool(b) => Ok(Value::bool(b)),
835            Token::Symbol(s) => Ok(Value::symbol(&s)),
836            Token::Keyword(s) => Ok(Value::keyword(&s)),
837
838            // Lists
839            Token::LeftParen | Token::LeftBracket => self.parse_list(start_pos),
840
841            // Vectors
842            Token::VectorStart => self.parse_vector(start_pos),
843
844            // Quote
845            Token::Quote => {
846                let quoted = self.parse_expr()?;
847                Ok(Value::cons(Value::symbol("quote"), Value::cons(quoted, Value::Nil)))
848            }
849
850            // Quasiquote
851            Token::Quasiquote => {
852                let quoted = self.parse_expr()?;
853                Ok(Value::cons(
854                    Value::symbol("quasiquote"),
855                    Value::cons(quoted, Value::Nil),
856                ))
857            }
858
859            // Unquote
860            Token::Unquote => {
861                let quoted = self.parse_expr()?;
862                Ok(Value::cons(
863                    Value::symbol("unquote"),
864                    Value::cons(quoted, Value::Nil),
865                ))
866            }
867
868            // Unquote-splicing
869            Token::UnquoteSplicing => {
870                let quoted = self.parse_expr()?;
871                Ok(Value::cons(
872                    Value::symbol("unquote-splicing"),
873                    Value::cons(quoted, Value::Nil),
874                ))
875            }
876
877            // Unexpected tokens
878            Token::RightParen | Token::RightBracket => Err(ParseError::new(
879                format!("Unexpected closing delimiter: {}", tok),
880                start_pos,
881            )),
882
883            Token::Dot => Err(ParseError::new(
884                "Unexpected dot outside of list".to_string(),
885                start_pos,
886            )),
887
888            Token::Eof => Err(ParseError::new(
889                "Unexpected end of input".to_string(),
890                start_pos,
891            )),
892        }
893    }
894
895    /// Parse a list (after opening paren consumed)
896    fn parse_list(&mut self, start_pos: Position) -> ParseResult<Value> {
897        let mut elements = Vec::new();
898        let mut dotted_tail = None;
899
900        loop {
901            let tok = self.tokenizer.peek_token()?;
902
903            match tok {
904                Token::RightParen | Token::RightBracket => {
905                    self.tokenizer.next_token()?; // Consume closing paren
906                    break;
907                }
908
909                Token::Dot => {
910                    self.tokenizer.next_token()?; // Consume dot
911
912                    // Parse the tail
913                    dotted_tail = Some(self.parse_expr()?);
914
915                    // Expect closing paren
916                    let tok = self.tokenizer.next_token()?;
917                    if !matches!(tok, Token::RightParen | Token::RightBracket) {
918                        return Err(ParseError::new(
919                            format!("Expected ) after dotted tail, got {}", tok),
920                            start_pos,
921                        ));
922                    }
923                    break;
924                }
925
926                Token::Eof => {
927                    return Err(ParseError::new(
928                        "Unexpected EOF in list".to_string(),
929                        start_pos,
930                    ))
931                }
932
933                _ => {
934                    elements.push(self.parse_expr()?);
935                }
936            }
937        }
938
939        // Build the list from right to left
940        let mut result = dotted_tail.unwrap_or(Value::Nil);
941        for elem in elements.into_iter().rev() {
942            result = Value::cons(elem, result);
943        }
944
945        Ok(result)
946    }
947
948    /// Parse a vector (after #( consumed)
949    fn parse_vector(&mut self, start_pos: Position) -> ParseResult<Value> {
950        let mut elements = Vec::new();
951
952        loop {
953            let tok = self.tokenizer.peek_token()?;
954
955            match tok {
956                Token::RightParen => {
957                    self.tokenizer.next_token()?; // Consume )
958                    break;
959                }
960
961                Token::Eof => {
962                    return Err(ParseError::new(
963                        "Unexpected EOF in vector".to_string(),
964                        start_pos,
965                    ))
966                }
967
968                _ => {
969                    elements.push(self.parse_expr()?);
970                }
971            }
972        }
973
974        Ok(Value::vector(elements))
975    }
976}
977
978// =============================================================================
979// Tests
980// =============================================================================
981
982#[cfg(test)]
983mod tests {
984    use super::*;
985
986    #[test]
987    fn test_tokenize_simple() {
988        let mut tok = Tokenizer::new("(+ 1 2)");
989        assert_eq!(tok.next_token().unwrap(), Token::LeftParen);
990        assert_eq!(tok.next_token().unwrap(), Token::Symbol("+".to_string()));
991        assert_eq!(tok.next_token().unwrap(), Token::Integer(1));
992        assert_eq!(tok.next_token().unwrap(), Token::Integer(2));
993        assert_eq!(tok.next_token().unwrap(), Token::RightParen);
994        assert_eq!(tok.next_token().unwrap(), Token::Eof);
995    }
996
997    #[test]
998    fn test_tokenize_whitespace_agnostic() {
999        // This is the critical test that Steel fails!
1000        let input = r#"(let ((x 1)
1001                            (y 2))
1002                         (+ x y))"#;
1003        let mut tok = Tokenizer::new(input);
1004
1005        assert_eq!(tok.next_token().unwrap(), Token::LeftParen);
1006        assert_eq!(tok.next_token().unwrap(), Token::Symbol("let".to_string()));
1007        assert_eq!(tok.next_token().unwrap(), Token::LeftParen);
1008        assert_eq!(tok.next_token().unwrap(), Token::LeftParen);
1009        assert_eq!(tok.next_token().unwrap(), Token::Symbol("x".to_string()));
1010        assert_eq!(tok.next_token().unwrap(), Token::Integer(1));
1011        assert_eq!(tok.next_token().unwrap(), Token::RightParen);
1012        // Multi-line whitespace handled correctly!
1013        assert_eq!(tok.next_token().unwrap(), Token::LeftParen);
1014        assert_eq!(tok.next_token().unwrap(), Token::Symbol("y".to_string()));
1015        assert_eq!(tok.next_token().unwrap(), Token::Integer(2));
1016    }
1017
1018    #[test]
1019    fn test_tokenize_strings() {
1020        let mut tok = Tokenizer::new(r#""hello world""#);
1021        assert_eq!(
1022            tok.next_token().unwrap(),
1023            Token::String("hello world".to_string())
1024        );
1025
1026        let mut tok = Tokenizer::new(r#""with\nnewline""#);
1027        assert_eq!(
1028            tok.next_token().unwrap(),
1029            Token::String("with\nnewline".to_string())
1030        );
1031    }
1032
1033    #[test]
1034    fn test_tokenize_cdata() {
1035        // Test CDATA section parsing (OpenJade extension)
1036        let mut tok = Tokenizer::new(r#"<![CDATA[<!DOCTYPE HTML>]]>"#);
1037        assert_eq!(
1038            tok.next_token().unwrap(),
1039            Token::String("<!DOCTYPE HTML>".to_string())
1040        );
1041
1042        // Test CDATA with newlines
1043        let mut tok = Tokenizer::new("<![CDATA[\nLine 1\nLine 2\n]]>");
1044        assert_eq!(
1045            tok.next_token().unwrap(),
1046            Token::String("\nLine 1\nLine 2\n".to_string())
1047        );
1048
1049        // Test CDATA in expression context
1050        let mut tok = Tokenizer::new("(define x <![CDATA[test]]>)");
1051        assert_eq!(tok.next_token().unwrap(), Token::LeftParen);
1052        assert_eq!(tok.next_token().unwrap(), Token::Symbol("define".to_string()));
1053        assert_eq!(tok.next_token().unwrap(), Token::Symbol("x".to_string()));
1054        assert_eq!(tok.next_token().unwrap(), Token::String("test".to_string()));
1055        assert_eq!(tok.next_token().unwrap(), Token::RightParen);
1056    }
1057
1058    #[test]
1059    fn test_tokenize_comments() {
1060        let mut tok = Tokenizer::new("(+ 1 ; comment\n 2)");
1061        assert_eq!(tok.next_token().unwrap(), Token::LeftParen);
1062        assert_eq!(tok.next_token().unwrap(), Token::Symbol("+".to_string()));
1063        assert_eq!(tok.next_token().unwrap(), Token::Integer(1));
1064        // Comment skipped!
1065        assert_eq!(tok.next_token().unwrap(), Token::Integer(2));
1066        assert_eq!(tok.next_token().unwrap(), Token::RightParen);
1067    }
1068
1069    #[test]
1070    fn test_tokenize_block_comments() {
1071        let mut tok = Tokenizer::new("(+ 1 #| block comment |# 2)");
1072        assert_eq!(tok.next_token().unwrap(), Token::LeftParen);
1073        assert_eq!(tok.next_token().unwrap(), Token::Symbol("+".to_string()));
1074        assert_eq!(tok.next_token().unwrap(), Token::Integer(1));
1075        // Block comment skipped!
1076        assert_eq!(tok.next_token().unwrap(), Token::Integer(2));
1077        assert_eq!(tok.next_token().unwrap(), Token::RightParen);
1078    }
1079
1080    #[test]
1081    fn test_tokenize_booleans() {
1082        let mut tok = Tokenizer::new("#t #f");
1083        assert_eq!(tok.next_token().unwrap(), Token::Bool(true));
1084        assert_eq!(tok.next_token().unwrap(), Token::Bool(false));
1085    }
1086
1087    #[test]
1088    fn test_tokenize_characters() {
1089        let mut tok = Tokenizer::new(r#"#\a #\space #\newline"#);
1090        assert_eq!(tok.next_token().unwrap(), Token::Char('a'));
1091        assert_eq!(tok.next_token().unwrap(), Token::Char(' '));
1092        assert_eq!(tok.next_token().unwrap(), Token::Char('\n'));
1093    }
1094
1095    #[test]
1096    fn test_tokenize_hex_numbers() {
1097        // Lowercase #x
1098        let mut tok = Tokenizer::new("#xff");
1099        assert_eq!(tok.next_token().unwrap(), Token::Integer(255));
1100
1101        // Uppercase #X
1102        let mut tok = Tokenizer::new("#X10");
1103        assert_eq!(tok.next_token().unwrap(), Token::Integer(16));
1104
1105        // Mixed case
1106        let mut tok = Tokenizer::new("#xDEADBEEF");
1107        assert_eq!(tok.next_token().unwrap(), Token::Integer(0xDEADBEEF));
1108
1109        // Zero
1110        let mut tok = Tokenizer::new("#x0");
1111        assert_eq!(tok.next_token().unwrap(), Token::Integer(0));
1112    }
1113
1114    #[test]
1115    fn test_tokenize_octal_numbers() {
1116        // Lowercase #o
1117        let mut tok = Tokenizer::new("#o77");
1118        assert_eq!(tok.next_token().unwrap(), Token::Integer(63));
1119
1120        // Uppercase #O
1121        let mut tok = Tokenizer::new("#O10");
1122        assert_eq!(tok.next_token().unwrap(), Token::Integer(8));
1123
1124        // Zero
1125        let mut tok = Tokenizer::new("#o0");
1126        assert_eq!(tok.next_token().unwrap(), Token::Integer(0));
1127
1128        // Max valid octal digits
1129        let mut tok = Tokenizer::new("#o777");
1130        assert_eq!(tok.next_token().unwrap(), Token::Integer(511));
1131    }
1132
1133    #[test]
1134    fn test_tokenize_binary_numbers() {
1135        // Lowercase #b
1136        let mut tok = Tokenizer::new("#b1010");
1137        assert_eq!(tok.next_token().unwrap(), Token::Integer(10));
1138
1139        // Uppercase #B
1140        let mut tok = Tokenizer::new("#B1111");
1141        assert_eq!(tok.next_token().unwrap(), Token::Integer(15));
1142
1143        // Zero
1144        let mut tok = Tokenizer::new("#b0");
1145        assert_eq!(tok.next_token().unwrap(), Token::Integer(0));
1146
1147        // All ones
1148        let mut tok = Tokenizer::new("#b11111111");
1149        assert_eq!(tok.next_token().unwrap(), Token::Integer(255));
1150    }
1151
1152    #[test]
1153    fn test_tokenize_quote() {
1154        let mut tok = Tokenizer::new("'(1 2)");
1155        assert_eq!(tok.next_token().unwrap(), Token::Quote);
1156        assert_eq!(tok.next_token().unwrap(), Token::LeftParen);
1157        assert_eq!(tok.next_token().unwrap(), Token::Integer(1));
1158        assert_eq!(tok.next_token().unwrap(), Token::Integer(2));
1159        assert_eq!(tok.next_token().unwrap(), Token::RightParen);
1160    }
1161
1162    #[test]
1163    fn test_error_positions() {
1164        let mut tok = Tokenizer::new("(+ 1\n  \"unclosed string");
1165        tok.next_token().unwrap(); // (
1166        tok.next_token().unwrap(); // +
1167        tok.next_token().unwrap(); // 1
1168
1169        let err = tok.next_token().unwrap_err();
1170        assert_eq!(err.position.line, 2); // Error on line 2
1171        assert!(err.message.contains("EOF in string"));
1172    }
1173
1174    // =================================================================
1175    // Parser Tests
1176    // =================================================================
1177
1178    #[test]
1179    fn test_parse_integer() {
1180        let mut parser = Parser::new("42");
1181        let val = parser.parse().unwrap();
1182        assert!(val.is_integer());
1183        if let Value::Integer(n) = val {
1184            assert_eq!(n, 42);
1185        }
1186    }
1187
1188    #[test]
1189    fn test_parse_simple_list() {
1190        let mut parser = Parser::new("(+ 1 2)");
1191        let val = parser.parse().unwrap();
1192        assert!(val.is_list());
1193
1194        // Check structure: (+ 1 2)
1195        if let Value::Pair(ref p) = val {
1196            let pair = p.borrow();
1197            assert!(pair.car.is_symbol());
1198        }
1199    }
1200
1201    #[test]
1202    fn test_parse_nested_list() {
1203        let mut parser = Parser::new("(+ (* 2 3) 4)");
1204        let val = parser.parse().unwrap();
1205        assert!(val.is_list());
1206    }
1207
1208    #[test]
1209    fn test_parse_quoted() {
1210        let mut parser = Parser::new("'(1 2 3)");
1211        let val = parser.parse().unwrap();
1212
1213        // Should be (quote (1 2 3))
1214        if let Value::Pair(ref p) = val {
1215            let pair = p.borrow();
1216            if let Value::Symbol(s) = &pair.car {
1217                assert_eq!(&**s, "quote");
1218            } else {
1219                panic!("Expected symbol 'quote'");
1220            }
1221        } else {
1222            panic!("Expected pair");
1223        }
1224    }
1225
1226    #[test]
1227    fn test_parse_vector() {
1228        let mut parser = Parser::new("#(1 2 3)");
1229        let val = parser.parse().unwrap();
1230        assert!(val.is_vector());
1231
1232        if let Value::Vector(ref v) = val {
1233            let vec = v.borrow();
1234            assert_eq!(vec.len(), 3);
1235        }
1236    }
1237
1238    #[test]
1239    fn test_parse_dotted_list() {
1240        let mut parser = Parser::new("(1 . 2)");
1241        let val = parser.parse().unwrap();
1242
1243        if let Value::Pair(ref p) = val {
1244            let pair = p.borrow();
1245            assert!(matches!(pair.car, Value::Integer(1)));
1246            assert!(matches!(pair.cdr, Value::Integer(2)));
1247        } else {
1248            panic!("Expected pair");
1249        }
1250    }
1251
1252    #[test]
1253    fn test_parse_string() {
1254        let mut parser = Parser::new(r#""hello world""#);
1255        let val = parser.parse().unwrap();
1256        assert!(val.is_string());
1257    }
1258
1259    #[test]
1260    fn test_parse_bool() {
1261        let mut parser = Parser::new("#t");
1262        let val = parser.parse().unwrap();
1263        assert!(val.is_bool());
1264        assert!(val.is_true());
1265
1266        let mut parser = Parser::new("#f");
1267        let val = parser.parse().unwrap();
1268        assert!(val.is_bool());
1269        assert!(!val.is_true());
1270    }
1271
1272    #[test]
1273    fn test_parse_multiline_let() {
1274        // THE CRITICAL TEST: Multi-line let bindings (breaks Steel!)
1275        let input = r#"
1276            (let ((x 1)
1277                  (y 2))
1278              (+ x y))
1279        "#;
1280
1281        let mut parser = Parser::new(input);
1282        let val = parser.parse().unwrap();
1283        assert!(val.is_list());
1284
1285        // Should parse successfully despite multi-line formatting
1286        // This is what Steel cannot handle!
1287    }
1288
1289    #[test]
1290    fn test_parse_all() {
1291        let input = "(define x 1) (define y 2) (+ x y)";
1292        let mut parser = Parser::new(input);
1293        let exprs = parser.parse_all().unwrap();
1294        assert_eq!(exprs.len(), 3);
1295    }
1296
1297    #[test]
1298    fn test_parse_empty_list() {
1299        let mut parser = Parser::new("()");
1300        let val = parser.parse().unwrap();
1301        assert!(val.is_nil());
1302    }
1303
1304    #[test]
1305    fn test_parse_keyword() {
1306        let mut parser = Parser::new("#:foo");
1307        let val = parser.parse().unwrap();
1308        if let Value::Keyword(ref k) = val {
1309            assert_eq!(&**k, "foo");
1310        } else {
1311            panic!("Expected keyword");
1312        }
1313    }
1314}
1315
dazzle_core/scheme/parser.rs

dazzle_core/scheme/
parser.rs
