forgedb_parser/
lexer.rs

1// Re-export Position from validation crate for consistency
2pub use forgedb_validation::Position;
3
4/// Token with position information
5#[derive(Debug, Clone, PartialEq)]
6pub struct TokenWithPos {
7    pub token: Token,
8    pub position: Position,
9}
10
11/// Token types for the schema language
12#[derive(Debug, Clone, PartialEq)]
13pub enum Token {
14    // Identifiers and literals
15    Ident(String),
16
17    // Types
18    TypeU32,
19    TypeU64,
20    TypeI32,
21    TypeI64,
22    TypeF64,
23    TypeBool,
24    TypeString,
25    TypeUuid,
26    TypeTimestamp,
27    TypeChar, // char(N) - fixed-size character array
28
29    // Keywords
30    KwStruct, // struct
31
32    // Symbols
33    Plus,        // +
34    Ampersand,   // &
35    Caret,       // ^
36    Colon,       // :
37    LBrace,      // {
38    RBrace,      // }
39    LBracket,    // [
40    RBracket,    // ]
41    Asterisk,    // *
42    Question,    // ?
43    At,          // @
44    LParen,      // (
45    RParen,      // )
46    Comma,       // ,
47    Semicolon,   // ;
48    Slash,       // /
49    Number(i64), // Numeric literal
50    Str(String), // String literal: "..." (directive arguments, e.g. @pattern("^[a-z]+$"))
51
52    // Whitespace and EOF
53    Newline,
54    Eof,
55}
56
57pub struct Lexer {
58    input: Vec<char>,
59    position: usize,
60    pub line: usize,
61    pub column: usize,
62}
63
64impl Lexer {
65    pub fn new(input: &str) -> Self {
66        Lexer {
67            input: input.chars().collect(),
68            position: 0,
69            line: 1,
70            column: 1,
71        }
72    }
73
74    fn current_char(&self) -> Option<char> {
75        if self.position < self.input.len() {
76            Some(self.input[self.position])
77        } else {
78            None
79        }
80    }
81
82    fn advance(&mut self) {
83        if let Some(ch) = self.current_char() {
84            if ch == '\n' {
85                self.line += 1;
86                self.column = 1;
87            } else {
88                self.column += 1;
89            }
90            self.position += 1;
91        }
92    }
93
94    fn skip_whitespace(&mut self) {
95        while let Some(ch) = self.current_char() {
96            if ch == ' ' || ch == '\t' || ch == '\r' {
97                self.advance();
98            } else {
99                break;
100            }
101        }
102    }
103
104    fn skip_whitespace_with_flag(&mut self) -> bool {
105        let start_pos = self.position;
106        self.skip_whitespace();
107        // Return true if we skipped any whitespace, or if we're at the start of a line
108        self.position > start_pos || self.column == 1
109    }
110
111    fn skip_comment(&mut self) {
112        // Skip until end of line
113        while let Some(ch) = self.current_char() {
114            if ch == '\n' {
115                break;
116            }
117            self.advance();
118        }
119    }
120
121    fn read_identifier(&mut self) -> String {
122        let mut ident = String::new();
123        while let Some(ch) = self.current_char() {
124            if ch.is_alphanumeric() || ch == '_' {
125                ident.push(ch);
126                self.advance();
127            } else {
128                break;
129            }
130        }
131        ident
132    }
133
134    fn read_number(&mut self) -> Result<i64, String> {
135        let mut num_str = String::new();
136        while let Some(ch) = self.current_char() {
137            if ch.is_numeric() {
138                num_str.push(ch);
139                self.advance();
140            } else {
141                break;
142            }
143        }
144        num_str.parse::<i64>().map_err(|e| {
145            format!(
146                "Numeric literal '{}' is out of range at line {}, column {}: {}",
147                num_str, self.line, self.column, e
148            )
149        })
150    }
151
152    /// Read a double-quoted string literal, consuming the opening and closing quotes.
153    /// Supports the escapes `\"`, `\\`, `\n`, `\t`, `\r`. Called with the cursor on the
154    /// opening `"`.
155    fn read_string(&mut self) -> Result<String, String> {
156        let start_line = self.line;
157        let start_column = self.column;
158        self.advance(); // consume opening quote
159
160        let mut value = String::new();
161        loop {
162            match self.current_char() {
163                None => {
164                    return Err(format!(
165                        "Unterminated string literal starting at line {}, column {}",
166                        start_line, start_column
167                    ));
168                }
169                Some('"') => {
170                    self.advance(); // consume closing quote
171                    return Ok(value);
172                }
173                Some('\\') => {
174                    self.advance();
175                    match self.current_char() {
176                        Some('"') => value.push('"'),
177                        Some('\\') => value.push('\\'),
178                        Some('n') => value.push('\n'),
179                        Some('t') => value.push('\t'),
180                        Some('r') => value.push('\r'),
181                        Some(other) => {
182                            return Err(format!(
183                                "Invalid escape sequence '\\{}' in string literal at line {}, column {}",
184                                other, self.line, self.column
185                            ));
186                        }
187                        None => {
188                            return Err(format!(
189                                "Unterminated string literal starting at line {}, column {}",
190                                start_line, start_column
191                            ));
192                        }
193                    }
194                    self.advance();
195                }
196                Some('\n') => {
197                    return Err(format!(
198                        "Unterminated string literal starting at line {}, column {} (newline before closing quote)",
199                        start_line, start_column
200                    ));
201                }
202                Some(ch) => {
203                    value.push(ch);
204                    self.advance();
205                }
206            }
207        }
208    }
209
210    pub fn next_token(&mut self) -> Result<Token, String> {
211        let had_whitespace = self.skip_whitespace_with_flag();
212
213        match self.current_char() {
214            None => Ok(Token::Eof),
215            Some('\n') => {
216                self.advance();
217                Ok(Token::Newline)
218            }
219            Some('/') => {
220                self.advance();
221                // Only treat // as a comment if there was preceding whitespace or we're at line start
222                // This allows tsx://path to work while preserving // comments
223                if self.current_char() == Some('/') && had_whitespace {
224                    self.advance();
225                    self.skip_comment();
226                    // After comment, get next token
227                    self.next_token()
228                } else {
229                    // Single slash token (for component paths like tsx://path)
230                    Ok(Token::Slash)
231                }
232            }
233            Some('+') => {
234                self.advance();
235                Ok(Token::Plus)
236            }
237            Some('&') => {
238                self.advance();
239                Ok(Token::Ampersand)
240            }
241            Some('^') => {
242                self.advance();
243                Ok(Token::Caret)
244            }
245            Some(':') => {
246                self.advance();
247                Ok(Token::Colon)
248            }
249            Some('{') => {
250                self.advance();
251                Ok(Token::LBrace)
252            }
253            Some('}') => {
254                self.advance();
255                Ok(Token::RBrace)
256            }
257            Some('[') => {
258                self.advance();
259                Ok(Token::LBracket)
260            }
261            Some(']') => {
262                self.advance();
263                Ok(Token::RBracket)
264            }
265            Some('*') => {
266                self.advance();
267                Ok(Token::Asterisk)
268            }
269            Some('?') => {
270                self.advance();
271                Ok(Token::Question)
272            }
273            Some('@') => {
274                self.advance();
275                Ok(Token::At)
276            }
277            Some('(') => {
278                self.advance();
279                Ok(Token::LParen)
280            }
281            Some(')') => {
282                self.advance();
283                Ok(Token::RParen)
284            }
285            Some(',') => {
286                self.advance();
287                Ok(Token::Comma)
288            }
289            Some(';') => {
290                self.advance();
291                Ok(Token::Semicolon)
292            }
293            Some('"') => {
294                let s = self.read_string()?;
295                Ok(Token::Str(s))
296            }
297            Some(ch) if ch.is_numeric() => {
298                let num = self.read_number()?;
299                Ok(Token::Number(num))
300            }
301            Some(ch) if ch.is_alphabetic() || ch == '_' => {
302                let ident = self.read_identifier();
303                let token = match ident.as_str() {
304                    "u32" => Token::TypeU32,
305                    "u64" => Token::TypeU64,
306                    "i32" => Token::TypeI32,
307                    "i64" => Token::TypeI64,
308                    "f64" => Token::TypeF64,
309                    "bool" => Token::TypeBool,
310                    "string" => Token::TypeString,
311                    "uuid" => Token::TypeUuid,
312                    "timestamp" => Token::TypeTimestamp,
313                    "char" => Token::TypeChar,
314                    "struct" => Token::KwStruct,
315                    _ => Token::Ident(ident),
316                };
317                Ok(token)
318            }
319            Some(ch) => Err(format!(
320                "Unexpected character '{}' at line {}, column {}",
321                ch, self.line, self.column
322            )),
323        }
324    }
325
326    pub fn next_token_with_pos(&mut self) -> Result<TokenWithPos, String> {
327        let pos = Position {
328            line: self.line,
329            column: self.column,
330        };
331        let token = self.next_token()?;
332        Ok(TokenWithPos {
333            token,
334            position: pos,
335        })
336    }
337
338    pub fn tokenize(&mut self) -> Result<Vec<Token>, String> {
339        let mut tokens = Vec::new();
340        loop {
341            let token = self.next_token()?;
342            if token == Token::Eof {
343                tokens.push(token);
344                break;
345            }
346            tokens.push(token);
347        }
348        Ok(tokens)
349    }
350
351    pub fn tokenize_with_pos(&mut self) -> Result<Vec<TokenWithPos>, String> {
352        let mut tokens = Vec::new();
353        loop {
354            let token_with_pos = self.next_token_with_pos()?;
355            let is_eof = token_with_pos.token == Token::Eof;
356            tokens.push(token_with_pos);
357            if is_eof {
358                break;
359            }
360        }
361        Ok(tokens)
362    }
363}
364
365#[cfg(test)]
366mod tests {
367    use super::*;
368
369    fn tokens(input: &str) -> Vec<Token> {
370        Lexer::new(input).tokenize().expect("lex error")
371    }
372
373    #[test]
374    fn lexes_basic_string_literal() {
375        assert_eq!(
376            tokens("\"pending\""),
377            vec![Token::Str("pending".to_string()), Token::Eof]
378        );
379    }
380
381    #[test]
382    fn lexes_string_with_regex_metacharacters() {
383        // A regex that could never be a bare identifier — the whole point of the feature.
384        assert_eq!(
385            tokens("\"^[a-z]+$\""),
386            vec![Token::Str("^[a-z]+$".to_string()), Token::Eof]
387        );
388    }
389
390    #[test]
391    fn lexes_string_escapes() {
392        assert_eq!(
393            tokens(r#""a\"b\\c\n\t\r""#),
394            vec![Token::Str("a\"b\\c\n\t\r".to_string()), Token::Eof]
395        );
396    }
397
398    #[test]
399    fn empty_string_literal() {
400        assert_eq!(tokens("\"\""), vec![Token::Str(String::new()), Token::Eof]);
401    }
402
403    #[test]
404    fn string_literal_in_directive_context() {
405        // @pattern("^[0-9]+$")
406        assert_eq!(
407            tokens("@pattern(\"^[0-9]+$\")"),
408            vec![
409                Token::At,
410                Token::Ident("pattern".to_string()),
411                Token::LParen,
412                Token::Str("^[0-9]+$".to_string()),
413                Token::RParen,
414                Token::Eof,
415            ]
416        );
417    }
418
419    #[test]
420    fn unterminated_string_is_an_error() {
421        let err = Lexer::new("\"oops").tokenize().unwrap_err();
422        assert!(err.contains("Unterminated string literal"), "got: {err}");
423    }
424
425    #[test]
426    fn newline_before_closing_quote_is_an_error() {
427        let err = Lexer::new("\"oops\n\"").tokenize().unwrap_err();
428        assert!(err.contains("Unterminated string literal"), "got: {err}");
429    }
430
431    #[test]
432    fn invalid_escape_is_an_error() {
433        let err = Lexer::new("\"a\\q\"").tokenize().unwrap_err();
434        assert!(err.contains("Invalid escape sequence"), "got: {err}");
435    }
436
437    #[test]
438    fn double_slash_comment_still_works_after_string_support() {
439        // Guards the tsx://path vs // comment disambiguation is unaffected.
440        assert_eq!(tokens("u32 // trailing comment"), vec![Token::TypeU32, Token::Eof]);
441    }
442}
forgedb_parser/lexer.rs

forgedb_parser/
lexer.rs
