forgedb_parser/
lexer.rs

1// Re-export Position from validation crate for consistency
2pub use forgedb_validation::Position;
3
4/// Token with position information
5#[derive(Debug, Clone, PartialEq)]
6pub struct TokenWithPos {
7    pub token: Token,
8    pub position: Position,
9}
10
11/// Token types for the schema language
12#[derive(Debug, Clone, PartialEq)]
13pub enum Token {
14    // Identifiers and literals
15    Ident(String),
16
17    // Types
18    TypeU32,
19    TypeU64,
20    TypeI32,
21    TypeI64,
22    TypeF64,
23    TypeBool,
24    TypeString,
25    TypeUuid,
26    TypeTimestamp,
27    TypeJson,    // json - variable-length column typed serde_json::Value
28    TypeDecimal, // decimal - fixed 16-byte column typed rust_decimal::Decimal
29    TypeChar,    // char(N) - fixed-size character array
30
31    // Keywords
32    KwStruct, // struct
33    KwEnum,   // enum
34
35    // Symbols
36    Plus,        // +
37    Ampersand,   // &
38    Caret,       // ^
39    Colon,       // :
40    LBrace,      // {
41    RBrace,      // }
42    LBracket,    // [
43    RBracket,    // ]
44    Asterisk,    // *
45    Question,    // ?
46    At,          // @
47    LParen,      // (
48    RParen,      // )
49    Comma,       // ,
50    Semicolon,   // ;
51    Slash,       // /
52    Number(i64), // Numeric literal
53    Str(String), // String literal: "..." (directive arguments, e.g. @pattern("^[a-z]+$"))
54
55    // Whitespace and EOF
56    Newline,
57    Eof,
58}
59
60pub struct Lexer {
61    input: Vec<char>,
62    position: usize,
63    pub line: usize,
64    pub column: usize,
65    /// Start position of the most recently produced token, recorded *after*
66    /// leading whitespace/comments are skipped so positions point at the token
67    /// itself (not the preceding indentation).
68    token_start: Position,
69}
70
71impl Lexer {
72    pub fn new(input: &str) -> Self {
73        Lexer {
74            input: input.chars().collect(),
75            position: 0,
76            line: 1,
77            column: 1,
78            token_start: Position { line: 1, column: 1 },
79        }
80    }
81
82    fn current_char(&self) -> Option<char> {
83        if self.position < self.input.len() {
84            Some(self.input[self.position])
85        } else {
86            None
87        }
88    }
89
90    fn advance(&mut self) {
91        if let Some(ch) = self.current_char() {
92            if ch == '\n' {
93                self.line += 1;
94                self.column = 1;
95            } else {
96                self.column += 1;
97            }
98            self.position += 1;
99        }
100    }
101
102    fn skip_whitespace(&mut self) {
103        while let Some(ch) = self.current_char() {
104            if ch == ' ' || ch == '\t' || ch == '\r' {
105                self.advance();
106            } else {
107                break;
108            }
109        }
110    }
111
112    fn skip_whitespace_with_flag(&mut self) -> bool {
113        let start_pos = self.position;
114        self.skip_whitespace();
115        // Return true if we skipped any whitespace, or if we're at the start of a line
116        self.position > start_pos || self.column == 1
117    }
118
119    fn skip_comment(&mut self) {
120        // Skip until end of line
121        while let Some(ch) = self.current_char() {
122            if ch == '\n' {
123                break;
124            }
125            self.advance();
126        }
127    }
128
129    fn read_identifier(&mut self) -> String {
130        let mut ident = String::new();
131        while let Some(ch) = self.current_char() {
132            if ch.is_alphanumeric() || ch == '_' {
133                ident.push(ch);
134                self.advance();
135            } else {
136                break;
137            }
138        }
139        ident
140    }
141
142    fn read_number(&mut self) -> Result<i64, String> {
143        let mut num_str = String::new();
144        while let Some(ch) = self.current_char() {
145            if ch.is_numeric() {
146                num_str.push(ch);
147                self.advance();
148            } else {
149                break;
150            }
151        }
152        num_str.parse::<i64>().map_err(|e| {
153            format!(
154                "Numeric literal '{}' is out of range at line {}, column {}: {}",
155                num_str, self.line, self.column, e
156            )
157        })
158    }
159
160    /// Read a double-quoted string literal, consuming the opening and closing quotes.
161    /// Supports the escapes `\"`, `\\`, `\n`, `\t`, `\r`. Called with the cursor on the
162    /// opening `"`.
163    fn read_string(&mut self) -> Result<String, String> {
164        let start_line = self.line;
165        let start_column = self.column;
166        self.advance(); // consume opening quote
167
168        let mut value = String::new();
169        loop {
170            match self.current_char() {
171                None => {
172                    return Err(format!(
173                        "Unterminated string literal starting at line {}, column {}",
174                        start_line, start_column
175                    ));
176                }
177                Some('"') => {
178                    self.advance(); // consume closing quote
179                    return Ok(value);
180                }
181                Some('\\') => {
182                    self.advance();
183                    match self.current_char() {
184                        Some('"') => value.push('"'),
185                        Some('\\') => value.push('\\'),
186                        Some('n') => value.push('\n'),
187                        Some('t') => value.push('\t'),
188                        Some('r') => value.push('\r'),
189                        Some(other) => {
190                            return Err(format!(
191                                "Invalid escape sequence '\\{}' in string literal at line {}, column {}",
192                                other, self.line, self.column
193                            ));
194                        }
195                        None => {
196                            return Err(format!(
197                                "Unterminated string literal starting at line {}, column {}",
198                                start_line, start_column
199                            ));
200                        }
201                    }
202                    self.advance();
203                }
204                Some('\n') => {
205                    return Err(format!(
206                        "Unterminated string literal starting at line {}, column {} (newline before closing quote)",
207                        start_line, start_column
208                    ));
209                }
210                Some(ch) => {
211                    value.push(ch);
212                    self.advance();
213                }
214            }
215        }
216    }
217
218    pub fn next_token(&mut self) -> Result<Token, String> {
219        let had_whitespace = self.skip_whitespace_with_flag();
220        // Record the token's true start (after skipping indentation/whitespace);
221        // comment recursion re-enters here and overwrites this with the real token.
222        self.token_start = Position {
223            line: self.line,
224            column: self.column,
225        };
226
227        match self.current_char() {
228            None => Ok(Token::Eof),
229            Some('\n') => {
230                self.advance();
231                Ok(Token::Newline)
232            }
233            Some('/') => {
234                self.advance();
235                // Only treat // as a comment if there was preceding whitespace or we're at line start
236                // This allows tsx://path to work while preserving // comments
237                if self.current_char() == Some('/') && had_whitespace {
238                    self.advance();
239                    self.skip_comment();
240                    // After comment, get next token
241                    self.next_token()
242                } else {
243                    // Single slash token (for component paths like tsx://path)
244                    Ok(Token::Slash)
245                }
246            }
247            Some('+') => {
248                self.advance();
249                Ok(Token::Plus)
250            }
251            Some('&') => {
252                self.advance();
253                Ok(Token::Ampersand)
254            }
255            Some('^') => {
256                self.advance();
257                Ok(Token::Caret)
258            }
259            Some(':') => {
260                self.advance();
261                Ok(Token::Colon)
262            }
263            Some('{') => {
264                self.advance();
265                Ok(Token::LBrace)
266            }
267            Some('}') => {
268                self.advance();
269                Ok(Token::RBrace)
270            }
271            Some('[') => {
272                self.advance();
273                Ok(Token::LBracket)
274            }
275            Some(']') => {
276                self.advance();
277                Ok(Token::RBracket)
278            }
279            Some('*') => {
280                self.advance();
281                Ok(Token::Asterisk)
282            }
283            Some('?') => {
284                self.advance();
285                Ok(Token::Question)
286            }
287            Some('@') => {
288                self.advance();
289                Ok(Token::At)
290            }
291            Some('(') => {
292                self.advance();
293                Ok(Token::LParen)
294            }
295            Some(')') => {
296                self.advance();
297                Ok(Token::RParen)
298            }
299            Some(',') => {
300                self.advance();
301                Ok(Token::Comma)
302            }
303            Some(';') => {
304                self.advance();
305                Ok(Token::Semicolon)
306            }
307            Some('"') => {
308                let s = self.read_string()?;
309                Ok(Token::Str(s))
310            }
311            Some(ch) if ch.is_numeric() => {
312                let num = self.read_number()?;
313                Ok(Token::Number(num))
314            }
315            Some(ch) if ch.is_alphabetic() || ch == '_' => {
316                let ident = self.read_identifier();
317                let token = match ident.as_str() {
318                    "u32" => Token::TypeU32,
319                    "u64" => Token::TypeU64,
320                    "i32" => Token::TypeI32,
321                    "i64" => Token::TypeI64,
322                    "f64" => Token::TypeF64,
323                    "bool" => Token::TypeBool,
324                    "string" => Token::TypeString,
325                    "uuid" => Token::TypeUuid,
326                    "timestamp" => Token::TypeTimestamp,
327                    "json" => Token::TypeJson,
328                    "decimal" => Token::TypeDecimal,
329                    "char" => Token::TypeChar,
330                    "struct" => Token::KwStruct,
331                    "enum" => Token::KwEnum,
332                    _ => Token::Ident(ident),
333                };
334                Ok(token)
335            }
336            Some(ch) => Err(format!(
337                "Unexpected character '{}' at line {}, column {}",
338                ch, self.line, self.column
339            )),
340        }
341    }
342
343    pub fn next_token_with_pos(&mut self) -> Result<TokenWithPos, String> {
344        let token = self.next_token()?;
345        Ok(TokenWithPos {
346            token,
347            position: self.token_start,
348        })
349    }
350
351    pub fn tokenize(&mut self) -> Result<Vec<Token>, String> {
352        let mut tokens = Vec::new();
353        loop {
354            let token = self.next_token()?;
355            if token == Token::Eof {
356                tokens.push(token);
357                break;
358            }
359            tokens.push(token);
360        }
361        Ok(tokens)
362    }
363
364    pub fn tokenize_with_pos(&mut self) -> Result<Vec<TokenWithPos>, String> {
365        let mut tokens = Vec::new();
366        loop {
367            let token_with_pos = self.next_token_with_pos()?;
368            let is_eof = token_with_pos.token == Token::Eof;
369            tokens.push(token_with_pos);
370            if is_eof {
371                break;
372            }
373        }
374        Ok(tokens)
375    }
376}
377
378#[cfg(test)]
379mod tests {
380    use super::*;
381
382    fn tokens(input: &str) -> Vec<Token> {
383        Lexer::new(input).tokenize().expect("lex error")
384    }
385
386    #[test]
387    fn lexes_basic_string_literal() {
388        assert_eq!(
389            tokens("\"pending\""),
390            vec![Token::Str("pending".to_string()), Token::Eof]
391        );
392    }
393
394    #[test]
395    fn lexes_string_with_regex_metacharacters() {
396        // A regex that could never be a bare identifier — the whole point of the feature.
397        assert_eq!(
398            tokens("\"^[a-z]+$\""),
399            vec![Token::Str("^[a-z]+$".to_string()), Token::Eof]
400        );
401    }
402
403    #[test]
404    fn lexes_string_escapes() {
405        assert_eq!(
406            tokens(r#""a\"b\\c\n\t\r""#),
407            vec![Token::Str("a\"b\\c\n\t\r".to_string()), Token::Eof]
408        );
409    }
410
411    #[test]
412    fn empty_string_literal() {
413        assert_eq!(tokens("\"\""), vec![Token::Str(String::new()), Token::Eof]);
414    }
415
416    #[test]
417    fn string_literal_in_directive_context() {
418        // @pattern("^[0-9]+$")
419        assert_eq!(
420            tokens("@pattern(\"^[0-9]+$\")"),
421            vec![
422                Token::At,
423                Token::Ident("pattern".to_string()),
424                Token::LParen,
425                Token::Str("^[0-9]+$".to_string()),
426                Token::RParen,
427                Token::Eof,
428            ]
429        );
430    }
431
432    #[test]
433    fn unterminated_string_is_an_error() {
434        let err = Lexer::new("\"oops").tokenize().unwrap_err();
435        assert!(err.contains("Unterminated string literal"), "got: {err}");
436    }
437
438    #[test]
439    fn newline_before_closing_quote_is_an_error() {
440        let err = Lexer::new("\"oops\n\"").tokenize().unwrap_err();
441        assert!(err.contains("Unterminated string literal"), "got: {err}");
442    }
443
444    #[test]
445    fn invalid_escape_is_an_error() {
446        let err = Lexer::new("\"a\\q\"").tokenize().unwrap_err();
447        assert!(err.contains("Invalid escape sequence"), "got: {err}");
448    }
449
450    #[test]
451    fn double_slash_comment_still_works_after_string_support() {
452        // Guards the tsx://path vs // comment disambiguation is unaffected.
453        assert_eq!(tokens("u32 // trailing comment"), vec![Token::TypeU32, Token::Eof]);
454    }
455}
forgedb_parser/lexer.rs

forgedb_parser/
lexer.rs
