rantlr_core/lexer/
scanner.rs

1use crate::diagnostic::{Diagnostic, DiagnosticKind};
2use crate::span::{BytePos, Span};
3use crate::token::{BuiltinType, Keyword, Token, TokenKind};
4use thiserror::Error;
5
6#[derive(Debug, Error, Clone, PartialEq, Eq)]
7pub enum LexerError {
8    /// Wraps a structured diagnostic; message comes from `Diagnostic::message`
9    /// so we do not rely on `Display` formatting of the whole value.
10    #[error("{}", .0.message)]
11    Diagnostic(Diagnostic),
12}
13
14impl LexerError {
15    pub fn diagnostic(&self) -> &Diagnostic {
16        match self {
17            Self::Diagnostic(d) => d,
18        }
19    }
20}
21
22/// Tokenize a complete `.gr` source string.
23pub fn lex(source: &str) -> Result<Vec<Token>, LexerError> {
24    Lexer::new(source).tokenize_all()
25}
26
27pub struct Lexer<'src> {
28    src: &'src str,
29    bytes: &'src [u8],
30    pos: usize,
31}
32
33impl<'src> Lexer<'src> {
34    pub fn new(src: &'src str) -> Self {
35        Self {
36            src,
37            bytes: src.as_bytes(),
38            pos: 0,
39        }
40    }
41
42    pub fn tokenize_all(mut self) -> Result<Vec<Token>, LexerError> {
43        let mut tokens = Vec::new();
44        loop {
45            let tok = self.next_token()?;
46            let is_eof = tok.kind.is_eof();
47            tokens.push(tok);
48            if is_eof {
49                break;
50            }
51        }
52        Ok(tokens)
53    }
54
55    pub fn next_token(&mut self) -> Result<Token, LexerError> {
56        self.skip_trivia()?;
57
58        let start = self.pos;
59        let Some(ch) = self.peek_char() else {
60            return Ok(Token::new(
61                TokenKind::Eof,
62                Span::from_offsets(start, start),
63            ));
64        };
65
66        match ch {
67            '{' => self.simple(TokenKind::LBrace, 1),
68            '}' => self.simple(TokenKind::RBrace, 1),
69            '(' => self.simple(TokenKind::LParen, 1),
70            ')' => self.simple(TokenKind::RParen, 1),
71            '[' => self.simple(TokenKind::LBracket, 1),
72            ']' => self.simple(TokenKind::RBracket, 1),
73            '|' => self.simple(TokenKind::Pipe, 1),
74            ',' => self.simple(TokenKind::Comma, 1),
75            ';' => self.simple(TokenKind::Semi, 1),
76            ':' => self.simple(TokenKind::Colon, 1),
77            '=' => self.simple(TokenKind::Eq, 1),
78            '-' if self.peek_at(1) == Some('>') => self.simple(TokenKind::Arrow, 2),
79            '.' if self.peek_at(1) == Some('.') => self.simple(TokenKind::DotDot, 2),
80            '"' => self.string_literal(),
81            '`' => self.raw_string(),
82            c if c.is_ascii_digit() => self.integer(),
83            c if is_ident_start(c) => self.ident_or_keyword(),
84            _ => {
85                let end = start + ch.len_utf8();
86                Err(self.err(
87                    start,
88                    end,
89                    format!("unexpected character `{ch}`"),
90                    Some("expected a keyword, identifier, string, or punctuation"),
91                ))
92            }
93        }
94    }
95
96    fn simple(&mut self, kind: TokenKind, len: usize) -> Result<Token, LexerError> {
97        let start = self.pos;
98        self.pos += len;
99        Ok(Token::new(kind, Span::from_offsets(start, self.pos)))
100    }
101
102    fn skip_trivia(&mut self) -> Result<(), LexerError> {
103        loop {
104            self.skip_whitespace();
105            if self.starts_with("//") {
106                self.skip_line_comment();
107                continue;
108            }
109            if self.starts_with("/*") {
110                self.skip_block_comment()?;
111                continue;
112            }
113            break;
114        }
115        Ok(())
116    }
117
118    fn skip_whitespace(&mut self) {
119        while let Some(ch) = self.peek_char() {
120            if ch.is_whitespace() {
121                self.bump();
122            } else {
123                break;
124            }
125        }
126    }
127
128    fn skip_line_comment(&mut self) {
129        debug_assert!(self.starts_with("//"));
130        self.pos += 2;
131        while let Some(ch) = self.peek_char() {
132            if ch == '\n' {
133                break;
134            }
135            self.bump();
136        }
137    }
138
139    fn skip_block_comment(&mut self) -> Result<(), LexerError> {
140        let start = self.pos;
141        debug_assert!(self.starts_with("/*"));
142        self.pos += 2;
143        while self.pos < self.bytes.len() {
144            if self.starts_with("*/") {
145                self.pos += 2;
146                return Ok(());
147            }
148            self.bump();
149        }
150        Err(self.err(
151            start,
152            self.pos,
153            "unterminated block comment",
154            Some("close the comment with `*/`"),
155        ))
156    }
157
158    fn string_literal(&mut self) -> Result<Token, LexerError> {
159        let start = self.pos;
160        self.bump(); // opening "
161        let mut value = String::new();
162
163        while let Some(ch) = self.peek_char() {
164            match ch {
165                '"' => {
166                    self.bump();
167                    return Ok(Token::new(
168                        TokenKind::String(value),
169                        Span::from_offsets(start, self.pos),
170                    ));
171                }
172                '\\' => {
173                    self.bump();
174                    let Some(esc) = self.peek_char() else {
175                        return Err(self.err(
176                            start,
177                            self.pos,
178                            "unterminated string literal",
179                            Some("close the string with `\"`"),
180                        ));
181                    };
182                    let decoded = match esc {
183                        'n' => '\n',
184                        'r' => '\r',
185                        't' => '\t',
186                        '\\' => '\\',
187                        '"' => '"',
188                        '0' => '\0',
189                        other => {
190                            let esc_start = self.pos;
191                            self.bump();
192                            return Err(self.err(
193                                esc_start,
194                                self.pos,
195                                format!("unknown escape sequence `\\{other}`"),
196                                Some("supported escapes: \\\\ \\\" \\n \\r \\t \\0"),
197                            ));
198                        }
199                    };
200                    self.bump();
201                    value.push(decoded);
202                }
203                '\n' | '\r' => {
204                    return Err(self.err(
205                        start,
206                        self.pos,
207                        "unterminated string literal",
208                        Some("close the string with `\"` before the newline"),
209                    ));
210                }
211                _ => {
212                    value.push(ch);
213                    self.bump();
214                }
215            }
216        }
217
218        Err(self.err(
219            start,
220            self.pos,
221            "unterminated string literal",
222            Some("close the string with `\"`"),
223        ))
224    }
225
226    fn raw_string(&mut self) -> Result<Token, LexerError> {
227        let start = self.pos;
228        self.bump(); // opening `
229        let content_start = self.pos;
230
231        while let Some(ch) = self.peek_char() {
232            if ch == '`' {
233                let value = self.src[content_start..self.pos].to_string();
234                self.bump();
235                return Ok(Token::new(
236                    TokenKind::RawString(value),
237                    Span::from_offsets(start, self.pos),
238                ));
239            }
240            self.bump();
241        }
242
243        Err(self.err(
244            start,
245            self.pos,
246            "unterminated raw string",
247            Some("close the raw string with a backtick (`)"),
248        ))
249    }
250
251    fn integer(&mut self) -> Result<Token, LexerError> {
252        let start = self.pos;
253        while matches!(self.peek_char(), Some(c) if c.is_ascii_digit()) {
254            self.bump();
255        }
256        let lexeme = &self.src[start..self.pos];
257        let value: u64 = lexeme.parse().map_err(|_| {
258            self.err(
259                start,
260                self.pos,
261                format!("integer literal `{lexeme}` is out of range"),
262                None,
263            )
264        })?;
265        Ok(Token::new(
266            TokenKind::Integer(value),
267            Span::from_offsets(start, self.pos),
268        ))
269    }
270
271    fn ident_or_keyword(&mut self) -> Result<Token, LexerError> {
272        let start = self.pos;
273        self.bump();
274        while matches!(self.peek_char(), Some(c) if is_ident_continue(c)) {
275            self.bump();
276        }
277        let lexeme = &self.src[start..self.pos];
278        let span = Span::from_offsets(start, self.pos);
279
280        let kind = if let Some(kw) = Keyword::from_ident(lexeme) {
281            TokenKind::Keyword(kw)
282        } else if let Some(builtin) = BuiltinType::from_ident(lexeme) {
283            TokenKind::Builtin(builtin)
284        } else {
285            TokenKind::Ident(lexeme.to_string())
286        };
287
288        Ok(Token::new(kind, span))
289    }
290
291    fn err(
292        &self,
293        start: usize,
294        end: usize,
295        message: impl Into<String>,
296        help: Option<&str>,
297    ) -> LexerError {
298        let mut diag = Diagnostic::error(
299            DiagnosticKind::LexError,
300            message,
301            Span::new(BytePos(start as u32), BytePos(end as u32)),
302        );
303        if let Some(help) = help {
304            diag = diag.with_help(help);
305        }
306        LexerError::Diagnostic(diag)
307    }
308
309    fn peek_char(&self) -> Option<char> {
310        self.src[self.pos..].chars().next()
311    }
312
313    fn peek_at(&self, offset: usize) -> Option<char> {
314        let idx = self.pos + offset;
315        if idx >= self.bytes.len() {
316            return None;
317        }
318        self.src[idx..].chars().next()
319    }
320
321    fn bump(&mut self) -> Option<char> {
322        let ch = self.peek_char()?;
323        self.pos += ch.len_utf8();
324        Some(ch)
325    }
326
327    fn starts_with(&self, s: &str) -> bool {
328        self.src[self.pos..].starts_with(s)
329    }
330}
331
332fn is_ident_start(c: char) -> bool {
333    c.is_ascii_alphabetic() || c == '_'
334}
335
336fn is_ident_continue(c: char) -> bool {
337    c.is_ascii_alphanumeric() || c == '_'
338}
339
340#[cfg(test)]
341mod tests {
342    use super::*;
343    use crate::token::{BuiltinType, Keyword, TokenKind};
344
345    fn kinds(src: &str) -> Vec<TokenKind> {
346        lex(src)
347            .expect("lex ok")
348            .into_iter()
349            .map(|t| t.kind)
350            .collect()
351    }
352
353    #[test]
354    fn lexes_grammar_header_and_rule() {
355        let src = r#"
356            grammar Calculator;
357
358            token Num = Number;
359
360            rule expr {
361                term
362                repeat {
363                    match "+" | "-"
364                    term
365                }
366            }
367        "#;
368
369        let toks = kinds(src);
370        assert!(toks.starts_with(&[
371            TokenKind::Keyword(Keyword::Grammar),
372            TokenKind::Ident("Calculator".into()),
373            TokenKind::Semi,
374            TokenKind::Keyword(Keyword::Token),
375            TokenKind::Ident("Num".into()),
376            TokenKind::Eq,
377            TokenKind::Builtin(BuiltinType::Number),
378            TokenKind::Semi,
379            TokenKind::Keyword(Keyword::Rule),
380            TokenKind::Ident("expr".into()),
381            TokenKind::LBrace,
382        ]));
383        assert_eq!(toks.last(), Some(&TokenKind::Eof));
384        assert!(toks.contains(&TokenKind::Keyword(Keyword::Repeat)));
385        assert!(toks.contains(&TokenKind::Keyword(Keyword::Match)));
386        assert!(toks.contains(&TokenKind::String("+".into())));
387        assert!(toks.contains(&TokenKind::Pipe));
388    }
389
390    #[test]
391    fn lexes_optional_repeat_bounds_and_arrow_skip() {
392        let src = r#"
393            token Ws = " " -> skip;
394            rule items {
395                optional { item }
396                repeat(0..) { "," item }
397            }
398        "#;
399        let toks = kinds(src);
400        assert!(toks.contains(&TokenKind::Arrow));
401        assert!(toks.contains(&TokenKind::Keyword(Keyword::Skip)));
402        assert!(toks.contains(&TokenKind::Keyword(Keyword::Optional)));
403        assert!(toks.contains(&TokenKind::Integer(0)));
404        assert!(toks.contains(&TokenKind::DotDot));
405    }
406
407    #[test]
408    fn lexes_example_block_with_raw_string() {
409        let src = r#"
410            example "object" {
411                input: `{"a": 1}`
412                expect: object
413            }
414        "#;
415        let toks = kinds(src);
416        assert!(toks.contains(&TokenKind::Keyword(Keyword::Example)));
417        assert!(toks.contains(&TokenKind::String("object".into())));
418        assert!(toks.contains(&TokenKind::Keyword(Keyword::Input)));
419        assert!(toks.contains(&TokenKind::RawString(r#"{"a": 1}"#.into())));
420        assert!(toks.contains(&TokenKind::Keyword(Keyword::Expect)));
421    }
422
423    #[test]
424    fn skips_line_and_block_comments() {
425        let src = r#"
426            // line comment
427            grammar /* block */ Mini;
428        "#;
429        assert_eq!(
430            kinds(src),
431            vec![
432                TokenKind::Keyword(Keyword::Grammar),
433                TokenKind::Ident("Mini".into()),
434                TokenKind::Semi,
435                TokenKind::Eof,
436            ]
437        );
438    }
439
440    #[test]
441    fn string_escapes() {
442        let toks = kinds(r#""a\nb\t\"c""#);
443        assert_eq!(
444            toks,
445            vec![TokenKind::String("a\nb\t\"c".into()), TokenKind::Eof]
446        );
447    }
448
449    #[test]
450    fn unterminated_string_is_error() {
451        let err = lex(r#"rule x { "oops"#).unwrap_err();
452        assert!(err.diagnostic().message.contains("unterminated"));
453    }
454
455    #[test]
456    fn unknown_character_is_error() {
457        let err = lex("grammar @").unwrap_err();
458        assert!(err.diagnostic().message.contains("unexpected character"));
459    }
460
461    #[test]
462    fn tracks_spans() {
463        let tokens = lex("rule").unwrap();
464        assert_eq!(tokens[0].span, Span::from_offsets(0, 4));
465    }
466}
rantlr_core/lexer/scanner.rs

rantlr_core/lexer/
scanner.rs
