opy_rs/
lexer.rs

1//! The native `.opy` lexer.
2//!
3//! Produces a flat token stream (newlines and indentation included) from one
4//! source file. Comments (`#`, `/* */`) are skipped; `#!` directives are
5//! captured as a single directive token for the preprocessor. Positions are
6//! 1-based line/column, matching the Opy HIR protocol.
7
8use crate::diag::{OpyError, OpyResult, Position, Span};
9
10/// The kind of a token.
11#[derive(Debug, Clone, Copy, PartialEq, Eq)]
12pub enum TokenKind {
13    /// An identifier or keyword (keywords are resolved by the parser).
14    Ident,
15    /// A numeric literal (`text` holds the source spelling).
16    Number,
17    /// A string literal (`text` holds the unescaped value).
18    String,
19    /// A `#!` directive line (`text` holds everything after `#!`).
20    Directive,
21    /// A preprocessing marker carrying the rule-prefix state active at the
22    /// following top-level rule or subroutine.
23    RulePrefixMarker,
24    /// `@Event` / `@Condition` / other `@` directives.
25    At,
26    Newline,
27    /// Indentation change: the column of the current line.
28    Indent(u32),
29    /// End of file.
30    Eof,
31    // Punctuation and operators.
32    LParen,
33    RParen,
34    LBracket,
35    RBracket,
36    LBrace,
37    RBrace,
38    Comma,
39    Colon,
40    Semicolon,
41    Dot,
42    Assign,
43    Plus,
44    Minus,
45    Star,
46    Slash,
47    Percent,
48    DoubleStar,
49    PlusAssign,
50    MinusAssign,
51    Increment,
52    Decrement,
53    StarAssign,
54    SlashAssign,
55    PercentAssign,
56    DoubleStarAssign,
57    Eq,
58    Ne,
59    Lt,
60    Le,
61    Gt,
62    Ge,
63    /// A bare `!` that is not `!=`; the parser rejects it as unsupported OPY.
64    LexBang,
65}
66
67/// One token with its source span and payload text.
68#[derive(Debug, Clone, PartialEq)]
69pub struct Token {
70    pub kind: TokenKind,
71    /// The source text of this token (numbers keep their spelling; strings
72    /// keep their unescaped value; identifiers keep their name).
73    pub text: String,
74    /// The exact characters between string quotes, before escape decoding.
75    /// Other token kinds leave this unset. It is retained so source-language
76    /// constructs such as f-string interpolations can recover expression
77    /// spans without losing provenance during preprocessing.
78    pub raw: Option<String>,
79    pub span: Span,
80}
81
82impl Token {
83    fn new(kind: TokenKind, text: impl Into<String>, span: Span) -> Token {
84        Token {
85            kind,
86            text: text.into(),
87            raw: None,
88            span,
89        }
90    }
91}
92
93/// The lexer input: one file's text with its file id.
94pub struct LexInput<'a> {
95    pub file_id: u32,
96    pub text: &'a str,
97}
98
99/// Lex one source file into a token stream.
100pub fn lex(input: LexInput<'_>) -> OpyResult<Vec<Token>> {
101    Lexer::new(input.file_id, input.text).run()
102}
103
104struct Lexer {
105    file_id: u32,
106    chars: Vec<char>,
107    pos: usize,
108    line: u32,
109    col: u32,
110    tokens: Vec<Token>,
111}
112
113impl Lexer {
114    fn new(file_id: u32, text: &str) -> Lexer {
115        Lexer {
116            file_id,
117            chars: text.chars().collect(),
118            pos: 0,
119            line: 1,
120            col: 1,
121            tokens: Vec::new(),
122        }
123    }
124
125    fn run(mut self) -> OpyResult<Vec<Token>> {
126        while self.pos < self.chars.len() {
127            let ch = self.chars[self.pos];
128            match ch {
129                '\n' => {
130                    self.tokens
131                        .push(Token::new(TokenKind::Newline, "\n", self.here(1)));
132                    self.advance();
133                    self.line += 1;
134                    self.col = 1;
135                }
136                ' ' | '\r' => {
137                    self.advance();
138                }
139                '\t' => {
140                    self.pos += 1;
141                    self.col += 4;
142                }
143                '\\' => {
144                    if !self.skip_line_continuation() {
145                        return Err(OpyError::at(
146                            "lex-error",
147                            "unexpected character '\\'",
148                            self.here(1),
149                        ));
150                    }
151                }
152                '#' => self.lex_hash()?,
153                '/' if self.peek(1) == Some('*') => self.skip_block_comment()?,
154                '"' | '\'' => self.lex_string(ch)?,
155                c if c.is_ascii_digit() => self.lex_number()?,
156                c if is_ident_start(c) => self.lex_ident(),
157                '(' => self.single(TokenKind::LParen),
158                ')' => self.single(TokenKind::RParen),
159                '[' => self.single(TokenKind::LBracket),
160                ']' => self.single(TokenKind::RBracket),
161                '{' => self.single(TokenKind::LBrace),
162                '}' => self.single(TokenKind::RBrace),
163                ',' => self.single(TokenKind::Comma),
164                ':' => self.single(TokenKind::Colon),
165                ';' => self.single(TokenKind::Semicolon),
166                '.' => self.single(TokenKind::Dot),
167                '@' => self.single(TokenKind::At),
168                '=' => self.two(TokenKind::Assign, TokenKind::Eq, '='),
169                '+' => {
170                    if self.peek(1) == Some('+') {
171                        self.lex_duplicate(TokenKind::Increment, "++");
172                    } else {
173                        self.lex_two(TokenKind::Plus, TokenKind::PlusAssign, '=');
174                    }
175                }
176                '-' => {
177                    if self.peek(1) == Some('-') {
178                        self.lex_duplicate(TokenKind::Decrement, "--");
179                    } else {
180                        self.lex_two(TokenKind::Minus, TokenKind::MinusAssign, '=');
181                    }
182                }
183                '*' => {
184                    if self.peek(1) == Some('*') {
185                        if self.peek(2) == Some('=') {
186                            let start = self.here(3);
187                            self.advance();
188                            self.advance();
189                            self.advance();
190                            let end = self.here(0);
191                            self.tokens.push(Token::new(
192                                TokenKind::DoubleStarAssign,
193                                "**=",
194                                Span::new(self.file_id, start.start, end.start),
195                            ));
196                        } else {
197                            self.advance();
198                            self.single(TokenKind::DoubleStar)
199                        }
200                    } else {
201                        self.lex_two(TokenKind::Star, TokenKind::StarAssign, '=')
202                    }
203                }
204                '/' => self.lex_two(TokenKind::Slash, TokenKind::SlashAssign, '='),
205                '%' => self.lex_two(TokenKind::Percent, TokenKind::PercentAssign, '='),
206                '<' => self.two(TokenKind::Lt, TokenKind::Le, '='),
207                '>' => self.two(TokenKind::Gt, TokenKind::Ge, '='),
208                '!' => self.two(TokenKind::LexBang, TokenKind::Ne, '='),
209                other => {
210                    return Err(OpyError::at(
211                        "lex-error",
212                        format!("unexpected character '{other}'"),
213                        self.here(1),
214                    ));
215                }
216            }
217        }
218        let here = self.here(0);
219        self.tokens.push(Token::new(TokenKind::Eof, "", here));
220        Ok(self.tokens)
221    }
222
223    /// `#` starts a `#!` directive (captured as one token) or a comment.
224    fn lex_hash(&mut self) -> OpyResult<()> {
225        if self.peek(1) == Some('!') {
226            let start = self.here(2);
227            self.advance();
228            self.advance();
229            let mut text = String::new();
230            while self.pos < self.chars.len() {
231                if self.chars[self.pos] == '\\' && self.skip_line_continuation() {
232                    text.push('\n');
233                    continue;
234                }
235                if self.chars[self.pos] == '\n' {
236                    break;
237                }
238                text.push(self.chars[self.pos]);
239                self.advance();
240            }
241            let end = self.here(0);
242            self.tokens.push(Token::new(
243                TokenKind::Directive,
244                text,
245                Span::new(self.file_id, start.start, end.start),
246            ));
247        } else {
248            while self.pos < self.chars.len() && self.chars[self.pos] != '\n' {
249                self.advance();
250            }
251        }
252        Ok(())
253    }
254
255    fn skip_block_comment(&mut self) -> OpyResult<()> {
256        let start = self.here(2);
257        self.advance();
258        self.advance();
259        while self.pos < self.chars.len() {
260            if self.chars[self.pos] == '*' && self.peek(1) == Some('/') {
261                self.advance();
262                self.advance();
263                return Ok(());
264            }
265            if self.chars[self.pos] == '\n' {
266                self.advance();
267                self.line += 1;
268                self.col = 1;
269            } else {
270                self.advance();
271            }
272        }
273        Err(OpyError::at(
274            "lex-error",
275            "unterminated block comment",
276            start,
277        ))
278    }
279
280    fn lex_string(&mut self, quote: char) -> OpyResult<()> {
281        let start = self.here(1);
282        self.advance();
283        let mut value = String::new();
284        let mut raw = String::new();
285        while self.pos < self.chars.len() {
286            let ch = self.chars[self.pos];
287            if ch == quote {
288                self.advance();
289                let end = self.here(0);
290                let mut token = Token::new(
291                    TokenKind::String,
292                    value,
293                    Span::new(self.file_id, start.start, end.start),
294                );
295                token.raw = Some(raw);
296                self.tokens.push(token);
297                return Ok(());
298            }
299            if ch == '\\' {
300                raw.push(ch);
301                self.advance();
302                if self.pos >= self.chars.len() {
303                    break;
304                }
305                let escaped = self.chars[self.pos];
306                raw.push(escaped);
307                value.push(match escaped {
308                    'n' => '\n',
309                    't' => '\t',
310                    'r' => '\r',
311                    '\\' => '\\',
312                    '"' => '"',
313                    '\'' => '\'',
314                    other => other,
315                });
316                self.advance();
317                continue;
318            }
319            if ch == '\n' {
320                return Err(OpyError::at(
321                    "lex-error",
322                    "unterminated string literal",
323                    start,
324                ));
325            }
326            raw.push(ch);
327            value.push(ch);
328            self.advance();
329        }
330        Err(OpyError::at(
331            "lex-error",
332            "unterminated string literal",
333            start,
334        ))
335    }
336
337    fn skip_line_continuation(&mut self) -> bool {
338        let mut offset = 1;
339        while matches!(self.peek(offset), Some(' ' | '\r')) {
340            offset += 1;
341        }
342        if self.peek(offset) != Some('\n') {
343            return false;
344        }
345        for _ in 0..=offset {
346            self.advance();
347        }
348        self.line += 1;
349        self.col = 1;
350        true
351    }
352
353    fn lex_number(&mut self) -> OpyResult<()> {
354        let start = self.here(1);
355        let mut text = String::new();
356        if self.chars[self.pos] == '0' && matches!(self.peek(1), Some('x' | 'X')) {
357            text.push('0');
358            self.advance();
359            text.push(self.chars[self.pos]);
360            self.advance();
361            let digits_start = self.pos;
362            while self.pos < self.chars.len() && self.chars[self.pos].is_ascii_hexdigit() {
363                text.push(self.chars[self.pos]);
364                self.advance();
365            }
366            if self.pos == digits_start {
367                return Err(OpyError::at(
368                    "lex-error",
369                    "hexadecimal literal requires at least one hexadecimal digit",
370                    Span::new(self.file_id, start.start, self.here(0).start),
371                ));
372            }
373            let end = self.here(0);
374            self.tokens.push(Token::new(
375                TokenKind::Number,
376                text,
377                Span::new(self.file_id, start.start, end.start),
378            ));
379            return Ok(());
380        }
381        while self.pos < self.chars.len() && self.chars[self.pos].is_ascii_digit() {
382            text.push(self.chars[self.pos]);
383            self.advance();
384        }
385        if self.pos < self.chars.len()
386            && self.chars[self.pos] == '.'
387            && self.peek(1).is_some_and(|c| c.is_ascii_digit())
388        {
389            text.push('.');
390            self.advance();
391            while self.pos < self.chars.len() && self.chars[self.pos].is_ascii_digit() {
392                text.push(self.chars[self.pos]);
393                self.advance();
394            }
395        }
396        // Optional exponent (not exercised by the corpus, supported for
397        // completeness of the number surface).
398        if self.pos < self.chars.len()
399            && (self.chars[self.pos] == 'e' || self.chars[self.pos] == 'E')
400        {
401            let mut lookahead = self.pos + 1;
402            if lookahead < self.chars.len()
403                && (self.chars[lookahead] == '+' || self.chars[lookahead] == '-')
404            {
405                lookahead += 1;
406            }
407            if lookahead < self.chars.len() && self.chars[lookahead].is_ascii_digit() {
408                text.push('e');
409                self.advance();
410                if self.pos < self.chars.len()
411                    && (self.chars[self.pos] == '+' || self.chars[self.pos] == '-')
412                {
413                    text.push(self.chars[self.pos]);
414                    self.advance();
415                }
416                while self.pos < self.chars.len() && self.chars[self.pos].is_ascii_digit() {
417                    text.push(self.chars[self.pos]);
418                    self.advance();
419                }
420            }
421        }
422        let end = self.here(0);
423        self.tokens.push(Token::new(
424            TokenKind::Number,
425            text,
426            Span::new(self.file_id, start.start, end.start),
427        ));
428        Ok(())
429    }
430
431    fn lex_ident(&mut self) {
432        let start = self.here(1);
433        let mut text = String::new();
434        while self.pos < self.chars.len() && is_ident_continue(self.chars[self.pos]) {
435            text.push(self.chars[self.pos]);
436            self.advance();
437        }
438        let end = self.here(0);
439        self.tokens.push(Token::new(
440            TokenKind::Ident,
441            text,
442            Span::new(self.file_id, start.start, end.start),
443        ));
444    }
445
446    fn single(&mut self, kind: TokenKind) {
447        let start = self.here(1);
448        let text = self.chars[self.pos].to_string();
449        self.advance();
450        let end = self.here(0);
451        self.tokens.push(Token::new(
452            kind,
453            text,
454            Span::new(self.file_id, start.start, end.start),
455        ));
456    }
457
458    /// Two-char operator where the second char may be `=`.
459    fn lex_two(&mut self, plain: TokenKind, assign: TokenKind, second: char) {
460        let start = self.here(1);
461        if self.peek(1) == Some(second) {
462            self.advance();
463            let text = format!("{}{}", self.chars[self.pos - 1], second);
464            self.advance();
465            let end = self.here(0);
466            self.tokens.push(Token::new(
467                assign,
468                text,
469                Span::new(self.file_id, start.start, end.start),
470            ));
471        } else {
472            let text = self.chars[self.pos].to_string();
473            self.advance();
474            let end = self.here(0);
475            self.tokens.push(Token::new(
476                plain,
477                text,
478                Span::new(self.file_id, start.start, end.start),
479            ));
480        }
481    }
482
483    fn lex_duplicate(&mut self, kind: TokenKind, text: &str) {
484        let start = self.here(1);
485        self.advance();
486        self.advance();
487        let end = self.here(0);
488        self.tokens.push(Token::new(
489            kind,
490            text,
491            Span::new(self.file_id, start.start, end.start),
492        ));
493    }
494
495    /// Two-char operator with a fixed second char (e.g. `==`, `<=`).
496    fn two(&mut self, plain: TokenKind, combined: TokenKind, second: char) {
497        let start = self.here(1);
498        let text = self.chars[self.pos].to_string();
499        if self.peek(1) == Some(second) {
500            self.advance();
501            let combined_text = format!("{}{}", text, second);
502            self.advance();
503            let end = self.here(0);
504            self.tokens.push(Token::new(
505                combined,
506                combined_text,
507                Span::new(self.file_id, start.start, end.start),
508            ));
509        } else {
510            self.advance();
511            let end = self.here(0);
512            self.tokens.push(Token::new(
513                plain,
514                text,
515                Span::new(self.file_id, start.start, end.start),
516            ));
517        }
518    }
519
520    fn here(&self, width: usize) -> Span {
521        Span::new(
522            self.file_id,
523            Position::new(self.line, self.col),
524            Position::new(self.line, self.col + width as u32),
525        )
526    }
527
528    fn peek(&self, offset: usize) -> Option<char> {
529        self.chars.get(self.pos + offset).copied()
530    }
531
532    fn advance(&mut self) {
533        self.pos += 1;
534        self.col += 1;
535    }
536}
537
538fn is_ident_start(c: char) -> bool {
539    c.is_ascii_alphabetic() || c == '_'
540}
541
542fn is_ident_continue(c: char) -> bool {
543    c.is_ascii_alphanumeric() || c == '_'
544}
545
546#[cfg(test)]
547mod tests {
548    use super::*;
549
550    fn lex_ok(text: &str) -> Vec<Token> {
551        lex(LexInput { file_id: 0, text }).unwrap()
552    }
553
554    #[test]
555    fn lexes_basic_rule() {
556        let tokens = lex_ok("rule \"setup\":\n    @Event global\n    disableInspector()\n");
557        let kinds: Vec<TokenKind> = tokens.iter().map(|t| t.kind).collect();
558        assert!(kinds.contains(&TokenKind::Ident));
559        assert!(kinds.contains(&TokenKind::String));
560        assert!(kinds.contains(&TokenKind::Colon));
561        assert!(kinds.contains(&TokenKind::At));
562        assert!(kinds.contains(&TokenKind::LParen));
563        assert!(kinds.contains(&TokenKind::Eof));
564    }
565
566    #[test]
567    fn numbers_preserve_text() {
568        let tokens = lex_ok("1 2.5 0.016 100");
569        let numbers: Vec<&str> = tokens
570            .iter()
571            .filter(|t| t.kind == TokenKind::Number)
572            .map(|t| t.text.as_str())
573            .collect();
574        assert_eq!(numbers, vec!["1", "2.5", "0.016", "100"]);
575    }
576
577    #[test]
578    fn directives_and_comments() {
579        let tokens = lex_ok("#!define X 1\n# comment\nrule \"r\":\n");
580        let directive = tokens
581            .iter()
582            .find(|t| t.kind == TokenKind::Directive)
583            .unwrap();
584        assert_eq!(directive.text, "define X 1");
585        assert!(!tokens.iter().any(|t| t.text == "comment"));
586    }
587
588    #[test]
589    fn directive_line_continuation_is_part_of_one_directive() {
590        let tokens = lex_ok("#!define X first + \\\n  second\n");
591        let directive = tokens
592            .iter()
593            .find(|token| token.kind == TokenKind::Directive)
594            .unwrap();
595        assert_eq!(directive.text, "define X first + \n  second");
596        assert_eq!(directive.span.start, Position::new(1, 1));
597        assert_eq!(directive.span.end, Position::new(2, 9));
598    }
599
600    #[test]
601    fn operators() {
602        let tokens = lex_ok("a += b == c <= d != e / f");
603        let kinds: Vec<TokenKind> = tokens.iter().map(|t| t.kind).collect();
604        for expected in [
605            TokenKind::PlusAssign,
606            TokenKind::Eq,
607            TokenKind::Le,
608            TokenKind::Ne,
609            TokenKind::Slash,
610        ] {
611            assert!(
612                kinds.contains(&expected),
613                "missing {expected:?} in {kinds:?}"
614            );
615        }
616    }
617
618    #[test]
619    fn power_operators_disambiguate() {
620        // The pinned OverPy 9.7.10 reference lexes `**=` as one power-assign
621        // operator distinct from `**` and `*=`.
622        let tokens = lex_ok("a **= b ** c *= d");
623        let kinds: Vec<TokenKind> = tokens.iter().map(|t| t.kind).collect();
624        assert_eq!(
625            kinds,
626            vec![
627                TokenKind::Ident,
628                TokenKind::DoubleStarAssign,
629                TokenKind::Ident,
630                TokenKind::DoubleStar,
631                TokenKind::Ident,
632                TokenKind::StarAssign,
633                TokenKind::Ident,
634                TokenKind::Eof,
635            ]
636        );
637        let assign = tokens
638            .iter()
639            .find(|t| t.kind == TokenKind::DoubleStarAssign)
640            .unwrap();
641        assert_eq!(assign.text, "**=");
642    }
643
644    #[test]
645    fn postfix_operators_are_single_tokens() {
646        let tokens = lex_ok("counter++ points--");
647        assert_eq!(
648            tokens.iter().map(|token| token.kind).collect::<Vec<_>>(),
649            vec![
650                TokenKind::Ident,
651                TokenKind::Increment,
652                TokenKind::Ident,
653                TokenKind::Decrement,
654                TokenKind::Eof,
655            ]
656        );
657        assert_eq!(tokens[1].span.start.col, 8);
658        assert_eq!(tokens[1].span.end.col, 10);
659    }
660
661    #[test]
662    fn unterminated_string_is_structured() {
663        let error = lex(LexInput {
664            file_id: 0,
665            text: "rule \"x\n",
666        })
667        .unwrap_err();
668        assert_eq!(error.code, "lex-error");
669        assert!(error.span.is_some());
670    }
671
672    #[test]
673    fn backslash_line_continuation_is_not_a_token() {
674        let tokens = lex_ok("one \\\ntwo");
675        assert_eq!(
676            tokens.iter().map(|token| token.kind).collect::<Vec<_>>(),
677            vec![TokenKind::Ident, TokenKind::Ident, TokenKind::Eof]
678        );
679        assert_eq!(tokens[1].span.start.line, 2);
680        assert_eq!(tokens[1].span.start.col, 1);
681    }
682
683    #[test]
684    fn crlf_line_continuation_tracks_the_next_line() {
685        let tokens = lex_ok("one \\\r\ntwo");
686        assert_eq!(tokens[1].span.start, Position::new(2, 1));
687    }
688
689    #[test]
690    fn whitespace_before_line_ending_is_part_of_the_continuation() {
691        let tokens = lex_ok("one \\  \ntwo");
692        assert_eq!(tokens[1].span.start, Position::new(2, 1));
693    }
694
695    #[test]
696    fn non_newline_backslash_remains_a_lex_error() {
697        for text in ["one \\ two", "one \\", "one \\ \t\ntwo"] {
698            let error = lex(LexInput { file_id: 0, text }).unwrap_err();
699            assert_eq!(error.code, "lex-error");
700            assert_eq!(error.message, "unexpected character '\\'");
701            assert_eq!(error.span.unwrap().start, Position::new(1, 5));
702        }
703    }
704}
opy_rs/lexer.rs

opy_rs/
lexer.rs
