opy_rs/
lexer.rs

1//! The native `.opy` lexer.
2//!
3//! Produces a flat token stream (newlines and indentation included) from one
4//! source file. Comments (`#`, `/* */`) are skipped; `#!` directives are
5//! captured as a single directive token for the preprocessor. Positions are
6//! 1-based line/column, matching the Opy HIR protocol.
7
8use crate::diag::{OpyError, OpyResult, Position, Span};
9
10/// The kind of a token.
11#[derive(Debug, Clone, Copy, PartialEq, Eq)]
12pub enum TokenKind {
13    /// An identifier or keyword (keywords are resolved by the parser).
14    Ident,
15    /// A numeric literal (`text` holds the source spelling).
16    Number,
17    /// A string literal (`text` holds the unescaped value).
18    String,
19    /// A `#!` directive line (`text` holds everything after `#!`).
20    Directive,
21    /// A preprocessing marker carrying the rule-prefix state active at the
22    /// following top-level rule or subroutine.
23    RulePrefixMarker,
24    /// `@Event` / `@Condition` / other `@` directives.
25    At,
26    Newline,
27    /// Indentation change: the column of the current line.
28    Indent(u32),
29    /// End of file.
30    Eof,
31    // Punctuation and operators.
32    LParen,
33    RParen,
34    LBracket,
35    RBracket,
36    LBrace,
37    RBrace,
38    Comma,
39    Colon,
40    Semicolon,
41    Dot,
42    Assign,
43    Plus,
44    Minus,
45    Star,
46    Slash,
47    Percent,
48    DoubleStar,
49    PlusAssign,
50    MinusAssign,
51    Increment,
52    Decrement,
53    StarAssign,
54    SlashAssign,
55    PercentAssign,
56    DoubleStarAssign,
57    Eq,
58    Ne,
59    Lt,
60    Le,
61    Gt,
62    Ge,
63    /// A bare `!` that is not `!=`; the parser rejects it as unsupported OPY.
64    LexBang,
65}
66
67/// One token with its source span and payload text.
68#[derive(Debug, Clone, PartialEq)]
69pub struct Token {
70    pub kind: TokenKind,
71    /// The source text of this token (numbers keep their spelling; strings
72    /// keep their unescaped value; identifiers keep their name).
73    pub text: String,
74    /// The exact characters between string quotes, before escape decoding.
75    /// Other token kinds leave this unset. It is retained so source-language
76    /// constructs such as f-string interpolations can recover expression
77    /// spans without losing provenance during preprocessing.
78    pub raw: Option<String>,
79    pub span: Span,
80}
81
82impl Token {
83    fn new(kind: TokenKind, text: impl Into<String>, span: Span) -> Token {
84        Token {
85            kind,
86            text: text.into(),
87            raw: None,
88            span,
89        }
90    }
91}
92
93/// The lexer input: one file's text with its file id.
94pub struct LexInput<'a> {
95    pub file_id: u32,
96    pub text: &'a str,
97}
98
99/// Lex one source file into a token stream.
100pub fn lex(input: LexInput<'_>) -> OpyResult<Vec<Token>> {
101    Lexer::new(input.file_id, input.text).run()
102}
103
104struct Lexer {
105    file_id: u32,
106    chars: Vec<char>,
107    pos: usize,
108    line: u32,
109    col: u32,
110    tokens: Vec<Token>,
111}
112
113impl Lexer {
114    fn new(file_id: u32, text: &str) -> Lexer {
115        Lexer {
116            file_id,
117            chars: text.chars().collect(),
118            pos: 0,
119            line: 1,
120            col: 1,
121            tokens: Vec::new(),
122        }
123    }
124
125    fn run(mut self) -> OpyResult<Vec<Token>> {
126        while self.pos < self.chars.len() {
127            let ch = self.chars[self.pos];
128            match ch {
129                '\n' => {
130                    self.tokens
131                        .push(Token::new(TokenKind::Newline, "\n", self.here(1)));
132                    self.advance();
133                    self.line += 1;
134                    self.col = 1;
135                }
136                ' ' | '\r' => {
137                    self.advance();
138                }
139                '\t' => {
140                    self.pos += 1;
141                    self.col += 4;
142                }
143                '\\' => {
144                    if !self.skip_line_continuation() {
145                        return Err(OpyError::at(
146                            "lex-error",
147                            "unexpected character '\\'",
148                            self.here(1),
149                        ));
150                    }
151                }
152                '#' => self.lex_hash()?,
153                '/' if self.peek(1) == Some('*') => self.skip_block_comment()?,
154                '"' | '\'' => self.lex_string(ch)?,
155                c if c.is_ascii_digit() => self.lex_number()?,
156                c if is_ident_start(c) => self.lex_ident(),
157                '(' => self.single(TokenKind::LParen),
158                ')' => self.single(TokenKind::RParen),
159                '[' => self.single(TokenKind::LBracket),
160                ']' => self.single(TokenKind::RBracket),
161                '{' => self.single(TokenKind::LBrace),
162                '}' => self.single(TokenKind::RBrace),
163                ',' => self.single(TokenKind::Comma),
164                ':' => self.single(TokenKind::Colon),
165                ';' => self.single(TokenKind::Semicolon),
166                '.' => self.single(TokenKind::Dot),
167                '@' => self.single(TokenKind::At),
168                '=' => self.two(TokenKind::Assign, TokenKind::Eq, '='),
169                '+' => {
170                    if self.peek(1) == Some('+') {
171                        self.lex_duplicate(TokenKind::Increment, "++");
172                    } else {
173                        self.lex_two(TokenKind::Plus, TokenKind::PlusAssign, '=');
174                    }
175                }
176                '-' => {
177                    if self.peek(1) == Some('-') {
178                        self.lex_duplicate(TokenKind::Decrement, "--");
179                    } else {
180                        self.lex_two(TokenKind::Minus, TokenKind::MinusAssign, '=');
181                    }
182                }
183                '*' => {
184                    if self.peek(1) == Some('*') {
185                        if self.peek(2) == Some('=') {
186                            let start = self.here(3);
187                            self.advance();
188                            self.advance();
189                            self.advance();
190                            let end = self.here(0);
191                            self.tokens.push(Token::new(
192                                TokenKind::DoubleStarAssign,
193                                "**=",
194                                Span::new(self.file_id, start.start, end.start),
195                            ));
196                        } else {
197                            self.advance();
198                            self.single(TokenKind::DoubleStar)
199                        }
200                    } else {
201                        self.lex_two(TokenKind::Star, TokenKind::StarAssign, '=')
202                    }
203                }
204                '/' => self.lex_two(TokenKind::Slash, TokenKind::SlashAssign, '='),
205                '%' => self.lex_two(TokenKind::Percent, TokenKind::PercentAssign, '='),
206                '<' => self.two(TokenKind::Lt, TokenKind::Le, '='),
207                '>' => self.two(TokenKind::Gt, TokenKind::Ge, '='),
208                '!' => self.two(TokenKind::LexBang, TokenKind::Ne, '='),
209                other => {
210                    return Err(OpyError::at(
211                        "lex-error",
212                        format!("unexpected character '{other}'"),
213                        self.here(1),
214                    ));
215                }
216            }
217        }
218        let here = self.here(0);
219        self.tokens.push(Token::new(TokenKind::Eof, "", here));
220        Ok(self.tokens)
221    }
222
223    /// `#` starts a `#!` directive (captured as one token) or a comment.
224    fn lex_hash(&mut self) -> OpyResult<()> {
225        if self.peek(1) == Some('!') {
226            let start = self.here(2);
227            self.advance();
228            self.advance();
229            let mut text = String::new();
230            while self.pos < self.chars.len() {
231                if self.chars[self.pos] == '\\' && self.skip_line_continuation() {
232                    continue;
233                }
234                if self.chars[self.pos] == '\n' {
235                    break;
236                }
237                text.push(self.chars[self.pos]);
238                self.advance();
239            }
240            let end = self.here(0);
241            self.tokens.push(Token::new(
242                TokenKind::Directive,
243                text,
244                Span::new(self.file_id, start.start, end.start),
245            ));
246        } else {
247            while self.pos < self.chars.len() && self.chars[self.pos] != '\n' {
248                self.advance();
249            }
250        }
251        Ok(())
252    }
253
254    fn skip_block_comment(&mut self) -> OpyResult<()> {
255        let start = self.here(2);
256        self.advance();
257        self.advance();
258        while self.pos < self.chars.len() {
259            if self.chars[self.pos] == '*' && self.peek(1) == Some('/') {
260                self.advance();
261                self.advance();
262                return Ok(());
263            }
264            if self.chars[self.pos] == '\n' {
265                self.advance();
266                self.line += 1;
267                self.col = 1;
268            } else {
269                self.advance();
270            }
271        }
272        Err(OpyError::at(
273            "lex-error",
274            "unterminated block comment",
275            start,
276        ))
277    }
278
279    fn lex_string(&mut self, quote: char) -> OpyResult<()> {
280        let start = self.here(1);
281        self.advance();
282        let mut value = String::new();
283        let mut raw = String::new();
284        while self.pos < self.chars.len() {
285            let ch = self.chars[self.pos];
286            if ch == quote {
287                self.advance();
288                let end = self.here(0);
289                let mut token = Token::new(
290                    TokenKind::String,
291                    value,
292                    Span::new(self.file_id, start.start, end.start),
293                );
294                token.raw = Some(raw);
295                self.tokens.push(token);
296                return Ok(());
297            }
298            if ch == '\\' {
299                raw.push(ch);
300                self.advance();
301                if self.pos >= self.chars.len() {
302                    break;
303                }
304                let escaped = self.chars[self.pos];
305                raw.push(escaped);
306                value.push(match escaped {
307                    'n' => '\n',
308                    't' => '\t',
309                    'r' => '\r',
310                    '\\' => '\\',
311                    '"' => '"',
312                    '\'' => '\'',
313                    other => other,
314                });
315                self.advance();
316                continue;
317            }
318            if ch == '\n' {
319                return Err(OpyError::at(
320                    "lex-error",
321                    "unterminated string literal",
322                    start,
323                ));
324            }
325            raw.push(ch);
326            value.push(ch);
327            self.advance();
328        }
329        Err(OpyError::at(
330            "lex-error",
331            "unterminated string literal",
332            start,
333        ))
334    }
335
336    fn skip_line_continuation(&mut self) -> bool {
337        let mut offset = 1;
338        while matches!(self.peek(offset), Some(' ' | '\r')) {
339            offset += 1;
340        }
341        if self.peek(offset) != Some('\n') {
342            return false;
343        }
344        for _ in 0..=offset {
345            self.advance();
346        }
347        self.line += 1;
348        self.col = 1;
349        true
350    }
351
352    fn lex_number(&mut self) -> OpyResult<()> {
353        let start = self.here(1);
354        let mut text = String::new();
355        if self.chars[self.pos] == '0' && matches!(self.peek(1), Some('x' | 'X')) {
356            text.push('0');
357            self.advance();
358            text.push(self.chars[self.pos]);
359            self.advance();
360            let digits_start = self.pos;
361            while self.pos < self.chars.len() && self.chars[self.pos].is_ascii_hexdigit() {
362                text.push(self.chars[self.pos]);
363                self.advance();
364            }
365            if self.pos == digits_start {
366                return Err(OpyError::at(
367                    "lex-error",
368                    "hexadecimal literal requires at least one hexadecimal digit",
369                    Span::new(self.file_id, start.start, self.here(0).start),
370                ));
371            }
372            let end = self.here(0);
373            self.tokens.push(Token::new(
374                TokenKind::Number,
375                text,
376                Span::new(self.file_id, start.start, end.start),
377            ));
378            return Ok(());
379        }
380        while self.pos < self.chars.len() && self.chars[self.pos].is_ascii_digit() {
381            text.push(self.chars[self.pos]);
382            self.advance();
383        }
384        if self.pos < self.chars.len()
385            && self.chars[self.pos] == '.'
386            && self.peek(1).is_some_and(|c| c.is_ascii_digit())
387        {
388            text.push('.');
389            self.advance();
390            while self.pos < self.chars.len() && self.chars[self.pos].is_ascii_digit() {
391                text.push(self.chars[self.pos]);
392                self.advance();
393            }
394        }
395        // Optional exponent (not exercised by the corpus, supported for
396        // completeness of the number surface).
397        if self.pos < self.chars.len()
398            && (self.chars[self.pos] == 'e' || self.chars[self.pos] == 'E')
399        {
400            let mut lookahead = self.pos + 1;
401            if lookahead < self.chars.len()
402                && (self.chars[lookahead] == '+' || self.chars[lookahead] == '-')
403            {
404                lookahead += 1;
405            }
406            if lookahead < self.chars.len() && self.chars[lookahead].is_ascii_digit() {
407                text.push('e');
408                self.advance();
409                if self.pos < self.chars.len()
410                    && (self.chars[self.pos] == '+' || self.chars[self.pos] == '-')
411                {
412                    text.push(self.chars[self.pos]);
413                    self.advance();
414                }
415                while self.pos < self.chars.len() && self.chars[self.pos].is_ascii_digit() {
416                    text.push(self.chars[self.pos]);
417                    self.advance();
418                }
419            }
420        }
421        let end = self.here(0);
422        self.tokens.push(Token::new(
423            TokenKind::Number,
424            text,
425            Span::new(self.file_id, start.start, end.start),
426        ));
427        Ok(())
428    }
429
430    fn lex_ident(&mut self) {
431        let start = self.here(1);
432        let mut text = String::new();
433        while self.pos < self.chars.len() && is_ident_continue(self.chars[self.pos]) {
434            text.push(self.chars[self.pos]);
435            self.advance();
436        }
437        let end = self.here(0);
438        self.tokens.push(Token::new(
439            TokenKind::Ident,
440            text,
441            Span::new(self.file_id, start.start, end.start),
442        ));
443    }
444
445    fn single(&mut self, kind: TokenKind) {
446        let start = self.here(1);
447        let text = self.chars[self.pos].to_string();
448        self.advance();
449        let end = self.here(0);
450        self.tokens.push(Token::new(
451            kind,
452            text,
453            Span::new(self.file_id, start.start, end.start),
454        ));
455    }
456
457    /// Two-char operator where the second char may be `=`.
458    fn lex_two(&mut self, plain: TokenKind, assign: TokenKind, second: char) {
459        let start = self.here(1);
460        if self.peek(1) == Some(second) {
461            self.advance();
462            let text = format!("{}{}", self.chars[self.pos - 1], second);
463            self.advance();
464            let end = self.here(0);
465            self.tokens.push(Token::new(
466                assign,
467                text,
468                Span::new(self.file_id, start.start, end.start),
469            ));
470        } else {
471            let text = self.chars[self.pos].to_string();
472            self.advance();
473            let end = self.here(0);
474            self.tokens.push(Token::new(
475                plain,
476                text,
477                Span::new(self.file_id, start.start, end.start),
478            ));
479        }
480    }
481
482    fn lex_duplicate(&mut self, kind: TokenKind, text: &str) {
483        let start = self.here(1);
484        self.advance();
485        self.advance();
486        let end = self.here(0);
487        self.tokens.push(Token::new(
488            kind,
489            text,
490            Span::new(self.file_id, start.start, end.start),
491        ));
492    }
493
494    /// Two-char operator with a fixed second char (e.g. `==`, `<=`).
495    fn two(&mut self, plain: TokenKind, combined: TokenKind, second: char) {
496        let start = self.here(1);
497        let text = self.chars[self.pos].to_string();
498        if self.peek(1) == Some(second) {
499            self.advance();
500            let combined_text = format!("{}{}", text, second);
501            self.advance();
502            let end = self.here(0);
503            self.tokens.push(Token::new(
504                combined,
505                combined_text,
506                Span::new(self.file_id, start.start, end.start),
507            ));
508        } else {
509            self.advance();
510            let end = self.here(0);
511            self.tokens.push(Token::new(
512                plain,
513                text,
514                Span::new(self.file_id, start.start, end.start),
515            ));
516        }
517    }
518
519    fn here(&self, width: usize) -> Span {
520        Span::new(
521            self.file_id,
522            Position::new(self.line, self.col),
523            Position::new(self.line, self.col + width as u32),
524        )
525    }
526
527    fn peek(&self, offset: usize) -> Option<char> {
528        self.chars.get(self.pos + offset).copied()
529    }
530
531    fn advance(&mut self) {
532        self.pos += 1;
533        self.col += 1;
534    }
535}
536
537fn is_ident_start(c: char) -> bool {
538    c.is_ascii_alphabetic() || c == '_'
539}
540
541fn is_ident_continue(c: char) -> bool {
542    c.is_ascii_alphanumeric() || c == '_'
543}
544
545#[cfg(test)]
546mod tests {
547    use super::*;
548
549    fn lex_ok(text: &str) -> Vec<Token> {
550        lex(LexInput { file_id: 0, text }).unwrap()
551    }
552
553    #[test]
554    fn lexes_basic_rule() {
555        let tokens = lex_ok("rule \"setup\":\n    @Event global\n    disableInspector()\n");
556        let kinds: Vec<TokenKind> = tokens.iter().map(|t| t.kind).collect();
557        assert!(kinds.contains(&TokenKind::Ident));
558        assert!(kinds.contains(&TokenKind::String));
559        assert!(kinds.contains(&TokenKind::Colon));
560        assert!(kinds.contains(&TokenKind::At));
561        assert!(kinds.contains(&TokenKind::LParen));
562        assert!(kinds.contains(&TokenKind::Eof));
563    }
564
565    #[test]
566    fn numbers_preserve_text() {
567        let tokens = lex_ok("1 2.5 0.016 100");
568        let numbers: Vec<&str> = tokens
569            .iter()
570            .filter(|t| t.kind == TokenKind::Number)
571            .map(|t| t.text.as_str())
572            .collect();
573        assert_eq!(numbers, vec!["1", "2.5", "0.016", "100"]);
574    }
575
576    #[test]
577    fn directives_and_comments() {
578        let tokens = lex_ok("#!define X 1\n# comment\nrule \"r\":\n");
579        let directive = tokens
580            .iter()
581            .find(|t| t.kind == TokenKind::Directive)
582            .unwrap();
583        assert_eq!(directive.text, "define X 1");
584        assert!(!tokens.iter().any(|t| t.text == "comment"));
585    }
586
587    #[test]
588    fn directive_line_continuation_is_part_of_one_directive() {
589        let tokens = lex_ok("#!define X first + \\\n  second\n");
590        let directive = tokens
591            .iter()
592            .find(|token| token.kind == TokenKind::Directive)
593            .unwrap();
594        assert_eq!(directive.text, "define X first +   second");
595        assert_eq!(directive.span.start, Position::new(1, 1));
596        assert_eq!(directive.span.end, Position::new(2, 9));
597    }
598
599    #[test]
600    fn operators() {
601        let tokens = lex_ok("a += b == c <= d != e / f");
602        let kinds: Vec<TokenKind> = tokens.iter().map(|t| t.kind).collect();
603        for expected in [
604            TokenKind::PlusAssign,
605            TokenKind::Eq,
606            TokenKind::Le,
607            TokenKind::Ne,
608            TokenKind::Slash,
609        ] {
610            assert!(
611                kinds.contains(&expected),
612                "missing {expected:?} in {kinds:?}"
613            );
614        }
615    }
616
617    #[test]
618    fn power_operators_disambiguate() {
619        // The pinned OverPy 9.7.10 reference lexes `**=` as one power-assign
620        // operator distinct from `**` and `*=`.
621        let tokens = lex_ok("a **= b ** c *= d");
622        let kinds: Vec<TokenKind> = tokens.iter().map(|t| t.kind).collect();
623        assert_eq!(
624            kinds,
625            vec![
626                TokenKind::Ident,
627                TokenKind::DoubleStarAssign,
628                TokenKind::Ident,
629                TokenKind::DoubleStar,
630                TokenKind::Ident,
631                TokenKind::StarAssign,
632                TokenKind::Ident,
633                TokenKind::Eof,
634            ]
635        );
636        let assign = tokens
637            .iter()
638            .find(|t| t.kind == TokenKind::DoubleStarAssign)
639            .unwrap();
640        assert_eq!(assign.text, "**=");
641    }
642
643    #[test]
644    fn postfix_operators_are_single_tokens() {
645        let tokens = lex_ok("counter++ points--");
646        assert_eq!(
647            tokens.iter().map(|token| token.kind).collect::<Vec<_>>(),
648            vec![
649                TokenKind::Ident,
650                TokenKind::Increment,
651                TokenKind::Ident,
652                TokenKind::Decrement,
653                TokenKind::Eof,
654            ]
655        );
656        assert_eq!(tokens[1].span.start.col, 8);
657        assert_eq!(tokens[1].span.end.col, 10);
658    }
659
660    #[test]
661    fn unterminated_string_is_structured() {
662        let error = lex(LexInput {
663            file_id: 0,
664            text: "rule \"x\n",
665        })
666        .unwrap_err();
667        assert_eq!(error.code, "lex-error");
668        assert!(error.span.is_some());
669    }
670
671    #[test]
672    fn backslash_line_continuation_is_not_a_token() {
673        let tokens = lex_ok("one \\\ntwo");
674        assert_eq!(
675            tokens.iter().map(|token| token.kind).collect::<Vec<_>>(),
676            vec![TokenKind::Ident, TokenKind::Ident, TokenKind::Eof]
677        );
678        assert_eq!(tokens[1].span.start.line, 2);
679        assert_eq!(tokens[1].span.start.col, 1);
680    }
681
682    #[test]
683    fn crlf_line_continuation_tracks_the_next_line() {
684        let tokens = lex_ok("one \\\r\ntwo");
685        assert_eq!(tokens[1].span.start, Position::new(2, 1));
686    }
687
688    #[test]
689    fn whitespace_before_line_ending_is_part_of_the_continuation() {
690        let tokens = lex_ok("one \\  \ntwo");
691        assert_eq!(tokens[1].span.start, Position::new(2, 1));
692    }
693
694    #[test]
695    fn non_newline_backslash_remains_a_lex_error() {
696        for text in ["one \\ two", "one \\", "one \\ \t\ntwo"] {
697            let error = lex(LexInput { file_id: 0, text }).unwrap_err();
698            assert_eq!(error.code, "lex-error");
699            assert_eq!(error.message, "unexpected character '\\'");
700            assert_eq!(error.span.unwrap().start, Position::new(1, 5));
701        }
702    }
703}
opy_rs/lexer.rs

opy_rs/
lexer.rs
