opy_rs/
lexer.rs

1//! The native `.opy` lexer.
2//!
3//! Produces a flat token stream (newlines and indentation included) from one
4//! source file. Comments (`#`, `/* */`) are skipped; `#!` directives are
5//! captured as a single directive token for the preprocessor. Positions are
6//! 1-based line/column, matching the Opy HIR protocol.
7
8use crate::diag::{OpyError, OpyResult, Position, Span};
9
10/// The kind of a token.
11#[derive(Debug, Clone, Copy, PartialEq, Eq)]
12pub enum TokenKind {
13    /// An identifier or keyword (keywords are resolved by the parser).
14    Ident,
15    /// A numeric literal (`text` holds the source spelling).
16    Number,
17    /// A string literal (`text` holds the unescaped value).
18    String,
19    /// A `#!` directive line (`text` holds everything after `#!`).
20    Directive,
21    /// A preprocessing marker carrying the rule-prefix state active at the
22    /// following top-level rule or subroutine.
23    RulePrefixMarker,
24    /// `@Event` / `@Condition` / other `@` directives.
25    At,
26    Newline,
27    /// Indentation change: the column of the current line.
28    Indent(u32),
29    /// End of file.
30    Eof,
31    // Punctuation and operators.
32    LParen,
33    RParen,
34    LBracket,
35    RBracket,
36    LBrace,
37    RBrace,
38    Comma,
39    Colon,
40    Semicolon,
41    Dot,
42    Assign,
43    Plus,
44    Minus,
45    Star,
46    Slash,
47    Percent,
48    DoubleStar,
49    PlusAssign,
50    MinusAssign,
51    Increment,
52    Decrement,
53    StarAssign,
54    SlashAssign,
55    PercentAssign,
56    DoubleStarAssign,
57    Eq,
58    Ne,
59    Lt,
60    Le,
61    Gt,
62    Ge,
63    /// A bare `!` that is not `!=`; the parser rejects it as unsupported OPY.
64    LexBang,
65}
66
67/// One token with its source span and payload text.
68#[derive(Debug, Clone, PartialEq)]
69pub struct Token {
70    pub kind: TokenKind,
71    /// The source text of this token (numbers keep their spelling; strings
72    /// keep their unescaped value; identifiers keep their name).
73    pub text: String,
74    /// The exact characters between string quotes, before escape decoding.
75    /// Other token kinds leave this unset. It is retained so source-language
76    /// constructs such as f-string interpolations can recover expression
77    /// spans without losing provenance during preprocessing.
78    pub raw: Option<String>,
79    pub span: Span,
80}
81
82impl Token {
83    fn new(kind: TokenKind, text: impl Into<String>, span: Span) -> Token {
84        Token {
85            kind,
86            text: text.into(),
87            raw: None,
88            span,
89        }
90    }
91}
92
93/// The lexer input: one file's text with its file id.
94pub struct LexInput<'a> {
95    pub file_id: u32,
96    pub text: &'a str,
97}
98
99/// Lex one source file into a token stream.
100pub fn lex(input: LexInput<'_>) -> OpyResult<Vec<Token>> {
101    Lexer::new(input.file_id, input.text).run()
102}
103
104struct Lexer {
105    file_id: u32,
106    chars: Vec<char>,
107    pos: usize,
108    line: u32,
109    col: u32,
110    tokens: Vec<Token>,
111}
112
113impl Lexer {
114    fn new(file_id: u32, text: &str) -> Lexer {
115        Lexer {
116            file_id,
117            chars: text.chars().collect(),
118            pos: 0,
119            line: 1,
120            col: 1,
121            tokens: Vec::new(),
122        }
123    }
124
125    fn run(mut self) -> OpyResult<Vec<Token>> {
126        while self.pos < self.chars.len() {
127            let ch = self.chars[self.pos];
128            match ch {
129                '\n' => {
130                    self.tokens
131                        .push(Token::new(TokenKind::Newline, "\n", self.here(1)));
132                    self.advance();
133                    self.line += 1;
134                    self.col = 1;
135                }
136                ' ' | '\t' | '\r' => {
137                    self.advance();
138                }
139                '\\' => {
140                    if !self.skip_line_continuation() {
141                        return Err(OpyError::at(
142                            "lex-error",
143                            "unexpected character '\\'",
144                            self.here(1),
145                        ));
146                    }
147                }
148                '#' => self.lex_hash()?,
149                '/' if self.peek(1) == Some('*') => self.skip_block_comment()?,
150                '"' | '\'' => self.lex_string(ch)?,
151                c if c.is_ascii_digit() => self.lex_number()?,
152                c if is_ident_start(c) => self.lex_ident(),
153                '(' => self.single(TokenKind::LParen),
154                ')' => self.single(TokenKind::RParen),
155                '[' => self.single(TokenKind::LBracket),
156                ']' => self.single(TokenKind::RBracket),
157                '{' => self.single(TokenKind::LBrace),
158                '}' => self.single(TokenKind::RBrace),
159                ',' => self.single(TokenKind::Comma),
160                ':' => self.single(TokenKind::Colon),
161                ';' => self.single(TokenKind::Semicolon),
162                '.' => self.single(TokenKind::Dot),
163                '@' => self.single(TokenKind::At),
164                '=' => self.two(TokenKind::Assign, TokenKind::Eq, '='),
165                '+' => {
166                    if self.peek(1) == Some('+') {
167                        self.lex_duplicate(TokenKind::Increment, "++");
168                    } else {
169                        self.lex_two(TokenKind::Plus, TokenKind::PlusAssign, '=');
170                    }
171                }
172                '-' => {
173                    if self.peek(1) == Some('-') {
174                        self.lex_duplicate(TokenKind::Decrement, "--");
175                    } else {
176                        self.lex_two(TokenKind::Minus, TokenKind::MinusAssign, '=');
177                    }
178                }
179                '*' => {
180                    if self.peek(1) == Some('*') {
181                        if self.peek(2) == Some('=') {
182                            let start = self.here(3);
183                            self.advance();
184                            self.advance();
185                            self.advance();
186                            let end = self.here(0);
187                            self.tokens.push(Token::new(
188                                TokenKind::DoubleStarAssign,
189                                "**=",
190                                Span::new(self.file_id, start.start, end.start),
191                            ));
192                        } else {
193                            self.advance();
194                            self.single(TokenKind::DoubleStar)
195                        }
196                    } else {
197                        self.lex_two(TokenKind::Star, TokenKind::StarAssign, '=')
198                    }
199                }
200                '/' => self.lex_two(TokenKind::Slash, TokenKind::SlashAssign, '='),
201                '%' => self.lex_two(TokenKind::Percent, TokenKind::PercentAssign, '='),
202                '<' => self.two(TokenKind::Lt, TokenKind::Le, '='),
203                '>' => self.two(TokenKind::Gt, TokenKind::Ge, '='),
204                '!' => self.two(TokenKind::LexBang, TokenKind::Ne, '='),
205                other => {
206                    return Err(OpyError::at(
207                        "lex-error",
208                        format!("unexpected character '{other}'"),
209                        self.here(1),
210                    ));
211                }
212            }
213        }
214        let here = self.here(0);
215        self.tokens.push(Token::new(TokenKind::Eof, "", here));
216        Ok(self.tokens)
217    }
218
219    /// `#` starts a `#!` directive (captured as one token) or a comment.
220    fn lex_hash(&mut self) -> OpyResult<()> {
221        if self.peek(1) == Some('!') {
222            let start = self.here(2);
223            self.advance();
224            self.advance();
225            let mut text = String::new();
226            while self.pos < self.chars.len() {
227                if self.chars[self.pos] == '\\' && self.skip_line_continuation() {
228                    continue;
229                }
230                if self.chars[self.pos] == '\n' {
231                    break;
232                }
233                text.push(self.chars[self.pos]);
234                self.advance();
235            }
236            let end = self.here(0);
237            self.tokens.push(Token::new(
238                TokenKind::Directive,
239                text,
240                Span::new(self.file_id, start.start, end.start),
241            ));
242        } else {
243            while self.pos < self.chars.len() && self.chars[self.pos] != '\n' {
244                self.advance();
245            }
246        }
247        Ok(())
248    }
249
250    fn skip_block_comment(&mut self) -> OpyResult<()> {
251        let start = self.here(2);
252        self.advance();
253        self.advance();
254        while self.pos < self.chars.len() {
255            if self.chars[self.pos] == '*' && self.peek(1) == Some('/') {
256                self.advance();
257                self.advance();
258                return Ok(());
259            }
260            if self.chars[self.pos] == '\n' {
261                self.advance();
262                self.line += 1;
263                self.col = 1;
264            } else {
265                self.advance();
266            }
267        }
268        Err(OpyError::at(
269            "lex-error",
270            "unterminated block comment",
271            start,
272        ))
273    }
274
275    fn lex_string(&mut self, quote: char) -> OpyResult<()> {
276        let start = self.here(1);
277        self.advance();
278        let mut value = String::new();
279        let mut raw = String::new();
280        while self.pos < self.chars.len() {
281            let ch = self.chars[self.pos];
282            if ch == quote {
283                self.advance();
284                let end = self.here(0);
285                let mut token = Token::new(
286                    TokenKind::String,
287                    value,
288                    Span::new(self.file_id, start.start, end.start),
289                );
290                token.raw = Some(raw);
291                self.tokens.push(token);
292                return Ok(());
293            }
294            if ch == '\\' {
295                raw.push(ch);
296                self.advance();
297                if self.pos >= self.chars.len() {
298                    break;
299                }
300                let escaped = self.chars[self.pos];
301                raw.push(escaped);
302                value.push(match escaped {
303                    'n' => '\n',
304                    't' => '\t',
305                    'r' => '\r',
306                    '\\' => '\\',
307                    '"' => '"',
308                    '\'' => '\'',
309                    other => other,
310                });
311                self.advance();
312                continue;
313            }
314            if ch == '\n' {
315                return Err(OpyError::at(
316                    "lex-error",
317                    "unterminated string literal",
318                    start,
319                ));
320            }
321            raw.push(ch);
322            value.push(ch);
323            self.advance();
324        }
325        Err(OpyError::at(
326            "lex-error",
327            "unterminated string literal",
328            start,
329        ))
330    }
331
332    fn skip_line_continuation(&mut self) -> bool {
333        let mut offset = 1;
334        while matches!(self.peek(offset), Some(' ' | '\r')) {
335            offset += 1;
336        }
337        if self.peek(offset) != Some('\n') {
338            return false;
339        }
340        for _ in 0..=offset {
341            self.advance();
342        }
343        self.line += 1;
344        self.col = 1;
345        true
346    }
347
348    fn lex_number(&mut self) -> OpyResult<()> {
349        let start = self.here(1);
350        let mut text = String::new();
351        if self.chars[self.pos] == '0' && matches!(self.peek(1), Some('x' | 'X')) {
352            text.push('0');
353            self.advance();
354            text.push(self.chars[self.pos]);
355            self.advance();
356            let digits_start = self.pos;
357            while self.pos < self.chars.len() && self.chars[self.pos].is_ascii_hexdigit() {
358                text.push(self.chars[self.pos]);
359                self.advance();
360            }
361            if self.pos == digits_start {
362                return Err(OpyError::at(
363                    "lex-error",
364                    "hexadecimal literal requires at least one hexadecimal digit",
365                    Span::new(self.file_id, start.start, self.here(0).start),
366                ));
367            }
368            let end = self.here(0);
369            self.tokens.push(Token::new(
370                TokenKind::Number,
371                text,
372                Span::new(self.file_id, start.start, end.start),
373            ));
374            return Ok(());
375        }
376        while self.pos < self.chars.len() && self.chars[self.pos].is_ascii_digit() {
377            text.push(self.chars[self.pos]);
378            self.advance();
379        }
380        if self.pos < self.chars.len()
381            && self.chars[self.pos] == '.'
382            && self.peek(1).is_some_and(|c| c.is_ascii_digit())
383        {
384            text.push('.');
385            self.advance();
386            while self.pos < self.chars.len() && self.chars[self.pos].is_ascii_digit() {
387                text.push(self.chars[self.pos]);
388                self.advance();
389            }
390        }
391        // Optional exponent (not exercised by the corpus, supported for
392        // completeness of the number surface).
393        if self.pos < self.chars.len()
394            && (self.chars[self.pos] == 'e' || self.chars[self.pos] == 'E')
395        {
396            let mut lookahead = self.pos + 1;
397            if lookahead < self.chars.len()
398                && (self.chars[lookahead] == '+' || self.chars[lookahead] == '-')
399            {
400                lookahead += 1;
401            }
402            if lookahead < self.chars.len() && self.chars[lookahead].is_ascii_digit() {
403                text.push('e');
404                self.advance();
405                if self.pos < self.chars.len()
406                    && (self.chars[self.pos] == '+' || self.chars[self.pos] == '-')
407                {
408                    text.push(self.chars[self.pos]);
409                    self.advance();
410                }
411                while self.pos < self.chars.len() && self.chars[self.pos].is_ascii_digit() {
412                    text.push(self.chars[self.pos]);
413                    self.advance();
414                }
415            }
416        }
417        let end = self.here(0);
418        self.tokens.push(Token::new(
419            TokenKind::Number,
420            text,
421            Span::new(self.file_id, start.start, end.start),
422        ));
423        Ok(())
424    }
425
426    fn lex_ident(&mut self) {
427        let start = self.here(1);
428        let mut text = String::new();
429        while self.pos < self.chars.len() && is_ident_continue(self.chars[self.pos]) {
430            text.push(self.chars[self.pos]);
431            self.advance();
432        }
433        let end = self.here(0);
434        self.tokens.push(Token::new(
435            TokenKind::Ident,
436            text,
437            Span::new(self.file_id, start.start, end.start),
438        ));
439    }
440
441    fn single(&mut self, kind: TokenKind) {
442        let start = self.here(1);
443        let text = self.chars[self.pos].to_string();
444        self.advance();
445        let end = self.here(0);
446        self.tokens.push(Token::new(
447            kind,
448            text,
449            Span::new(self.file_id, start.start, end.start),
450        ));
451    }
452
453    /// Two-char operator where the second char may be `=`.
454    fn lex_two(&mut self, plain: TokenKind, assign: TokenKind, second: char) {
455        let start = self.here(1);
456        if self.peek(1) == Some(second) {
457            self.advance();
458            let text = format!("{}{}", self.chars[self.pos - 1], second);
459            self.advance();
460            let end = self.here(0);
461            self.tokens.push(Token::new(
462                assign,
463                text,
464                Span::new(self.file_id, start.start, end.start),
465            ));
466        } else {
467            let text = self.chars[self.pos].to_string();
468            self.advance();
469            let end = self.here(0);
470            self.tokens.push(Token::new(
471                plain,
472                text,
473                Span::new(self.file_id, start.start, end.start),
474            ));
475        }
476    }
477
478    fn lex_duplicate(&mut self, kind: TokenKind, text: &str) {
479        let start = self.here(1);
480        self.advance();
481        self.advance();
482        let end = self.here(0);
483        self.tokens.push(Token::new(
484            kind,
485            text,
486            Span::new(self.file_id, start.start, end.start),
487        ));
488    }
489
490    /// Two-char operator with a fixed second char (e.g. `==`, `<=`).
491    fn two(&mut self, plain: TokenKind, combined: TokenKind, second: char) {
492        let start = self.here(1);
493        let text = self.chars[self.pos].to_string();
494        if self.peek(1) == Some(second) {
495            self.advance();
496            let combined_text = format!("{}{}", text, second);
497            self.advance();
498            let end = self.here(0);
499            self.tokens.push(Token::new(
500                combined,
501                combined_text,
502                Span::new(self.file_id, start.start, end.start),
503            ));
504        } else {
505            self.advance();
506            let end = self.here(0);
507            self.tokens.push(Token::new(
508                plain,
509                text,
510                Span::new(self.file_id, start.start, end.start),
511            ));
512        }
513    }
514
515    fn here(&self, width: usize) -> Span {
516        Span::new(
517            self.file_id,
518            Position::new(self.line, self.col),
519            Position::new(self.line, self.col + width as u32),
520        )
521    }
522
523    fn peek(&self, offset: usize) -> Option<char> {
524        self.chars.get(self.pos + offset).copied()
525    }
526
527    fn advance(&mut self) {
528        self.pos += 1;
529        self.col += 1;
530    }
531}
532
533fn is_ident_start(c: char) -> bool {
534    c.is_ascii_alphabetic() || c == '_'
535}
536
537fn is_ident_continue(c: char) -> bool {
538    c.is_ascii_alphanumeric() || c == '_'
539}
540
541#[cfg(test)]
542mod tests {
543    use super::*;
544
545    fn lex_ok(text: &str) -> Vec<Token> {
546        lex(LexInput { file_id: 0, text }).unwrap()
547    }
548
549    #[test]
550    fn lexes_basic_rule() {
551        let tokens = lex_ok("rule \"setup\":\n    @Event global\n    disableInspector()\n");
552        let kinds: Vec<TokenKind> = tokens.iter().map(|t| t.kind).collect();
553        assert!(kinds.contains(&TokenKind::Ident));
554        assert!(kinds.contains(&TokenKind::String));
555        assert!(kinds.contains(&TokenKind::Colon));
556        assert!(kinds.contains(&TokenKind::At));
557        assert!(kinds.contains(&TokenKind::LParen));
558        assert!(kinds.contains(&TokenKind::Eof));
559    }
560
561    #[test]
562    fn numbers_preserve_text() {
563        let tokens = lex_ok("1 2.5 0.016 100");
564        let numbers: Vec<&str> = tokens
565            .iter()
566            .filter(|t| t.kind == TokenKind::Number)
567            .map(|t| t.text.as_str())
568            .collect();
569        assert_eq!(numbers, vec!["1", "2.5", "0.016", "100"]);
570    }
571
572    #[test]
573    fn directives_and_comments() {
574        let tokens = lex_ok("#!define X 1\n# comment\nrule \"r\":\n");
575        let directive = tokens
576            .iter()
577            .find(|t| t.kind == TokenKind::Directive)
578            .unwrap();
579        assert_eq!(directive.text, "define X 1");
580        assert!(!tokens.iter().any(|t| t.text == "comment"));
581    }
582
583    #[test]
584    fn directive_line_continuation_is_part_of_one_directive() {
585        let tokens = lex_ok("#!define X first + \\\n  second\n");
586        let directive = tokens
587            .iter()
588            .find(|token| token.kind == TokenKind::Directive)
589            .unwrap();
590        assert_eq!(directive.text, "define X first +   second");
591        assert_eq!(directive.span.start, Position::new(1, 1));
592        assert_eq!(directive.span.end, Position::new(2, 9));
593    }
594
595    #[test]
596    fn operators() {
597        let tokens = lex_ok("a += b == c <= d != e / f");
598        let kinds: Vec<TokenKind> = tokens.iter().map(|t| t.kind).collect();
599        for expected in [
600            TokenKind::PlusAssign,
601            TokenKind::Eq,
602            TokenKind::Le,
603            TokenKind::Ne,
604            TokenKind::Slash,
605        ] {
606            assert!(
607                kinds.contains(&expected),
608                "missing {expected:?} in {kinds:?}"
609            );
610        }
611    }
612
613    #[test]
614    fn power_operators_disambiguate() {
615        // The pinned OverPy 9.7.10 reference lexes `**=` as one power-assign
616        // operator distinct from `**` and `*=`.
617        let tokens = lex_ok("a **= b ** c *= d");
618        let kinds: Vec<TokenKind> = tokens.iter().map(|t| t.kind).collect();
619        assert_eq!(
620            kinds,
621            vec![
622                TokenKind::Ident,
623                TokenKind::DoubleStarAssign,
624                TokenKind::Ident,
625                TokenKind::DoubleStar,
626                TokenKind::Ident,
627                TokenKind::StarAssign,
628                TokenKind::Ident,
629                TokenKind::Eof,
630            ]
631        );
632        let assign = tokens
633            .iter()
634            .find(|t| t.kind == TokenKind::DoubleStarAssign)
635            .unwrap();
636        assert_eq!(assign.text, "**=");
637    }
638
639    #[test]
640    fn postfix_operators_are_single_tokens() {
641        let tokens = lex_ok("counter++ points--");
642        assert_eq!(
643            tokens.iter().map(|token| token.kind).collect::<Vec<_>>(),
644            vec![
645                TokenKind::Ident,
646                TokenKind::Increment,
647                TokenKind::Ident,
648                TokenKind::Decrement,
649                TokenKind::Eof,
650            ]
651        );
652        assert_eq!(tokens[1].span.start.col, 8);
653        assert_eq!(tokens[1].span.end.col, 10);
654    }
655
656    #[test]
657    fn unterminated_string_is_structured() {
658        let error = lex(LexInput {
659            file_id: 0,
660            text: "rule \"x\n",
661        })
662        .unwrap_err();
663        assert_eq!(error.code, "lex-error");
664        assert!(error.span.is_some());
665    }
666
667    #[test]
668    fn backslash_line_continuation_is_not_a_token() {
669        let tokens = lex_ok("one \\\ntwo");
670        assert_eq!(
671            tokens.iter().map(|token| token.kind).collect::<Vec<_>>(),
672            vec![TokenKind::Ident, TokenKind::Ident, TokenKind::Eof]
673        );
674        assert_eq!(tokens[1].span.start.line, 2);
675        assert_eq!(tokens[1].span.start.col, 1);
676    }
677
678    #[test]
679    fn crlf_line_continuation_tracks_the_next_line() {
680        let tokens = lex_ok("one \\\r\ntwo");
681        assert_eq!(tokens[1].span.start, Position::new(2, 1));
682    }
683
684    #[test]
685    fn whitespace_before_line_ending_is_part_of_the_continuation() {
686        let tokens = lex_ok("one \\  \ntwo");
687        assert_eq!(tokens[1].span.start, Position::new(2, 1));
688    }
689
690    #[test]
691    fn non_newline_backslash_remains_a_lex_error() {
692        for text in ["one \\ two", "one \\", "one \\ \t\ntwo"] {
693            let error = lex(LexInput { file_id: 0, text }).unwrap_err();
694            assert_eq!(error.code, "lex-error");
695            assert_eq!(error.message, "unexpected character '\\'");
696            assert_eq!(error.span.unwrap().start, Position::new(1, 5));
697        }
698    }
699}
opy_rs/lexer.rs

opy_rs/
lexer.rs
