opy_rs/
lexer.rs

1//! The native `.opy` lexer.
2//!
3//! Produces a flat token stream (newlines and indentation included) from one
4//! source file. Comments (`#`, `/* */`) are skipped; `#!` directives are
5//! captured as a single directive token for the preprocessor. Positions are
6//! 1-based line/column, matching the Opy HIR protocol.
7
8use crate::diag::{OpyError, OpyResult, Position, Span};
9
10/// The kind of a token.
11#[derive(Debug, Clone, Copy, PartialEq, Eq)]
12pub enum TokenKind {
13    /// An identifier or keyword (keywords are resolved by the parser).
14    Ident,
15    /// A numeric literal (`text` holds the source spelling).
16    Number,
17    /// A string literal (`text` holds the unescaped value).
18    String,
19    /// A `#!` directive line (`text` holds everything after `#!`).
20    Directive,
21    /// A preprocessing marker carrying the rule-prefix state active at the
22    /// following top-level rule or subroutine.
23    RulePrefixMarker,
24    /// `@Event` / `@Condition` / other `@` directives.
25    At,
26    Newline,
27    /// Indentation change: the column of the current line.
28    Indent(u32),
29    /// End of file.
30    Eof,
31    // Punctuation and operators.
32    LParen,
33    RParen,
34    LBracket,
35    RBracket,
36    LBrace,
37    RBrace,
38    Comma,
39    Colon,
40    Dot,
41    Assign,
42    Plus,
43    Minus,
44    Star,
45    Slash,
46    Percent,
47    DoubleStar,
48    PlusAssign,
49    MinusAssign,
50    Increment,
51    Decrement,
52    StarAssign,
53    SlashAssign,
54    PercentAssign,
55    DoubleStarAssign,
56    Eq,
57    Ne,
58    Lt,
59    Le,
60    Gt,
61    Ge,
62    /// A bare `!` that is not `!=`; the parser rejects it as unsupported OPY.
63    LexBang,
64}
65
66/// One token with its source span and payload text.
67#[derive(Debug, Clone, PartialEq)]
68pub struct Token {
69    pub kind: TokenKind,
70    /// The source text of this token (numbers keep their spelling; strings
71    /// keep their unescaped value; identifiers keep their name).
72    pub text: String,
73    /// The exact characters between string quotes, before escape decoding.
74    /// Other token kinds leave this unset. It is retained so source-language
75    /// constructs such as f-string interpolations can recover expression
76    /// spans without losing provenance during preprocessing.
77    pub raw: Option<String>,
78    pub span: Span,
79}
80
81impl Token {
82    fn new(kind: TokenKind, text: impl Into<String>, span: Span) -> Token {
83        Token {
84            kind,
85            text: text.into(),
86            raw: None,
87            span,
88        }
89    }
90}
91
92/// The lexer input: one file's text with its file id.
93pub struct LexInput<'a> {
94    pub file_id: u32,
95    pub text: &'a str,
96}
97
98/// Lex one source file into a token stream.
99pub fn lex(input: LexInput<'_>) -> OpyResult<Vec<Token>> {
100    Lexer::new(input.file_id, input.text).run()
101}
102
103struct Lexer {
104    file_id: u32,
105    chars: Vec<char>,
106    pos: usize,
107    line: u32,
108    col: u32,
109    tokens: Vec<Token>,
110}
111
112impl Lexer {
113    fn new(file_id: u32, text: &str) -> Lexer {
114        Lexer {
115            file_id,
116            chars: text.chars().collect(),
117            pos: 0,
118            line: 1,
119            col: 1,
120            tokens: Vec::new(),
121        }
122    }
123
124    fn run(mut self) -> OpyResult<Vec<Token>> {
125        while self.pos < self.chars.len() {
126            let ch = self.chars[self.pos];
127            match ch {
128                '\n' => {
129                    self.tokens
130                        .push(Token::new(TokenKind::Newline, "\n", self.here(1)));
131                    self.advance();
132                    self.line += 1;
133                    self.col = 1;
134                }
135                ' ' | '\t' | '\r' => {
136                    self.advance();
137                }
138                '\\' => {
139                    if !self.skip_line_continuation() {
140                        return Err(OpyError::at(
141                            "lex-error",
142                            "unexpected character '\\'",
143                            self.here(1),
144                        ));
145                    }
146                }
147                '#' => self.lex_hash()?,
148                '/' if self.peek(1) == Some('*') => self.skip_block_comment()?,
149                '"' | '\'' => self.lex_string(ch)?,
150                c if c.is_ascii_digit() => self.lex_number()?,
151                c if is_ident_start(c) => self.lex_ident(),
152                '(' => self.single(TokenKind::LParen),
153                ')' => self.single(TokenKind::RParen),
154                '[' => self.single(TokenKind::LBracket),
155                ']' => self.single(TokenKind::RBracket),
156                '{' => self.single(TokenKind::LBrace),
157                '}' => self.single(TokenKind::RBrace),
158                ',' => self.single(TokenKind::Comma),
159                ':' => self.single(TokenKind::Colon),
160                '.' => self.single(TokenKind::Dot),
161                '@' => self.single(TokenKind::At),
162                '=' => self.two(TokenKind::Assign, TokenKind::Eq, '='),
163                '+' => {
164                    if self.peek(1) == Some('+') {
165                        self.lex_duplicate(TokenKind::Increment, "++");
166                    } else {
167                        self.lex_two(TokenKind::Plus, TokenKind::PlusAssign, '=');
168                    }
169                }
170                '-' => {
171                    if self.peek(1) == Some('-') {
172                        self.lex_duplicate(TokenKind::Decrement, "--");
173                    } else {
174                        self.lex_two(TokenKind::Minus, TokenKind::MinusAssign, '=');
175                    }
176                }
177                '*' => {
178                    if self.peek(1) == Some('*') {
179                        if self.peek(2) == Some('=') {
180                            let start = self.here(3);
181                            self.advance();
182                            self.advance();
183                            self.advance();
184                            let end = self.here(0);
185                            self.tokens.push(Token::new(
186                                TokenKind::DoubleStarAssign,
187                                "**=",
188                                Span::new(self.file_id, start.start, end.start),
189                            ));
190                        } else {
191                            self.advance();
192                            self.single(TokenKind::DoubleStar)
193                        }
194                    } else {
195                        self.lex_two(TokenKind::Star, TokenKind::StarAssign, '=')
196                    }
197                }
198                '/' => self.lex_two(TokenKind::Slash, TokenKind::SlashAssign, '='),
199                '%' => self.lex_two(TokenKind::Percent, TokenKind::PercentAssign, '='),
200                '<' => self.two(TokenKind::Lt, TokenKind::Le, '='),
201                '>' => self.two(TokenKind::Gt, TokenKind::Ge, '='),
202                '!' => self.two(TokenKind::LexBang, TokenKind::Ne, '='),
203                other => {
204                    return Err(OpyError::at(
205                        "lex-error",
206                        format!("unexpected character '{other}'"),
207                        self.here(1),
208                    ));
209                }
210            }
211        }
212        let here = self.here(0);
213        self.tokens.push(Token::new(TokenKind::Eof, "", here));
214        Ok(self.tokens)
215    }
216
217    /// `#` starts a `#!` directive (captured as one token) or a comment.
218    fn lex_hash(&mut self) -> OpyResult<()> {
219        if self.peek(1) == Some('!') {
220            let start = self.here(2);
221            self.advance();
222            self.advance();
223            let mut text = String::new();
224            while self.pos < self.chars.len() && self.chars[self.pos] != '\n' {
225                text.push(self.chars[self.pos]);
226                self.advance();
227            }
228            let end = self.here(0);
229            self.tokens.push(Token::new(
230                TokenKind::Directive,
231                text,
232                Span::new(self.file_id, start.start, end.start),
233            ));
234        } else {
235            while self.pos < self.chars.len() && self.chars[self.pos] != '\n' {
236                self.advance();
237            }
238        }
239        Ok(())
240    }
241
242    fn skip_block_comment(&mut self) -> OpyResult<()> {
243        let start = self.here(2);
244        self.advance();
245        self.advance();
246        while self.pos < self.chars.len() {
247            if self.chars[self.pos] == '*' && self.peek(1) == Some('/') {
248                self.advance();
249                self.advance();
250                return Ok(());
251            }
252            if self.chars[self.pos] == '\n' {
253                self.advance();
254                self.line += 1;
255                self.col = 1;
256            } else {
257                self.advance();
258            }
259        }
260        Err(OpyError::at(
261            "lex-error",
262            "unterminated block comment",
263            start,
264        ))
265    }
266
267    fn lex_string(&mut self, quote: char) -> OpyResult<()> {
268        let start = self.here(1);
269        self.advance();
270        let mut value = String::new();
271        let mut raw = String::new();
272        while self.pos < self.chars.len() {
273            let ch = self.chars[self.pos];
274            if ch == quote {
275                self.advance();
276                let end = self.here(0);
277                let mut token = Token::new(
278                    TokenKind::String,
279                    value,
280                    Span::new(self.file_id, start.start, end.start),
281                );
282                token.raw = Some(raw);
283                self.tokens.push(token);
284                return Ok(());
285            }
286            if ch == '\\' {
287                raw.push(ch);
288                self.advance();
289                if self.pos >= self.chars.len() {
290                    break;
291                }
292                let escaped = self.chars[self.pos];
293                raw.push(escaped);
294                value.push(match escaped {
295                    'n' => '\n',
296                    't' => '\t',
297                    'r' => '\r',
298                    '\\' => '\\',
299                    '"' => '"',
300                    '\'' => '\'',
301                    other => other,
302                });
303                self.advance();
304                continue;
305            }
306            if ch == '\n' {
307                return Err(OpyError::at(
308                    "lex-error",
309                    "unterminated string literal",
310                    start,
311                ));
312            }
313            raw.push(ch);
314            value.push(ch);
315            self.advance();
316        }
317        Err(OpyError::at(
318            "lex-error",
319            "unterminated string literal",
320            start,
321        ))
322    }
323
324    fn skip_line_continuation(&mut self) -> bool {
325        let mut offset = 1;
326        while matches!(self.peek(offset), Some(' ' | '\r')) {
327            offset += 1;
328        }
329        if self.peek(offset) != Some('\n') {
330            return false;
331        }
332        for _ in 0..=offset {
333            self.advance();
334        }
335        self.line += 1;
336        self.col = 1;
337        true
338    }
339
340    fn lex_number(&mut self) -> OpyResult<()> {
341        let start = self.here(1);
342        let mut text = String::new();
343        if self.chars[self.pos] == '0' && matches!(self.peek(1), Some('x' | 'X')) {
344            text.push('0');
345            self.advance();
346            text.push(self.chars[self.pos]);
347            self.advance();
348            let digits_start = self.pos;
349            while self.pos < self.chars.len() && self.chars[self.pos].is_ascii_hexdigit() {
350                text.push(self.chars[self.pos]);
351                self.advance();
352            }
353            if self.pos == digits_start {
354                return Err(OpyError::at(
355                    "lex-error",
356                    "hexadecimal literal requires at least one hexadecimal digit",
357                    Span::new(self.file_id, start.start, self.here(0).start),
358                ));
359            }
360            let end = self.here(0);
361            self.tokens.push(Token::new(
362                TokenKind::Number,
363                text,
364                Span::new(self.file_id, start.start, end.start),
365            ));
366            return Ok(());
367        }
368        while self.pos < self.chars.len() && self.chars[self.pos].is_ascii_digit() {
369            text.push(self.chars[self.pos]);
370            self.advance();
371        }
372        if self.pos < self.chars.len()
373            && self.chars[self.pos] == '.'
374            && self.peek(1).is_some_and(|c| c.is_ascii_digit())
375        {
376            text.push('.');
377            self.advance();
378            while self.pos < self.chars.len() && self.chars[self.pos].is_ascii_digit() {
379                text.push(self.chars[self.pos]);
380                self.advance();
381            }
382        }
383        // Optional exponent (not exercised by the corpus, supported for
384        // completeness of the number surface).
385        if self.pos < self.chars.len()
386            && (self.chars[self.pos] == 'e' || self.chars[self.pos] == 'E')
387        {
388            let mut lookahead = self.pos + 1;
389            if lookahead < self.chars.len()
390                && (self.chars[lookahead] == '+' || self.chars[lookahead] == '-')
391            {
392                lookahead += 1;
393            }
394            if lookahead < self.chars.len() && self.chars[lookahead].is_ascii_digit() {
395                text.push('e');
396                self.advance();
397                if self.pos < self.chars.len()
398                    && (self.chars[self.pos] == '+' || self.chars[self.pos] == '-')
399                {
400                    text.push(self.chars[self.pos]);
401                    self.advance();
402                }
403                while self.pos < self.chars.len() && self.chars[self.pos].is_ascii_digit() {
404                    text.push(self.chars[self.pos]);
405                    self.advance();
406                }
407            }
408        }
409        let end = self.here(0);
410        self.tokens.push(Token::new(
411            TokenKind::Number,
412            text,
413            Span::new(self.file_id, start.start, end.start),
414        ));
415        Ok(())
416    }
417
418    fn lex_ident(&mut self) {
419        let start = self.here(1);
420        let mut text = String::new();
421        while self.pos < self.chars.len() && is_ident_continue(self.chars[self.pos]) {
422            text.push(self.chars[self.pos]);
423            self.advance();
424        }
425        let end = self.here(0);
426        self.tokens.push(Token::new(
427            TokenKind::Ident,
428            text,
429            Span::new(self.file_id, start.start, end.start),
430        ));
431    }
432
433    fn single(&mut self, kind: TokenKind) {
434        let start = self.here(1);
435        let text = self.chars[self.pos].to_string();
436        self.advance();
437        let end = self.here(0);
438        self.tokens.push(Token::new(
439            kind,
440            text,
441            Span::new(self.file_id, start.start, end.start),
442        ));
443    }
444
445    /// Two-char operator where the second char may be `=`.
446    fn lex_two(&mut self, plain: TokenKind, assign: TokenKind, second: char) {
447        let start = self.here(1);
448        if self.peek(1) == Some(second) {
449            self.advance();
450            let text = format!("{}{}", self.chars[self.pos - 1], second);
451            self.advance();
452            let end = self.here(0);
453            self.tokens.push(Token::new(
454                assign,
455                text,
456                Span::new(self.file_id, start.start, end.start),
457            ));
458        } else {
459            let text = self.chars[self.pos].to_string();
460            self.advance();
461            let end = self.here(0);
462            self.tokens.push(Token::new(
463                plain,
464                text,
465                Span::new(self.file_id, start.start, end.start),
466            ));
467        }
468    }
469
470    fn lex_duplicate(&mut self, kind: TokenKind, text: &str) {
471        let start = self.here(1);
472        self.advance();
473        self.advance();
474        let end = self.here(0);
475        self.tokens.push(Token::new(
476            kind,
477            text,
478            Span::new(self.file_id, start.start, end.start),
479        ));
480    }
481
482    /// Two-char operator with a fixed second char (e.g. `==`, `<=`).
483    fn two(&mut self, plain: TokenKind, combined: TokenKind, second: char) {
484        let start = self.here(1);
485        let text = self.chars[self.pos].to_string();
486        if self.peek(1) == Some(second) {
487            self.advance();
488            let combined_text = format!("{}{}", text, second);
489            self.advance();
490            let end = self.here(0);
491            self.tokens.push(Token::new(
492                combined,
493                combined_text,
494                Span::new(self.file_id, start.start, end.start),
495            ));
496        } else {
497            self.advance();
498            let end = self.here(0);
499            self.tokens.push(Token::new(
500                plain,
501                text,
502                Span::new(self.file_id, start.start, end.start),
503            ));
504        }
505    }
506
507    fn here(&self, width: usize) -> Span {
508        Span::new(
509            self.file_id,
510            Position::new(self.line, self.col),
511            Position::new(self.line, self.col + width as u32),
512        )
513    }
514
515    fn peek(&self, offset: usize) -> Option<char> {
516        self.chars.get(self.pos + offset).copied()
517    }
518
519    fn advance(&mut self) {
520        self.pos += 1;
521        self.col += 1;
522    }
523}
524
525fn is_ident_start(c: char) -> bool {
526    c.is_ascii_alphabetic() || c == '_'
527}
528
529fn is_ident_continue(c: char) -> bool {
530    c.is_ascii_alphanumeric() || c == '_'
531}
532
533#[cfg(test)]
534mod tests {
535    use super::*;
536
537    fn lex_ok(text: &str) -> Vec<Token> {
538        lex(LexInput { file_id: 0, text }).unwrap()
539    }
540
541    #[test]
542    fn lexes_basic_rule() {
543        let tokens = lex_ok("rule \"setup\":\n    @Event global\n    disableInspector()\n");
544        let kinds: Vec<TokenKind> = tokens.iter().map(|t| t.kind).collect();
545        assert!(kinds.contains(&TokenKind::Ident));
546        assert!(kinds.contains(&TokenKind::String));
547        assert!(kinds.contains(&TokenKind::Colon));
548        assert!(kinds.contains(&TokenKind::At));
549        assert!(kinds.contains(&TokenKind::LParen));
550        assert!(kinds.contains(&TokenKind::Eof));
551    }
552
553    #[test]
554    fn numbers_preserve_text() {
555        let tokens = lex_ok("1 2.5 0.016 100");
556        let numbers: Vec<&str> = tokens
557            .iter()
558            .filter(|t| t.kind == TokenKind::Number)
559            .map(|t| t.text.as_str())
560            .collect();
561        assert_eq!(numbers, vec!["1", "2.5", "0.016", "100"]);
562    }
563
564    #[test]
565    fn directives_and_comments() {
566        let tokens = lex_ok("#!define X 1\n# comment\nrule \"r\":\n");
567        let directive = tokens
568            .iter()
569            .find(|t| t.kind == TokenKind::Directive)
570            .unwrap();
571        assert_eq!(directive.text, "define X 1");
572        assert!(!tokens.iter().any(|t| t.text == "comment"));
573    }
574
575    #[test]
576    fn operators() {
577        let tokens = lex_ok("a += b == c <= d != e / f");
578        let kinds: Vec<TokenKind> = tokens.iter().map(|t| t.kind).collect();
579        for expected in [
580            TokenKind::PlusAssign,
581            TokenKind::Eq,
582            TokenKind::Le,
583            TokenKind::Ne,
584            TokenKind::Slash,
585        ] {
586            assert!(
587                kinds.contains(&expected),
588                "missing {expected:?} in {kinds:?}"
589            );
590        }
591    }
592
593    #[test]
594    fn power_operators_disambiguate() {
595        // The pinned OverPy 9.7.10 reference lexes `**=` as one power-assign
596        // operator distinct from `**` and `*=`.
597        let tokens = lex_ok("a **= b ** c *= d");
598        let kinds: Vec<TokenKind> = tokens.iter().map(|t| t.kind).collect();
599        assert_eq!(
600            kinds,
601            vec![
602                TokenKind::Ident,
603                TokenKind::DoubleStarAssign,
604                TokenKind::Ident,
605                TokenKind::DoubleStar,
606                TokenKind::Ident,
607                TokenKind::StarAssign,
608                TokenKind::Ident,
609                TokenKind::Eof,
610            ]
611        );
612        let assign = tokens
613            .iter()
614            .find(|t| t.kind == TokenKind::DoubleStarAssign)
615            .unwrap();
616        assert_eq!(assign.text, "**=");
617    }
618
619    #[test]
620    fn postfix_operators_are_single_tokens() {
621        let tokens = lex_ok("counter++ points--");
622        assert_eq!(
623            tokens.iter().map(|token| token.kind).collect::<Vec<_>>(),
624            vec![
625                TokenKind::Ident,
626                TokenKind::Increment,
627                TokenKind::Ident,
628                TokenKind::Decrement,
629                TokenKind::Eof,
630            ]
631        );
632        assert_eq!(tokens[1].span.start.col, 8);
633        assert_eq!(tokens[1].span.end.col, 10);
634    }
635
636    #[test]
637    fn unterminated_string_is_structured() {
638        let error = lex(LexInput {
639            file_id: 0,
640            text: "rule \"x\n",
641        })
642        .unwrap_err();
643        assert_eq!(error.code, "lex-error");
644        assert!(error.span.is_some());
645    }
646
647    #[test]
648    fn backslash_line_continuation_is_not_a_token() {
649        let tokens = lex_ok("one \\\ntwo");
650        assert_eq!(
651            tokens.iter().map(|token| token.kind).collect::<Vec<_>>(),
652            vec![TokenKind::Ident, TokenKind::Ident, TokenKind::Eof]
653        );
654        assert_eq!(tokens[1].span.start.line, 2);
655        assert_eq!(tokens[1].span.start.col, 1);
656    }
657
658    #[test]
659    fn crlf_line_continuation_tracks_the_next_line() {
660        let tokens = lex_ok("one \\\r\ntwo");
661        assert_eq!(tokens[1].span.start, Position::new(2, 1));
662    }
663
664    #[test]
665    fn whitespace_before_line_ending_is_part_of_the_continuation() {
666        let tokens = lex_ok("one \\  \ntwo");
667        assert_eq!(tokens[1].span.start, Position::new(2, 1));
668    }
669
670    #[test]
671    fn non_newline_backslash_remains_a_lex_error() {
672        for text in ["one \\ two", "one \\", "one \\ \t\ntwo"] {
673            let error = lex(LexInput { file_id: 0, text }).unwrap_err();
674            assert_eq!(error.code, "lex-error");
675            assert_eq!(error.message, "unexpected character '\\'");
676            assert_eq!(error.span.unwrap().start, Position::new(1, 5));
677        }
678    }
679}
opy_rs/lexer.rs

opy_rs/
lexer.rs
