opy_rs/
lexer.rs

1//! The native `.opy` lexer.
2//!
3//! Produces a flat token stream (newlines and indentation included) from one
4//! source file. Comments (`#`, `/* */`) are skipped; `#!` directives are
5//! captured as a single directive token for the preprocessor. Positions are
6//! 1-based line/column, matching the Opy HIR protocol.
7
8use crate::diag::{OpyError, OpyResult, Position, Span};
9
10/// The kind of a token.
11#[derive(Debug, Clone, Copy, PartialEq, Eq)]
12pub enum TokenKind {
13    /// An identifier or keyword (keywords are resolved by the parser).
14    Ident,
15    /// A numeric literal (`text` holds the source spelling).
16    Number,
17    /// A string literal (`text` holds the unescaped value).
18    String,
19    /// A `#!` directive line (`text` holds everything after `#!`).
20    Directive,
21    /// A preprocessing marker carrying the rule-prefix state active at the
22    /// following top-level rule or subroutine.
23    RulePrefixMarker,
24    /// `@Event` / `@Condition` / other `@` directives.
25    At,
26    Newline,
27    /// Indentation change: the column of the current line.
28    Indent(u32),
29    /// End of file.
30    Eof,
31    // Punctuation and operators.
32    LParen,
33    RParen,
34    LBracket,
35    RBracket,
36    LBrace,
37    RBrace,
38    Comma,
39    Colon,
40    Dot,
41    Assign,
42    Plus,
43    Minus,
44    Star,
45    Slash,
46    Percent,
47    DoubleStar,
48    PlusAssign,
49    MinusAssign,
50    StarAssign,
51    SlashAssign,
52    PercentAssign,
53    DoubleStarAssign,
54    Eq,
55    Ne,
56    Lt,
57    Le,
58    Gt,
59    Ge,
60    /// A bare `!` that is not `!=`; the parser rejects it as unsupported OPY.
61    LexBang,
62}
63
64/// One token with its source span and payload text.
65#[derive(Debug, Clone, PartialEq)]
66pub struct Token {
67    pub kind: TokenKind,
68    /// The source text of this token (numbers keep their spelling; strings
69    /// keep their unescaped value; identifiers keep their name).
70    pub text: String,
71    /// The exact characters between string quotes, before escape decoding.
72    /// Other token kinds leave this unset. It is retained so source-language
73    /// constructs such as f-string interpolations can recover expression
74    /// spans without losing provenance during preprocessing.
75    pub raw: Option<String>,
76    pub span: Span,
77}
78
79impl Token {
80    fn new(kind: TokenKind, text: impl Into<String>, span: Span) -> Token {
81        Token {
82            kind,
83            text: text.into(),
84            raw: None,
85            span,
86        }
87    }
88}
89
90/// The lexer input: one file's text with its file id.
91pub struct LexInput<'a> {
92    pub file_id: u32,
93    pub text: &'a str,
94}
95
96/// Lex one source file into a token stream.
97pub fn lex(input: LexInput<'_>) -> OpyResult<Vec<Token>> {
98    Lexer::new(input.file_id, input.text).run()
99}
100
101struct Lexer {
102    file_id: u32,
103    chars: Vec<char>,
104    pos: usize,
105    line: u32,
106    col: u32,
107    tokens: Vec<Token>,
108}
109
110impl Lexer {
111    fn new(file_id: u32, text: &str) -> Lexer {
112        Lexer {
113            file_id,
114            chars: text.chars().collect(),
115            pos: 0,
116            line: 1,
117            col: 1,
118            tokens: Vec::new(),
119        }
120    }
121
122    fn run(mut self) -> OpyResult<Vec<Token>> {
123        while self.pos < self.chars.len() {
124            let ch = self.chars[self.pos];
125            match ch {
126                '\n' => {
127                    self.tokens
128                        .push(Token::new(TokenKind::Newline, "\n", self.here(1)));
129                    self.advance();
130                    self.line += 1;
131                    self.col = 1;
132                }
133                ' ' | '\t' | '\r' => {
134                    self.advance();
135                }
136                '#' => self.lex_hash()?,
137                '/' if self.peek(1) == Some('*') => self.skip_block_comment()?,
138                '"' | '\'' => self.lex_string(ch)?,
139                c if c.is_ascii_digit() => self.lex_number()?,
140                c if is_ident_start(c) => self.lex_ident(),
141                '(' => self.single(TokenKind::LParen),
142                ')' => self.single(TokenKind::RParen),
143                '[' => self.single(TokenKind::LBracket),
144                ']' => self.single(TokenKind::RBracket),
145                '{' => self.single(TokenKind::LBrace),
146                '}' => self.single(TokenKind::RBrace),
147                ',' => self.single(TokenKind::Comma),
148                ':' => self.single(TokenKind::Colon),
149                '.' => self.single(TokenKind::Dot),
150                '@' => self.single(TokenKind::At),
151                '=' => self.two(TokenKind::Assign, TokenKind::Eq, '='),
152                '+' => self.lex_two(TokenKind::Plus, TokenKind::PlusAssign, '='),
153                '-' => self.lex_two(TokenKind::Minus, TokenKind::MinusAssign, '='),
154                '*' => {
155                    if self.peek(1) == Some('*') {
156                        if self.peek(2) == Some('=') {
157                            let start = self.here(3);
158                            self.advance();
159                            self.advance();
160                            self.advance();
161                            let end = self.here(0);
162                            self.tokens.push(Token::new(
163                                TokenKind::DoubleStarAssign,
164                                "**=",
165                                Span::new(self.file_id, start.start, end.start),
166                            ));
167                        } else {
168                            self.advance();
169                            self.single(TokenKind::DoubleStar)
170                        }
171                    } else {
172                        self.lex_two(TokenKind::Star, TokenKind::StarAssign, '=')
173                    }
174                }
175                '/' => self.lex_two(TokenKind::Slash, TokenKind::SlashAssign, '='),
176                '%' => self.lex_two(TokenKind::Percent, TokenKind::PercentAssign, '='),
177                '<' => self.two(TokenKind::Lt, TokenKind::Le, '='),
178                '>' => self.two(TokenKind::Gt, TokenKind::Ge, '='),
179                '!' => self.two(TokenKind::LexBang, TokenKind::Ne, '='),
180                other => {
181                    return Err(OpyError::at(
182                        "lex-error",
183                        format!("unexpected character '{other}'"),
184                        self.here(1),
185                    ));
186                }
187            }
188        }
189        let here = self.here(0);
190        self.tokens.push(Token::new(TokenKind::Eof, "", here));
191        Ok(self.tokens)
192    }
193
194    /// `#` starts a `#!` directive (captured as one token) or a comment.
195    fn lex_hash(&mut self) -> OpyResult<()> {
196        if self.peek(1) == Some('!') {
197            let start = self.here(2);
198            self.advance();
199            self.advance();
200            let mut text = String::new();
201            while self.pos < self.chars.len() && self.chars[self.pos] != '\n' {
202                text.push(self.chars[self.pos]);
203                self.advance();
204            }
205            let end = self.here(0);
206            self.tokens.push(Token::new(
207                TokenKind::Directive,
208                text,
209                Span::new(self.file_id, start.start, end.start),
210            ));
211        } else {
212            while self.pos < self.chars.len() && self.chars[self.pos] != '\n' {
213                self.advance();
214            }
215        }
216        Ok(())
217    }
218
219    fn skip_block_comment(&mut self) -> OpyResult<()> {
220        let start = self.here(2);
221        self.advance();
222        self.advance();
223        while self.pos < self.chars.len() {
224            if self.chars[self.pos] == '*' && self.peek(1) == Some('/') {
225                self.advance();
226                self.advance();
227                return Ok(());
228            }
229            if self.chars[self.pos] == '\n' {
230                self.advance();
231                self.line += 1;
232                self.col = 1;
233            } else {
234                self.advance();
235            }
236        }
237        Err(OpyError::at(
238            "lex-error",
239            "unterminated block comment",
240            start,
241        ))
242    }
243
244    fn lex_string(&mut self, quote: char) -> OpyResult<()> {
245        let start = self.here(1);
246        self.advance();
247        let mut value = String::new();
248        let mut raw = String::new();
249        while self.pos < self.chars.len() {
250            let ch = self.chars[self.pos];
251            if ch == quote {
252                self.advance();
253                let end = self.here(0);
254                let mut token = Token::new(
255                    TokenKind::String,
256                    value,
257                    Span::new(self.file_id, start.start, end.start),
258                );
259                token.raw = Some(raw);
260                self.tokens.push(token);
261                return Ok(());
262            }
263            if ch == '\\' {
264                raw.push(ch);
265                self.advance();
266                if self.pos >= self.chars.len() {
267                    break;
268                }
269                let escaped = self.chars[self.pos];
270                raw.push(escaped);
271                value.push(match escaped {
272                    'n' => '\n',
273                    't' => '\t',
274                    'r' => '\r',
275                    '\\' => '\\',
276                    '"' => '"',
277                    '\'' => '\'',
278                    other => other,
279                });
280                self.advance();
281                continue;
282            }
283            if ch == '\n' {
284                return Err(OpyError::at(
285                    "lex-error",
286                    "unterminated string literal",
287                    start,
288                ));
289            }
290            raw.push(ch);
291            value.push(ch);
292            self.advance();
293        }
294        Err(OpyError::at(
295            "lex-error",
296            "unterminated string literal",
297            start,
298        ))
299    }
300
301    fn lex_number(&mut self) -> OpyResult<()> {
302        let start = self.here(1);
303        let mut text = String::new();
304        if self.chars[self.pos] == '0' && matches!(self.peek(1), Some('x' | 'X')) {
305            text.push('0');
306            self.advance();
307            text.push(self.chars[self.pos]);
308            self.advance();
309            let digits_start = self.pos;
310            while self.pos < self.chars.len() && self.chars[self.pos].is_ascii_hexdigit() {
311                text.push(self.chars[self.pos]);
312                self.advance();
313            }
314            if self.pos == digits_start {
315                return Err(OpyError::at(
316                    "lex-error",
317                    "hexadecimal literal requires at least one hexadecimal digit",
318                    Span::new(self.file_id, start.start, self.here(0).start),
319                ));
320            }
321            let end = self.here(0);
322            self.tokens.push(Token::new(
323                TokenKind::Number,
324                text,
325                Span::new(self.file_id, start.start, end.start),
326            ));
327            return Ok(());
328        }
329        while self.pos < self.chars.len() && self.chars[self.pos].is_ascii_digit() {
330            text.push(self.chars[self.pos]);
331            self.advance();
332        }
333        if self.pos < self.chars.len()
334            && self.chars[self.pos] == '.'
335            && self.peek(1).is_some_and(|c| c.is_ascii_digit())
336        {
337            text.push('.');
338            self.advance();
339            while self.pos < self.chars.len() && self.chars[self.pos].is_ascii_digit() {
340                text.push(self.chars[self.pos]);
341                self.advance();
342            }
343        }
344        // Optional exponent (not exercised by the corpus, supported for
345        // completeness of the number surface).
346        if self.pos < self.chars.len()
347            && (self.chars[self.pos] == 'e' || self.chars[self.pos] == 'E')
348        {
349            let mut lookahead = self.pos + 1;
350            if lookahead < self.chars.len()
351                && (self.chars[lookahead] == '+' || self.chars[lookahead] == '-')
352            {
353                lookahead += 1;
354            }
355            if lookahead < self.chars.len() && self.chars[lookahead].is_ascii_digit() {
356                text.push('e');
357                self.advance();
358                if self.pos < self.chars.len()
359                    && (self.chars[self.pos] == '+' || self.chars[self.pos] == '-')
360                {
361                    text.push(self.chars[self.pos]);
362                    self.advance();
363                }
364                while self.pos < self.chars.len() && self.chars[self.pos].is_ascii_digit() {
365                    text.push(self.chars[self.pos]);
366                    self.advance();
367                }
368            }
369        }
370        let end = self.here(0);
371        self.tokens.push(Token::new(
372            TokenKind::Number,
373            text,
374            Span::new(self.file_id, start.start, end.start),
375        ));
376        Ok(())
377    }
378
379    fn lex_ident(&mut self) {
380        let start = self.here(1);
381        let mut text = String::new();
382        while self.pos < self.chars.len() && is_ident_continue(self.chars[self.pos]) {
383            text.push(self.chars[self.pos]);
384            self.advance();
385        }
386        let end = self.here(0);
387        self.tokens.push(Token::new(
388            TokenKind::Ident,
389            text,
390            Span::new(self.file_id, start.start, end.start),
391        ));
392    }
393
394    fn single(&mut self, kind: TokenKind) {
395        let start = self.here(1);
396        let text = self.chars[self.pos].to_string();
397        self.advance();
398        let end = self.here(0);
399        self.tokens.push(Token::new(
400            kind,
401            text,
402            Span::new(self.file_id, start.start, end.start),
403        ));
404    }
405
406    /// Two-char operator where the second char may be `=`.
407    fn lex_two(&mut self, plain: TokenKind, assign: TokenKind, second: char) {
408        let start = self.here(1);
409        if self.peek(1) == Some(second) {
410            self.advance();
411            let text = format!("{}{}", self.chars[self.pos - 1], second);
412            self.advance();
413            let end = self.here(0);
414            self.tokens.push(Token::new(
415                assign,
416                text,
417                Span::new(self.file_id, start.start, end.start),
418            ));
419        } else {
420            let text = self.chars[self.pos].to_string();
421            self.advance();
422            let end = self.here(0);
423            self.tokens.push(Token::new(
424                plain,
425                text,
426                Span::new(self.file_id, start.start, end.start),
427            ));
428        }
429    }
430
431    /// Two-char operator with a fixed second char (e.g. `==`, `<=`).
432    fn two(&mut self, plain: TokenKind, combined: TokenKind, second: char) {
433        let start = self.here(1);
434        let text = self.chars[self.pos].to_string();
435        if self.peek(1) == Some(second) {
436            self.advance();
437            let combined_text = format!("{}{}", text, second);
438            self.advance();
439            let end = self.here(0);
440            self.tokens.push(Token::new(
441                combined,
442                combined_text,
443                Span::new(self.file_id, start.start, end.start),
444            ));
445        } else {
446            self.advance();
447            let end = self.here(0);
448            self.tokens.push(Token::new(
449                plain,
450                text,
451                Span::new(self.file_id, start.start, end.start),
452            ));
453        }
454    }
455
456    fn here(&self, width: usize) -> Span {
457        Span::new(
458            self.file_id,
459            Position::new(self.line, self.col),
460            Position::new(self.line, self.col + width as u32),
461        )
462    }
463
464    fn peek(&self, offset: usize) -> Option<char> {
465        self.chars.get(self.pos + offset).copied()
466    }
467
468    fn advance(&mut self) {
469        self.pos += 1;
470        self.col += 1;
471    }
472}
473
474fn is_ident_start(c: char) -> bool {
475    c.is_ascii_alphabetic() || c == '_'
476}
477
478fn is_ident_continue(c: char) -> bool {
479    c.is_ascii_alphanumeric() || c == '_'
480}
481
482#[cfg(test)]
483mod tests {
484    use super::*;
485
486    fn lex_ok(text: &str) -> Vec<Token> {
487        lex(LexInput { file_id: 0, text }).unwrap()
488    }
489
490    #[test]
491    fn lexes_basic_rule() {
492        let tokens = lex_ok("rule \"setup\":\n    @Event global\n    disableInspector()\n");
493        let kinds: Vec<TokenKind> = tokens.iter().map(|t| t.kind).collect();
494        assert!(kinds.contains(&TokenKind::Ident));
495        assert!(kinds.contains(&TokenKind::String));
496        assert!(kinds.contains(&TokenKind::Colon));
497        assert!(kinds.contains(&TokenKind::At));
498        assert!(kinds.contains(&TokenKind::LParen));
499        assert!(kinds.contains(&TokenKind::Eof));
500    }
501
502    #[test]
503    fn numbers_preserve_text() {
504        let tokens = lex_ok("1 2.5 0.016 100");
505        let numbers: Vec<&str> = tokens
506            .iter()
507            .filter(|t| t.kind == TokenKind::Number)
508            .map(|t| t.text.as_str())
509            .collect();
510        assert_eq!(numbers, vec!["1", "2.5", "0.016", "100"]);
511    }
512
513    #[test]
514    fn directives_and_comments() {
515        let tokens = lex_ok("#!define X 1\n# comment\nrule \"r\":\n");
516        let directive = tokens
517            .iter()
518            .find(|t| t.kind == TokenKind::Directive)
519            .unwrap();
520        assert_eq!(directive.text, "define X 1");
521        assert!(!tokens.iter().any(|t| t.text == "comment"));
522    }
523
524    #[test]
525    fn operators() {
526        let tokens = lex_ok("a += b == c <= d != e / f");
527        let kinds: Vec<TokenKind> = tokens.iter().map(|t| t.kind).collect();
528        for expected in [
529            TokenKind::PlusAssign,
530            TokenKind::Eq,
531            TokenKind::Le,
532            TokenKind::Ne,
533            TokenKind::Slash,
534        ] {
535            assert!(
536                kinds.contains(&expected),
537                "missing {expected:?} in {kinds:?}"
538            );
539        }
540    }
541
542    #[test]
543    fn power_operators_disambiguate() {
544        // The pinned OverPy 9.7.10 reference lexes `**=` as one power-assign
545        // operator distinct from `**` and `*=`.
546        let tokens = lex_ok("a **= b ** c *= d");
547        let kinds: Vec<TokenKind> = tokens.iter().map(|t| t.kind).collect();
548        assert_eq!(
549            kinds,
550            vec![
551                TokenKind::Ident,
552                TokenKind::DoubleStarAssign,
553                TokenKind::Ident,
554                TokenKind::DoubleStar,
555                TokenKind::Ident,
556                TokenKind::StarAssign,
557                TokenKind::Ident,
558                TokenKind::Eof,
559            ]
560        );
561        let assign = tokens
562            .iter()
563            .find(|t| t.kind == TokenKind::DoubleStarAssign)
564            .unwrap();
565        assert_eq!(assign.text, "**=");
566    }
567
568    #[test]
569    fn unterminated_string_is_structured() {
570        let error = lex(LexInput {
571            file_id: 0,
572            text: "rule \"x\n",
573        })
574        .unwrap_err();
575        assert_eq!(error.code, "lex-error");
576        assert!(error.span.is_some());
577    }
578}
opy_rs/lexer.rs

opy_rs/
lexer.rs
