opy_rs/
lexer.rs

1//! The native `.opy` lexer.
2//!
3//! Produces a flat token stream (newlines and indentation included) from one
4//! source file. Comments (`#`, `/* */`) are skipped; `#!` directives are
5//! captured as a single directive token for the preprocessor. Positions are
6//! 1-based line/column, matching the Opy HIR protocol.
7
8use crate::diag::{OpyError, OpyResult, Position, Span};
9
10/// The kind of a token.
11#[derive(Debug, Clone, Copy, PartialEq, Eq)]
12pub enum TokenKind {
13    /// An identifier or keyword (keywords are resolved by the parser).
14    Ident,
15    /// A numeric literal (`text` holds the source spelling).
16    Number,
17    /// A string literal (`text` holds the unescaped value).
18    String,
19    /// A `#!` directive line (`text` holds everything after `#!`).
20    Directive,
21    /// A preprocessing marker carrying the rule-prefix state active at the
22    /// following top-level rule or subroutine.
23    RulePrefixMarker,
24    /// `@Event` / `@Condition` / other `@` directives.
25    At,
26    Newline,
27    /// Indentation change: the column of the current line.
28    Indent(u32),
29    /// End of file.
30    Eof,
31    // Punctuation and operators.
32    LParen,
33    RParen,
34    LBracket,
35    RBracket,
36    LBrace,
37    RBrace,
38    Comma,
39    Colon,
40    Semicolon,
41    Dot,
42    Assign,
43    Plus,
44    Minus,
45    Star,
46    Slash,
47    Percent,
48    DoubleStar,
49    PlusAssign,
50    MinusAssign,
51    Increment,
52    Decrement,
53    StarAssign,
54    SlashAssign,
55    PercentAssign,
56    DoubleStarAssign,
57    Eq,
58    Ne,
59    Lt,
60    Le,
61    Gt,
62    Ge,
63    /// A bare `!` that is not `!=`; the parser rejects it as unsupported OPY.
64    LexBang,
65}
66
67/// One token with its source span and payload text.
68#[derive(Debug, Clone, PartialEq)]
69pub struct Token {
70    pub kind: TokenKind,
71    /// The source text of this token (numbers keep their spelling; strings
72    /// keep their unescaped value; identifiers keep their name).
73    pub text: String,
74    /// The exact characters between string quotes, before escape decoding.
75    /// Other token kinds leave this unset. It is retained so source-language
76    /// constructs such as f-string interpolations can recover expression
77    /// spans without losing provenance during preprocessing.
78    pub raw: Option<String>,
79    pub span: Span,
80}
81
82impl Token {
83    fn new(kind: TokenKind, text: impl Into<String>, span: Span) -> Token {
84        Token {
85            kind,
86            text: text.into(),
87            raw: None,
88            span,
89        }
90    }
91}
92
93/// The lexer input: one file's text with its file id.
94pub struct LexInput<'a> {
95    pub file_id: u32,
96    pub text: &'a str,
97}
98
99/// Lex one source file into a token stream.
100pub fn lex(input: LexInput<'_>) -> OpyResult<Vec<Token>> {
101    Lexer::new(input.file_id, input.text).run()
102}
103
104struct Lexer {
105    file_id: u32,
106    chars: Vec<char>,
107    pos: usize,
108    line: u32,
109    col: u32,
110    tokens: Vec<Token>,
111}
112
113impl Lexer {
114    fn new(file_id: u32, text: &str) -> Lexer {
115        Lexer {
116            file_id,
117            chars: text.chars().collect(),
118            pos: 0,
119            line: 1,
120            col: 1,
121            tokens: Vec::new(),
122        }
123    }
124
125    fn run(mut self) -> OpyResult<Vec<Token>> {
126        while self.pos < self.chars.len() {
127            let ch = self.chars[self.pos];
128            match ch {
129                '\n' => {
130                    self.tokens
131                        .push(Token::new(TokenKind::Newline, "\n", self.here(1)));
132                    self.advance();
133                    self.line += 1;
134                    self.col = 1;
135                }
136                ' ' | '\r' => {
137                    self.advance();
138                }
139                '\t' => {
140                    self.pos += 1;
141                    self.col += 4;
142                }
143                '\\' => {
144                    if !self.skip_line_continuation() {
145                        return Err(OpyError::at(
146                            "lex-error",
147                            "unexpected character '\\'",
148                            self.here(1),
149                        ));
150                    }
151                }
152                '#' => self.lex_hash()?,
153                '/' if self.peek(1) == Some('*') => self.skip_block_comment()?,
154                '"' | '\'' => self.lex_string(ch)?,
155                c if c.is_ascii_digit() => self.lex_number()?,
156                c if is_ident_start(c) => self.lex_ident(),
157                '(' => self.single(TokenKind::LParen),
158                ')' => self.single(TokenKind::RParen),
159                '[' => self.single(TokenKind::LBracket),
160                ']' => self.single(TokenKind::RBracket),
161                '{' => self.single(TokenKind::LBrace),
162                '}' => self.single(TokenKind::RBrace),
163                ',' => self.single(TokenKind::Comma),
164                ':' => self.single(TokenKind::Colon),
165                ';' => self.single(TokenKind::Semicolon),
166                '.' => self.single(TokenKind::Dot),
167                '@' => self.single(TokenKind::At),
168                '=' => self.two(TokenKind::Assign, TokenKind::Eq, '='),
169                '+' => {
170                    if self.peek(1) == Some('+') {
171                        self.lex_duplicate(TokenKind::Increment, "++");
172                    } else {
173                        self.lex_two(TokenKind::Plus, TokenKind::PlusAssign, '=');
174                    }
175                }
176                '-' => {
177                    if self.peek(1) == Some('-') {
178                        self.lex_duplicate(TokenKind::Decrement, "--");
179                    } else {
180                        self.lex_two(TokenKind::Minus, TokenKind::MinusAssign, '=');
181                    }
182                }
183                '*' => {
184                    if self.peek(1) == Some('*') {
185                        if self.peek(2) == Some('=') {
186                            let start = self.here(3);
187                            self.advance();
188                            self.advance();
189                            self.advance();
190                            let end = self.here(0);
191                            self.tokens.push(Token::new(
192                                TokenKind::DoubleStarAssign,
193                                "**=",
194                                Span::new(self.file_id, start.start, end.start),
195                            ));
196                        } else {
197                            self.advance();
198                            self.single(TokenKind::DoubleStar)
199                        }
200                    } else {
201                        self.lex_two(TokenKind::Star, TokenKind::StarAssign, '=')
202                    }
203                }
204                '/' => self.lex_two(TokenKind::Slash, TokenKind::SlashAssign, '='),
205                '%' => self.lex_two(TokenKind::Percent, TokenKind::PercentAssign, '='),
206                '<' => self.two(TokenKind::Lt, TokenKind::Le, '='),
207                '>' => self.two(TokenKind::Gt, TokenKind::Ge, '='),
208                '!' => self.two(TokenKind::LexBang, TokenKind::Ne, '='),
209                other => {
210                    return Err(OpyError::at(
211                        "lex-error",
212                        format!("unexpected character '{other}'"),
213                        self.here(1),
214                    ));
215                }
216            }
217        }
218        let here = self.here(0);
219        self.tokens.push(Token::new(TokenKind::Eof, "", here));
220        Ok(self.tokens)
221    }
222
223    /// `#` starts a `#!` directive (captured as one token) or a comment.
224    fn lex_hash(&mut self) -> OpyResult<()> {
225        if self.peek(1) == Some('!') {
226            let start = self.here(2);
227            self.advance();
228            self.advance();
229            let mut text = String::new();
230            while self.pos < self.chars.len() {
231                if self.chars[self.pos] == '\\' && self.skip_line_continuation() {
232                    text.push('\n');
233                    continue;
234                }
235                if self.chars[self.pos] == '\n' {
236                    break;
237                }
238                text.push(self.chars[self.pos]);
239                self.advance();
240            }
241            let end = self.here(0);
242            self.tokens.push(Token::new(
243                TokenKind::Directive,
244                text,
245                Span::new(self.file_id, start.start, end.start),
246            ));
247        } else {
248            while self.pos < self.chars.len() && self.chars[self.pos] != '\n' {
249                self.advance();
250            }
251        }
252        Ok(())
253    }
254
255    fn skip_block_comment(&mut self) -> OpyResult<()> {
256        let start = self.here(2);
257        self.advance();
258        self.advance();
259        while self.pos < self.chars.len() {
260            if self.chars[self.pos] == '*' && self.peek(1) == Some('/') {
261                self.advance();
262                self.advance();
263                return Ok(());
264            }
265            if self.chars[self.pos] == '\n' {
266                self.advance();
267                self.line += 1;
268                self.col = 1;
269            } else {
270                self.advance();
271            }
272        }
273        Err(OpyError::at(
274            "lex-error",
275            "unterminated block comment",
276            start,
277        ))
278    }
279
280    fn lex_string(&mut self, quote: char) -> OpyResult<()> {
281        let start = self.here(1);
282        self.advance();
283        let mut value = String::new();
284        let mut raw = String::new();
285        while self.pos < self.chars.len() {
286            let ch = self.chars[self.pos];
287            if ch == quote {
288                self.advance();
289                let end = self.here(0);
290                let mut token = Token::new(
291                    TokenKind::String,
292                    value,
293                    Span::new(self.file_id, start.start, end.start),
294                );
295                token.raw = Some(raw);
296                self.tokens.push(token);
297                return Ok(());
298            }
299            if ch == '\\' {
300                let escape_start = self.here(1);
301                raw.push(ch);
302                self.advance();
303                if self.pos >= self.chars.len() {
304                    break;
305                }
306                let escaped = self.chars[self.pos];
307                raw.push(escaped);
308                if escaped == 'u' {
309                    self.advance();
310                    let mut codepoint = 0_u32;
311                    for _ in 0..4 {
312                        let Some(digit) = self.chars.get(self.pos).copied() else {
313                            return Err(OpyError::at(
314                                "lex-error",
315                                "Unicode escape requires four hexadecimal digits",
316                                Span::new(self.file_id, escape_start.start, self.here(0).start),
317                            ));
318                        };
319                        let Some(digit_value) = digit.to_digit(16) else {
320                            return Err(OpyError::at(
321                                "lex-error",
322                                "Unicode escape requires four hexadecimal digits",
323                                Span::new(self.file_id, escape_start.start, self.here(1).end),
324                            ));
325                        };
326                        raw.push(digit);
327                        codepoint = codepoint * 16 + digit_value;
328                        self.advance();
329                    }
330                    let Some(decoded) = char::from_u32(codepoint) else {
331                        return Err(OpyError::at(
332                            "lex-error",
333                            "Unicode escape does not name a Unicode scalar value",
334                            Span::new(self.file_id, escape_start.start, self.here(0).start),
335                        ));
336                    };
337                    value.push(decoded);
338                    continue;
339                }
340                value.push(match escaped {
341                    'n' => '\n',
342                    't' => '\t',
343                    'r' => '\r',
344                    '\\' => '\\',
345                    '"' => '"',
346                    '\'' => '\'',
347                    other => other,
348                });
349                self.advance();
350                continue;
351            }
352            if ch == '\n' {
353                return Err(OpyError::at(
354                    "lex-error",
355                    "unterminated string literal",
356                    start,
357                ));
358            }
359            raw.push(ch);
360            value.push(ch);
361            self.advance();
362        }
363        Err(OpyError::at(
364            "lex-error",
365            "unterminated string literal",
366            start,
367        ))
368    }
369
370    fn skip_line_continuation(&mut self) -> bool {
371        let mut offset = 1;
372        while matches!(self.peek(offset), Some(' ' | '\r')) {
373            offset += 1;
374        }
375        if self.peek(offset) != Some('\n') {
376            return false;
377        }
378        for _ in 0..=offset {
379            self.advance();
380        }
381        self.line += 1;
382        self.col = 1;
383        true
384    }
385
386    fn lex_number(&mut self) -> OpyResult<()> {
387        let start = self.here(1);
388        let mut text = String::new();
389        if self.chars[self.pos] == '0' && matches!(self.peek(1), Some('x' | 'X')) {
390            text.push('0');
391            self.advance();
392            text.push(self.chars[self.pos]);
393            self.advance();
394            let digits_start = self.pos;
395            while self.pos < self.chars.len() && self.chars[self.pos].is_ascii_hexdigit() {
396                text.push(self.chars[self.pos]);
397                self.advance();
398            }
399            if self.pos == digits_start {
400                return Err(OpyError::at(
401                    "lex-error",
402                    "hexadecimal literal requires at least one hexadecimal digit",
403                    Span::new(self.file_id, start.start, self.here(0).start),
404                ));
405            }
406            let end = self.here(0);
407            self.tokens.push(Token::new(
408                TokenKind::Number,
409                text,
410                Span::new(self.file_id, start.start, end.start),
411            ));
412            return Ok(());
413        }
414        while self.pos < self.chars.len() && self.chars[self.pos].is_ascii_digit() {
415            text.push(self.chars[self.pos]);
416            self.advance();
417        }
418        if self.pos < self.chars.len()
419            && self.chars[self.pos] == '.'
420            && self.peek(1).is_some_and(|c| c.is_ascii_digit())
421        {
422            text.push('.');
423            self.advance();
424            while self.pos < self.chars.len() && self.chars[self.pos].is_ascii_digit() {
425                text.push(self.chars[self.pos]);
426                self.advance();
427            }
428        }
429        // Optional exponent (not exercised by the corpus, supported for
430        // completeness of the number surface).
431        if self.pos < self.chars.len()
432            && (self.chars[self.pos] == 'e' || self.chars[self.pos] == 'E')
433        {
434            let mut lookahead = self.pos + 1;
435            if lookahead < self.chars.len()
436                && (self.chars[lookahead] == '+' || self.chars[lookahead] == '-')
437            {
438                lookahead += 1;
439            }
440            if lookahead < self.chars.len() && self.chars[lookahead].is_ascii_digit() {
441                text.push('e');
442                self.advance();
443                if self.pos < self.chars.len()
444                    && (self.chars[self.pos] == '+' || self.chars[self.pos] == '-')
445                {
446                    text.push(self.chars[self.pos]);
447                    self.advance();
448                }
449                while self.pos < self.chars.len() && self.chars[self.pos].is_ascii_digit() {
450                    text.push(self.chars[self.pos]);
451                    self.advance();
452                }
453            }
454        }
455        let end = self.here(0);
456        self.tokens.push(Token::new(
457            TokenKind::Number,
458            text,
459            Span::new(self.file_id, start.start, end.start),
460        ));
461        Ok(())
462    }
463
464    fn lex_ident(&mut self) {
465        let start = self.here(1);
466        let mut text = String::new();
467        while self.pos < self.chars.len() && is_ident_continue(self.chars[self.pos]) {
468            text.push(self.chars[self.pos]);
469            self.advance();
470        }
471        let end = self.here(0);
472        self.tokens.push(Token::new(
473            TokenKind::Ident,
474            text,
475            Span::new(self.file_id, start.start, end.start),
476        ));
477    }
478
479    fn single(&mut self, kind: TokenKind) {
480        let start = self.here(1);
481        let text = self.chars[self.pos].to_string();
482        self.advance();
483        let end = self.here(0);
484        self.tokens.push(Token::new(
485            kind,
486            text,
487            Span::new(self.file_id, start.start, end.start),
488        ));
489    }
490
491    /// Two-char operator where the second char may be `=`.
492    fn lex_two(&mut self, plain: TokenKind, assign: TokenKind, second: char) {
493        let start = self.here(1);
494        if self.peek(1) == Some(second) {
495            self.advance();
496            let text = format!("{}{}", self.chars[self.pos - 1], second);
497            self.advance();
498            let end = self.here(0);
499            self.tokens.push(Token::new(
500                assign,
501                text,
502                Span::new(self.file_id, start.start, end.start),
503            ));
504        } else {
505            let text = self.chars[self.pos].to_string();
506            self.advance();
507            let end = self.here(0);
508            self.tokens.push(Token::new(
509                plain,
510                text,
511                Span::new(self.file_id, start.start, end.start),
512            ));
513        }
514    }
515
516    fn lex_duplicate(&mut self, kind: TokenKind, text: &str) {
517        let start = self.here(1);
518        self.advance();
519        self.advance();
520        let end = self.here(0);
521        self.tokens.push(Token::new(
522            kind,
523            text,
524            Span::new(self.file_id, start.start, end.start),
525        ));
526    }
527
528    /// Two-char operator with a fixed second char (e.g. `==`, `<=`).
529    fn two(&mut self, plain: TokenKind, combined: TokenKind, second: char) {
530        let start = self.here(1);
531        let text = self.chars[self.pos].to_string();
532        if self.peek(1) == Some(second) {
533            self.advance();
534            let combined_text = format!("{}{}", text, second);
535            self.advance();
536            let end = self.here(0);
537            self.tokens.push(Token::new(
538                combined,
539                combined_text,
540                Span::new(self.file_id, start.start, end.start),
541            ));
542        } else {
543            self.advance();
544            let end = self.here(0);
545            self.tokens.push(Token::new(
546                plain,
547                text,
548                Span::new(self.file_id, start.start, end.start),
549            ));
550        }
551    }
552
553    fn here(&self, width: usize) -> Span {
554        Span::new(
555            self.file_id,
556            Position::new(self.line, self.col),
557            Position::new(self.line, self.col + width as u32),
558        )
559    }
560
561    fn peek(&self, offset: usize) -> Option<char> {
562        self.chars.get(self.pos + offset).copied()
563    }
564
565    fn advance(&mut self) {
566        self.pos += 1;
567        self.col += 1;
568    }
569}
570
571fn is_ident_start(c: char) -> bool {
572    c.is_ascii_alphabetic() || c == '_'
573}
574
575fn is_ident_continue(c: char) -> bool {
576    c.is_ascii_alphanumeric() || c == '_'
577}
578
579#[cfg(test)]
580mod tests {
581    use super::*;
582
583    fn lex_ok(text: &str) -> Vec<Token> {
584        lex(LexInput { file_id: 0, text }).unwrap()
585    }
586
587    #[test]
588    fn lexes_basic_rule() {
589        let tokens = lex_ok("rule \"setup\":\n    @Event global\n    disableInspector()\n");
590        let kinds: Vec<TokenKind> = tokens.iter().map(|t| t.kind).collect();
591        assert!(kinds.contains(&TokenKind::Ident));
592        assert!(kinds.contains(&TokenKind::String));
593        assert!(kinds.contains(&TokenKind::Colon));
594        assert!(kinds.contains(&TokenKind::At));
595        assert!(kinds.contains(&TokenKind::LParen));
596        assert!(kinds.contains(&TokenKind::Eof));
597    }
598
599    #[test]
600    fn numbers_preserve_text() {
601        let tokens = lex_ok("1 2.5 0.016 100");
602        let numbers: Vec<&str> = tokens
603            .iter()
604            .filter(|t| t.kind == TokenKind::Number)
605            .map(|t| t.text.as_str())
606            .collect();
607        assert_eq!(numbers, vec!["1", "2.5", "0.016", "100"]);
608    }
609
610    #[test]
611    fn directives_and_comments() {
612        let tokens = lex_ok("#!define X 1\n# comment\nrule \"r\":\n");
613        let directive = tokens
614            .iter()
615            .find(|t| t.kind == TokenKind::Directive)
616            .unwrap();
617        assert_eq!(directive.text, "define X 1");
618        assert!(!tokens.iter().any(|t| t.text == "comment"));
619    }
620
621    #[test]
622    fn directive_line_continuation_is_part_of_one_directive() {
623        let tokens = lex_ok("#!define X first + \\\n  second\n");
624        let directive = tokens
625            .iter()
626            .find(|token| token.kind == TokenKind::Directive)
627            .unwrap();
628        assert_eq!(directive.text, "define X first + \n  second");
629        assert_eq!(directive.span.start, Position::new(1, 1));
630        assert_eq!(directive.span.end, Position::new(2, 9));
631    }
632
633    #[test]
634    fn operators() {
635        let tokens = lex_ok("a += b == c <= d != e / f");
636        let kinds: Vec<TokenKind> = tokens.iter().map(|t| t.kind).collect();
637        for expected in [
638            TokenKind::PlusAssign,
639            TokenKind::Eq,
640            TokenKind::Le,
641            TokenKind::Ne,
642            TokenKind::Slash,
643        ] {
644            assert!(
645                kinds.contains(&expected),
646                "missing {expected:?} in {kinds:?}"
647            );
648        }
649    }
650
651    #[test]
652    fn power_operators_disambiguate() {
653        // The pinned OverPy 9.7.10 reference lexes `**=` as one power-assign
654        // operator distinct from `**` and `*=`.
655        let tokens = lex_ok("a **= b ** c *= d");
656        let kinds: Vec<TokenKind> = tokens.iter().map(|t| t.kind).collect();
657        assert_eq!(
658            kinds,
659            vec![
660                TokenKind::Ident,
661                TokenKind::DoubleStarAssign,
662                TokenKind::Ident,
663                TokenKind::DoubleStar,
664                TokenKind::Ident,
665                TokenKind::StarAssign,
666                TokenKind::Ident,
667                TokenKind::Eof,
668            ]
669        );
670        let assign = tokens
671            .iter()
672            .find(|t| t.kind == TokenKind::DoubleStarAssign)
673            .unwrap();
674        assert_eq!(assign.text, "**=");
675    }
676
677    #[test]
678    fn postfix_operators_are_single_tokens() {
679        let tokens = lex_ok("counter++ points--");
680        assert_eq!(
681            tokens.iter().map(|token| token.kind).collect::<Vec<_>>(),
682            vec![
683                TokenKind::Ident,
684                TokenKind::Increment,
685                TokenKind::Ident,
686                TokenKind::Decrement,
687                TokenKind::Eof,
688            ]
689        );
690        assert_eq!(tokens[1].span.start.col, 8);
691        assert_eq!(tokens[1].span.end.col, 10);
692    }
693
694    #[test]
695    fn unterminated_string_is_structured() {
696        let error = lex(LexInput {
697            file_id: 0,
698            text: "rule \"x\n",
699        })
700        .unwrap_err();
701        assert_eq!(error.code, "lex-error");
702        assert!(error.span.is_some());
703    }
704
705    #[test]
706    fn unicode_escapes_decode_at_the_string_boundary_and_keep_source_span() {
707        // Minimized from OWBastion/Bastion commit
708        // 44e12e08f046ed7fbf865237a29ddf2f272ec3c2,
709        // Bastion/src/utilities/system/savePlayerData.opy.
710        let tokens = lex_ok(r#"rule "pa\ufeffssed":"#);
711        let string = tokens
712            .iter()
713            .find(|token| token.kind == TokenKind::String)
714            .expect("rule name string token");
715        assert_eq!(string.text, "pa\u{feff}ssed");
716        assert_eq!(string.raw.as_deref(), Some(r"pa\ufeffssed"));
717        assert_eq!(string.span.start, Position::new(1, 6));
718        assert_eq!(string.span.end, Position::new(1, 20));
719    }
720
721    #[test]
722    fn existing_string_escapes_keep_their_decoded_values() {
723        let source = concat!("\"", "\\n", "\\t", "\\r", "\\\"", "\\\\", "\"");
724        let string = lex_ok(source)
725            .into_iter()
726            .find(|token| token.kind == TokenKind::String)
727            .expect("string token");
728        assert_eq!(string.text, "\n\t\r\"\\");
729    }
730
731    #[test]
732    fn malformed_unicode_escapes_are_lex_errors() {
733        for source in [r#""\u"#, r#""\u12G4"#, r#""\u{1F600}"#] {
734            let error = lex(LexInput {
735                file_id: 0,
736                text: source,
737            })
738            .expect_err("malformed Unicode escape unexpectedly lexed");
739            assert_eq!(error.code, "lex-error");
740            assert!(error.message.contains("Unicode escape"));
741            assert_eq!(error.span.unwrap().start, Position::new(1, 2));
742        }
743    }
744
745    #[test]
746    fn backslash_line_continuation_is_not_a_token() {
747        let tokens = lex_ok("one \\\ntwo");
748        assert_eq!(
749            tokens.iter().map(|token| token.kind).collect::<Vec<_>>(),
750            vec![TokenKind::Ident, TokenKind::Ident, TokenKind::Eof]
751        );
752        assert_eq!(tokens[1].span.start.line, 2);
753        assert_eq!(tokens[1].span.start.col, 1);
754    }
755
756    #[test]
757    fn crlf_line_continuation_tracks_the_next_line() {
758        let tokens = lex_ok("one \\\r\ntwo");
759        assert_eq!(tokens[1].span.start, Position::new(2, 1));
760    }
761
762    #[test]
763    fn whitespace_before_line_ending_is_part_of_the_continuation() {
764        let tokens = lex_ok("one \\  \ntwo");
765        assert_eq!(tokens[1].span.start, Position::new(2, 1));
766    }
767
768    #[test]
769    fn non_newline_backslash_remains_a_lex_error() {
770        for text in ["one \\ two", "one \\", "one \\ \t\ntwo"] {
771            let error = lex(LexInput { file_id: 0, text }).unwrap_err();
772            assert_eq!(error.code, "lex-error");
773            assert_eq!(error.message, "unexpected character '\\'");
774            assert_eq!(error.span.unwrap().start, Position::new(1, 5));
775        }
776    }
777}
opy_rs/lexer.rs

opy_rs/
lexer.rs
