opy_rs/
lexer.rs

1//! The native `.opy` lexer.
2//!
3//! Produces a flat token stream (newlines and indentation included) from one
4//! source file. Comments (`#`, `/* */`) are skipped; `#!` directives are
5//! captured as a single directive token for the preprocessor. Positions are
6//! 1-based line/column, matching the Opy HIR protocol.
7
8use crate::diag::{OpyError, OpyResult, Position, Span};
9
10/// The kind of a token.
11#[derive(Debug, Clone, Copy, PartialEq, Eq)]
12pub enum TokenKind {
13    /// An identifier or keyword (keywords are resolved by the parser).
14    Ident,
15    /// A numeric literal (`text` holds the source spelling).
16    Number,
17    /// A string literal (`text` holds the unescaped value).
18    String,
19    /// A `#!` directive line (`text` holds everything after `#!`).
20    Directive,
21    /// A preprocessing marker carrying the rule-prefix state active at the
22    /// following top-level rule or subroutine.
23    RulePrefixMarker,
24    /// `@Event` / `@Condition` / other `@` directives.
25    At,
26    Newline,
27    /// Indentation change: the column of the current line.
28    Indent(u32),
29    /// End of file.
30    Eof,
31    // Punctuation and operators.
32    LParen,
33    RParen,
34    LBracket,
35    RBracket,
36    LBrace,
37    RBrace,
38    Comma,
39    Colon,
40    Semicolon,
41    Dot,
42    Assign,
43    Plus,
44    Minus,
45    Star,
46    Slash,
47    Percent,
48    DoubleStar,
49    PlusAssign,
50    MinusAssign,
51    Increment,
52    Decrement,
53    StarAssign,
54    SlashAssign,
55    PercentAssign,
56    DoubleStarAssign,
57    Eq,
58    Ne,
59    Lt,
60    Le,
61    Gt,
62    Ge,
63    /// A bare `!` that is not `!=`; the parser rejects it as unsupported OPY.
64    LexBang,
65}
66
67/// One token with its source span and payload text.
68#[derive(Debug, Clone, PartialEq)]
69pub struct Token {
70    pub kind: TokenKind,
71    /// The source text of this token (numbers keep their spelling; strings
72    /// keep their unescaped value; identifiers keep their name).
73    pub text: String,
74    /// The exact characters between string quotes, before escape decoding.
75    /// Other token kinds leave this unset. It is retained so source-language
76    /// constructs such as f-string interpolations can recover expression
77    /// spans without losing provenance during preprocessing.
78    pub raw: Option<String>,
79    pub span: Span,
80}
81
82impl Token {
83    fn new(kind: TokenKind, text: impl Into<String>, span: Span) -> Token {
84        Token {
85            kind,
86            text: text.into(),
87            raw: None,
88            span,
89        }
90    }
91}
92
93/// The lexer input: one file's text with its file id.
94pub struct LexInput<'a> {
95    pub file_id: u32,
96    pub text: &'a str,
97}
98
99/// Lex one source file into a token stream.
100pub fn lex(input: LexInput<'_>) -> OpyResult<Vec<Token>> {
101    Lexer::new(input.file_id, input.text).run()
102}
103
104struct Lexer {
105    file_id: u32,
106    chars: Vec<char>,
107    pos: usize,
108    line: u32,
109    col: u32,
110    tokens: Vec<Token>,
111}
112
113impl Lexer {
114    fn new(file_id: u32, text: &str) -> Lexer {
115        Lexer {
116            file_id,
117            chars: text.chars().collect(),
118            pos: 0,
119            line: 1,
120            col: 1,
121            tokens: Vec::new(),
122        }
123    }
124
125    fn run(mut self) -> OpyResult<Vec<Token>> {
126        while self.pos < self.chars.len() {
127            let ch = self.chars[self.pos];
128            match ch {
129                '\n' => {
130                    self.tokens
131                        .push(Token::new(TokenKind::Newline, "\n", self.here(1)));
132                    self.advance();
133                    self.line += 1;
134                    self.col = 1;
135                }
136                ' ' | '\r' => {
137                    self.advance();
138                }
139                '\t' => {
140                    self.pos += 1;
141                    self.col += 4;
142                }
143                '\\' => {
144                    if !self.skip_line_continuation() {
145                        return Err(OpyError::at(
146                            "lex-error",
147                            "unexpected character '\\'",
148                            self.here(1),
149                        ));
150                    }
151                }
152                '#' => self.lex_hash(),
153                '/' if self.peek(1) == Some('*') => self.skip_block_comment()?,
154                '"' | '\'' => self.lex_string(ch)?,
155                c if c.is_ascii_digit() => self.lex_number()?,
156                c if is_ident_start(c) => self.lex_ident(),
157                '(' => self.single(TokenKind::LParen),
158                ')' => self.single(TokenKind::RParen),
159                '[' => self.single(TokenKind::LBracket),
160                ']' => self.single(TokenKind::RBracket),
161                '{' => self.single(TokenKind::LBrace),
162                '}' => self.single(TokenKind::RBrace),
163                ',' => self.single(TokenKind::Comma),
164                ':' => self.single(TokenKind::Colon),
165                ';' => self.single(TokenKind::Semicolon),
166                '.' => self.single(TokenKind::Dot),
167                '@' => self.single(TokenKind::At),
168                '=' => self.lex_two(TokenKind::Assign, TokenKind::Eq, '='),
169                '+' => {
170                    if self.peek(1) == Some('+') {
171                        self.lex_duplicate(TokenKind::Increment, "++");
172                    } else {
173                        self.lex_two(TokenKind::Plus, TokenKind::PlusAssign, '=');
174                    }
175                }
176                '-' => {
177                    if self.peek(1) == Some('-') {
178                        self.lex_duplicate(TokenKind::Decrement, "--");
179                    } else {
180                        self.lex_two(TokenKind::Minus, TokenKind::MinusAssign, '=');
181                    }
182                }
183                '*' => {
184                    if self.peek(1) == Some('*') {
185                        if self.peek(2) == Some('=') {
186                            let start = self.here(3);
187                            self.advance();
188                            self.advance();
189                            self.advance();
190                            self.push_token(TokenKind::DoubleStarAssign, "**=", start);
191                        } else {
192                            self.advance();
193                            self.single(TokenKind::DoubleStar)
194                        }
195                    } else {
196                        self.lex_two(TokenKind::Star, TokenKind::StarAssign, '=')
197                    }
198                }
199                '/' => self.lex_two(TokenKind::Slash, TokenKind::SlashAssign, '='),
200                '%' => self.lex_two(TokenKind::Percent, TokenKind::PercentAssign, '='),
201                '<' => self.lex_two(TokenKind::Lt, TokenKind::Le, '='),
202                '>' => self.lex_two(TokenKind::Gt, TokenKind::Ge, '='),
203                '!' => self.lex_two(TokenKind::LexBang, TokenKind::Ne, '='),
204                other => {
205                    return Err(OpyError::at(
206                        "lex-error",
207                        format!("unexpected character '{other}'"),
208                        self.here(1),
209                    ));
210                }
211            }
212        }
213        let here = self.here(0);
214        self.push_token(TokenKind::Eof, "", here);
215        Ok(self.tokens)
216    }
217
218    /// `#` starts a `#!` directive (captured as one token) or a comment.
219    fn lex_hash(&mut self) {
220        if self.peek(1) == Some('!') {
221            let start = self.here(2);
222            self.advance();
223            self.advance();
224            let mut text = String::new();
225            while self.pos < self.chars.len() {
226                if self.chars[self.pos] == '\\' && self.skip_line_continuation() {
227                    text.push('\n');
228                    continue;
229                }
230                if self.chars[self.pos] == '\n' {
231                    break;
232                }
233                text.push(self.chars[self.pos]);
234                self.advance();
235            }
236            self.push_token(TokenKind::Directive, text, start);
237        } else {
238            while self.pos < self.chars.len() && self.chars[self.pos] != '\n' {
239                self.advance();
240            }
241        }
242    }
243
244    fn skip_block_comment(&mut self) -> OpyResult<()> {
245        let start = self.here(2);
246        self.advance();
247        self.advance();
248        while self.pos < self.chars.len() {
249            if self.chars[self.pos] == '*' && self.peek(1) == Some('/') {
250                self.advance();
251                self.advance();
252                return Ok(());
253            }
254            if self.chars[self.pos] == '\n' {
255                self.advance();
256                self.line += 1;
257                self.col = 1;
258            } else {
259                self.advance();
260            }
261        }
262        Err(OpyError::at(
263            "lex-error",
264            "unterminated block comment",
265            start,
266        ))
267    }
268
269    fn lex_string(&mut self, quote: char) -> OpyResult<()> {
270        let start = self.here(1);
271        self.advance();
272        let mut value = String::new();
273        let mut raw = String::new();
274        while self.pos < self.chars.len() {
275            let ch = self.chars[self.pos];
276            if ch == quote {
277                self.advance();
278                let mut token = self.make_token(TokenKind::String, value, start);
279                token.raw = Some(raw);
280                self.tokens.push(token);
281                return Ok(());
282            }
283            if ch == '\\' {
284                let escape_start = self.here(1);
285                raw.push(ch);
286                self.advance();
287                if self.pos >= self.chars.len() {
288                    break;
289                }
290                let escaped = self.chars[self.pos];
291                raw.push(escaped);
292                if escaped == '&' {
293                    self.advance();
294                    let mut entity_name = String::new();
295                    while let Some(character) = self.chars.get(self.pos).copied() {
296                        if character == ';' {
297                            break;
298                        }
299                        if !(character.is_ascii_alphanumeric() || character == '_') {
300                            return Err(OpyError::at(
301                                "invalid-string-entity",
302                                format!("invalid character '{character}' in string entity"),
303                                Span::new(self.file_id, escape_start.start, self.here(1).end),
304                            ));
305                        }
306                        entity_name.push(character);
307                        raw.push(character);
308                        self.advance();
309                    }
310                    if self.chars.get(self.pos) != Some(&';') {
311                        return Err(OpyError::at(
312                            "invalid-string-entity",
313                            "expected ';' to terminate string entity",
314                            Span::new(self.file_id, escape_start.start, self.here(0).start),
315                        ));
316                    }
317                    raw.push(';');
318                    self.advance();
319                    let Some(codepoint) = crate::string_entities::codepoint(&entity_name) else {
320                        return Err(OpyError::at(
321                            "unknown-string-entity",
322                            format!("unknown string entity '{entity_name}'"),
323                            Span::new(self.file_id, escape_start.start, self.here(0).start),
324                        ));
325                    };
326                    value.push(codepoint);
327                    continue;
328                }
329                if escaped == 'u' {
330                    self.advance();
331                    let mut codepoint = 0_u32;
332                    for _ in 0..4 {
333                        let Some(digit) = self.chars.get(self.pos).copied() else {
334                            return Err(OpyError::at(
335                                "lex-error",
336                                "Unicode escape requires four hexadecimal digits",
337                                Span::new(self.file_id, escape_start.start, self.here(0).start),
338                            ));
339                        };
340                        let Some(digit_value) = digit.to_digit(16) else {
341                            return Err(OpyError::at(
342                                "lex-error",
343                                "Unicode escape requires four hexadecimal digits",
344                                Span::new(self.file_id, escape_start.start, self.here(1).end),
345                            ));
346                        };
347                        raw.push(digit);
348                        codepoint = codepoint * 16 + digit_value;
349                        self.advance();
350                    }
351                    let Some(decoded) = char::from_u32(codepoint) else {
352                        return Err(OpyError::at(
353                            "lex-error",
354                            "Unicode escape does not name a Unicode scalar value",
355                            Span::new(self.file_id, escape_start.start, self.here(0).start),
356                        ));
357                    };
358                    value.push(decoded);
359                    continue;
360                }
361                value.push(decode_string_escape(escaped));
362                self.advance();
363                continue;
364            }
365            if ch == '\n' {
366                return Err(OpyError::at(
367                    "lex-error",
368                    "unterminated string literal",
369                    start,
370                ));
371            }
372            raw.push(ch);
373            value.push(ch);
374            self.advance();
375        }
376        Err(OpyError::at(
377            "lex-error",
378            "unterminated string literal",
379            start,
380        ))
381    }
382
383    fn skip_line_continuation(&mut self) -> bool {
384        let mut offset = 1;
385        while matches!(self.peek(offset), Some(' ' | '\r')) {
386            offset += 1;
387        }
388        if self.peek(offset) != Some('\n') {
389            return false;
390        }
391        for _ in 0..=offset {
392            self.advance();
393        }
394        self.line += 1;
395        self.col = 1;
396        true
397    }
398
399    fn lex_number(&mut self) -> OpyResult<()> {
400        let start = self.here(1);
401        let mut text = String::new();
402        if self.chars[self.pos] == '0' && matches!(self.peek(1), Some('x' | 'X')) {
403            text.push('0');
404            self.advance();
405            text.push(self.chars[self.pos]);
406            self.advance();
407            let digits_start = self.pos;
408            while self.pos < self.chars.len() && self.chars[self.pos].is_ascii_hexdigit() {
409                text.push(self.chars[self.pos]);
410                self.advance();
411            }
412            if self.pos == digits_start {
413                return Err(OpyError::at(
414                    "lex-error",
415                    "hexadecimal literal requires at least one hexadecimal digit",
416                    Span::new(self.file_id, start.start, self.here(0).start),
417                ));
418            }
419            self.push_token(TokenKind::Number, text, start);
420            return Ok(());
421        }
422        while self.pos < self.chars.len() && self.chars[self.pos].is_ascii_digit() {
423            text.push(self.chars[self.pos]);
424            self.advance();
425        }
426        if self.pos < self.chars.len()
427            && self.chars[self.pos] == '.'
428            && self.peek(1).is_some_and(|c| c.is_ascii_digit())
429        {
430            text.push('.');
431            self.advance();
432            while self.pos < self.chars.len() && self.chars[self.pos].is_ascii_digit() {
433                text.push(self.chars[self.pos]);
434                self.advance();
435            }
436        }
437        // Optional exponent (not exercised by the corpus, supported for
438        // completeness of the number surface).
439        if self.pos < self.chars.len()
440            && (self.chars[self.pos] == 'e' || self.chars[self.pos] == 'E')
441        {
442            let mut lookahead = self.pos + 1;
443            if lookahead < self.chars.len()
444                && (self.chars[lookahead] == '+' || self.chars[lookahead] == '-')
445            {
446                lookahead += 1;
447            }
448            if lookahead < self.chars.len() && self.chars[lookahead].is_ascii_digit() {
449                text.push('e');
450                self.advance();
451                if self.pos < self.chars.len()
452                    && (self.chars[self.pos] == '+' || self.chars[self.pos] == '-')
453                {
454                    text.push(self.chars[self.pos]);
455                    self.advance();
456                }
457                while self.pos < self.chars.len() && self.chars[self.pos].is_ascii_digit() {
458                    text.push(self.chars[self.pos]);
459                    self.advance();
460                }
461            }
462        }
463        self.push_token(TokenKind::Number, text, start);
464        Ok(())
465    }
466
467    fn lex_ident(&mut self) {
468        let start = self.here(1);
469        let mut text = String::new();
470        while self.pos < self.chars.len() && is_ident_continue(self.chars[self.pos]) {
471            text.push(self.chars[self.pos]);
472            self.advance();
473        }
474        self.push_token(TokenKind::Ident, text, start);
475    }
476
477    fn single(&mut self, kind: TokenKind) {
478        let start = self.here(1);
479        let text = self.chars[self.pos].to_string();
480        self.advance();
481        self.push_token(kind, text, start);
482    }
483
484    /// Two-char operator where the second char may be `=`.
485    fn lex_two(&mut self, plain: TokenKind, assign: TokenKind, second: char) {
486        let start = self.here(1);
487        let (kind, text) = if self.peek(1) == Some(second) {
488            self.advance();
489            let text = format!("{}{}", self.chars[self.pos - 1], second);
490            self.advance();
491            (assign, text)
492        } else {
493            let text = self.chars[self.pos].to_string();
494            self.advance();
495            (plain, text)
496        };
497        self.push_token(kind, text, start);
498    }
499
500    fn lex_duplicate(&mut self, kind: TokenKind, text: &str) {
501        let start = self.here(1);
502        self.advance();
503        self.advance();
504        self.push_token(kind, text, start);
505    }
506
507    fn make_token(&self, kind: TokenKind, text: impl Into<String>, start: Span) -> Token {
508        Token::new(
509            kind,
510            text,
511            Span::new(self.file_id, start.start, self.here(0).start),
512        )
513    }
514
515    fn push_token(&mut self, kind: TokenKind, text: impl Into<String>, start: Span) {
516        let token = self.make_token(kind, text, start);
517        self.tokens.push(token);
518    }
519
520    fn here(&self, width: usize) -> Span {
521        Span::new(
522            self.file_id,
523            Position::new(self.line, self.col),
524            Position::new(self.line, self.col + width as u32),
525        )
526    }
527
528    fn peek(&self, offset: usize) -> Option<char> {
529        self.chars.get(self.pos + offset).copied()
530    }
531
532    fn advance(&mut self) {
533        self.pos += 1;
534        self.col += 1;
535    }
536}
537
538/// Whether `text` is a well-formed OPY identifier: an identifier start
539/// character followed by identifier continuation characters.
540pub fn is_identifier(text: &str) -> bool {
541    let mut characters = text.chars();
542    characters.next().is_some_and(is_ident_start) && characters.all(is_ident_continue)
543}
544
545pub(crate) fn decode_string_escape(character: char) -> char {
546    match character {
547        'n' => '\n',
548        't' => '\t',
549        'r' => '\r',
550        '\\' => '\\',
551        '"' => '"',
552        '\'' => '\'',
553        other => other,
554    }
555}
556
557/// Whether `c` may start an OPY identifier.
558pub fn is_ident_start(c: char) -> bool {
559    c.is_ascii_alphabetic() || c == '_'
560}
561
562/// Whether `c` may continue an OPY identifier.
563pub fn is_ident_continue(c: char) -> bool {
564    c.is_ascii_alphanumeric() || c == '_'
565}
566
567#[cfg(test)]
568mod tests {
569    use super::*;
570
571    fn lex_ok(text: &str) -> Vec<Token> {
572        lex(LexInput { file_id: 0, text }).unwrap()
573    }
574
575    #[test]
576    fn lexes_basic_rule() {
577        let tokens = lex_ok("rule \"setup\":\n    @Event global\n    disableInspector()\n");
578        let kinds: Vec<TokenKind> = tokens.iter().map(|t| t.kind).collect();
579        assert!(kinds.contains(&TokenKind::Ident));
580        assert!(kinds.contains(&TokenKind::String));
581        assert!(kinds.contains(&TokenKind::Colon));
582        assert!(kinds.contains(&TokenKind::At));
583        assert!(kinds.contains(&TokenKind::LParen));
584        assert!(kinds.contains(&TokenKind::Eof));
585    }
586
587    #[test]
588    fn numbers_preserve_text() {
589        let tokens = lex_ok("1 2.5 0.016 100");
590        let numbers: Vec<&str> = tokens
591            .iter()
592            .filter(|t| t.kind == TokenKind::Number)
593            .map(|t| t.text.as_str())
594            .collect();
595        assert_eq!(numbers, vec!["1", "2.5", "0.016", "100"]);
596    }
597
598    #[test]
599    fn directives_and_comments() {
600        let tokens = lex_ok("#!define X 1\n# comment\nrule \"r\":\n");
601        let directive = tokens
602            .iter()
603            .find(|t| t.kind == TokenKind::Directive)
604            .unwrap();
605        assert_eq!(directive.text, "define X 1");
606        assert!(!tokens.iter().any(|t| t.text == "comment"));
607    }
608
609    #[test]
610    fn directive_line_continuation_is_part_of_one_directive() {
611        let tokens = lex_ok("#!define X first + \\\n  second\n");
612        let directive = tokens
613            .iter()
614            .find(|token| token.kind == TokenKind::Directive)
615            .unwrap();
616        assert_eq!(directive.text, "define X first + \n  second");
617        assert_eq!(directive.span.start, Position::new(1, 1));
618        assert_eq!(directive.span.end, Position::new(2, 9));
619    }
620
621    #[test]
622    fn operators() {
623        let tokens = lex_ok("a += b == c <= d != e / f");
624        let kinds: Vec<TokenKind> = tokens.iter().map(|t| t.kind).collect();
625        for expected in [
626            TokenKind::PlusAssign,
627            TokenKind::Eq,
628            TokenKind::Le,
629            TokenKind::Ne,
630            TokenKind::Slash,
631        ] {
632            assert!(
633                kinds.contains(&expected),
634                "missing {expected:?} in {kinds:?}"
635            );
636        }
637    }
638
639    #[test]
640    fn power_operators_disambiguate() {
641        // The pinned OverPy 9.7.10 reference lexes `**=` as one power-assign
642        // operator distinct from `**` and `*=`.
643        let tokens = lex_ok("a **= b ** c *= d");
644        let kinds: Vec<TokenKind> = tokens.iter().map(|t| t.kind).collect();
645        assert_eq!(
646            kinds,
647            vec![
648                TokenKind::Ident,
649                TokenKind::DoubleStarAssign,
650                TokenKind::Ident,
651                TokenKind::DoubleStar,
652                TokenKind::Ident,
653                TokenKind::StarAssign,
654                TokenKind::Ident,
655                TokenKind::Eof,
656            ]
657        );
658        let assign = tokens
659            .iter()
660            .find(|t| t.kind == TokenKind::DoubleStarAssign)
661            .unwrap();
662        assert_eq!(assign.text, "**=");
663    }
664
665    #[test]
666    fn postfix_operators_are_single_tokens() {
667        let tokens = lex_ok("counter++ points--");
668        assert_eq!(
669            tokens.iter().map(|token| token.kind).collect::<Vec<_>>(),
670            vec![
671                TokenKind::Ident,
672                TokenKind::Increment,
673                TokenKind::Ident,
674                TokenKind::Decrement,
675                TokenKind::Eof,
676            ]
677        );
678        assert_eq!(tokens[1].span.start.col, 8);
679        assert_eq!(tokens[1].span.end.col, 10);
680    }
681
682    #[test]
683    fn unterminated_string_is_structured() {
684        let error = lex(LexInput {
685            file_id: 0,
686            text: "rule \"x\n",
687        })
688        .unwrap_err();
689        assert_eq!(error.code, "lex-error");
690        assert!(error.span.is_some());
691    }
692
693    #[test]
694    fn unicode_escapes_decode_at_the_string_boundary_and_keep_source_span() {
695        // Minimized from OWBastion/Bastion commit
696        // 44e12e08f046ed7fbf865237a29ddf2f272ec3c2,
697        // Bastion/src/utilities/system/savePlayerData.opy.
698        let tokens = lex_ok(r#"rule "pa\ufeffssed":"#);
699        let string = tokens
700            .iter()
701            .find(|token| token.kind == TokenKind::String)
702            .expect("rule name string token");
703        assert_eq!(string.text, "pa\u{feff}ssed");
704        assert_eq!(string.raw.as_deref(), Some(r"pa\ufeffssed"));
705        assert_eq!(string.span.start, Position::new(1, 6));
706        assert_eq!(string.span.end, Position::new(1, 20));
707    }
708
709    #[test]
710    fn existing_string_escapes_keep_their_decoded_values() {
711        let source = concat!("\"", "\\n", "\\t", "\\r", "\\\"", "\\\\", "\"");
712        let string = lex_ok(source)
713            .into_iter()
714            .find(|token| token.kind == TokenKind::String)
715            .expect("string token");
716        assert_eq!(string.text, "\n\t\r\"\\");
717    }
718
719    #[test]
720    fn named_string_entities_decode_with_raw_provenance() {
721        let string = lex_ok(r#""a\&black_square;b\&fullwidth_space;c""#)
722            .into_iter()
723            .find(|token| token.kind == TokenKind::String)
724            .expect("string token");
725        assert_eq!(string.text, "a■b　c");
726        assert_eq!(
727            string.raw.as_deref(),
728            Some(r"a\&black_square;b\&fullwidth_space;c")
729        );
730    }
731
732    #[test]
733    fn invalid_string_entities_are_structured_lex_errors() {
734        for (source, code) in [
735            (r#""\&missing;""#, "unknown-string-entity"),
736            (r#""\&black-square;""#, "invalid-string-entity"),
737            (r#""\&black_square""#, "invalid-string-entity"),
738        ] {
739            let error = lex(LexInput {
740                file_id: 0,
741                text: source,
742            })
743            .expect_err("invalid string entity unexpectedly lexed");
744            assert_eq!(error.code, code);
745            assert_eq!(error.span.unwrap().start, Position::new(1, 2));
746        }
747    }
748
749    #[test]
750    fn malformed_unicode_escapes_are_lex_errors() {
751        for source in [r#""\u"#, r#""\u12G4"#, r#""\u{1F600}"#] {
752            let error = lex(LexInput {
753                file_id: 0,
754                text: source,
755            })
756            .expect_err("malformed Unicode escape unexpectedly lexed");
757            assert_eq!(error.code, "lex-error");
758            assert!(error.message.contains("Unicode escape"));
759            assert_eq!(error.span.unwrap().start, Position::new(1, 2));
760        }
761    }
762
763    #[test]
764    fn backslash_line_continuation_is_not_a_token() {
765        let tokens = lex_ok("one \\\ntwo");
766        assert_eq!(
767            tokens.iter().map(|token| token.kind).collect::<Vec<_>>(),
768            vec![TokenKind::Ident, TokenKind::Ident, TokenKind::Eof]
769        );
770        assert_eq!(tokens[1].span.start.line, 2);
771        assert_eq!(tokens[1].span.start.col, 1);
772    }
773
774    #[test]
775    fn crlf_line_continuation_tracks_the_next_line() {
776        let tokens = lex_ok("one \\\r\ntwo");
777        assert_eq!(tokens[1].span.start, Position::new(2, 1));
778    }
779
780    #[test]
781    fn whitespace_before_line_ending_is_part_of_the_continuation() {
782        let tokens = lex_ok("one \\  \ntwo");
783        assert_eq!(tokens[1].span.start, Position::new(2, 1));
784    }
785
786    #[test]
787    fn non_newline_backslash_remains_a_lex_error() {
788        for text in ["one \\ two", "one \\", "one \\ \t\ntwo"] {
789            let error = lex(LexInput { file_id: 0, text }).unwrap_err();
790            assert_eq!(error.code, "lex-error");
791            assert_eq!(error.message, "unexpected character '\\'");
792            assert_eq!(error.span.unwrap().start, Position::new(1, 5));
793        }
794    }
795}
opy_rs/lexer.rs

opy_rs/
lexer.rs
