opy_rs/
lexer.rs

1//! The native `.opy` lexer.
2//!
3//! Produces a flat token stream (newlines and indentation included) from one
4//! source file. Comments (`#`, `/* */`) are skipped; `#!` directives are
5//! captured as a single directive token for the preprocessor. Positions are
6//! 1-based line/column, matching the Opy HIR protocol.
7
8use crate::diag::{OpyError, OpyResult, Position, Span};
9
10/// The kind of a token.
11#[derive(Debug, Clone, Copy, PartialEq, Eq)]
12pub enum TokenKind {
13    /// An identifier or keyword (keywords are resolved by the parser).
14    Ident,
15    /// A numeric literal (`text` holds the source spelling).
16    Number,
17    /// A string literal (`text` holds the unescaped value).
18    String,
19    /// A `#!` directive line (`text` holds everything after `#!`).
20    Directive,
21    /// A preprocessing marker carrying the rule-prefix state active at the
22    /// following top-level rule or subroutine.
23    RulePrefixMarker,
24    /// `@Event` / `@Condition` / other `@` directives.
25    At,
26    Newline,
27    /// Indentation change: the column of the current line.
28    Indent(u32),
29    /// End of file.
30    Eof,
31    // Punctuation and operators.
32    LParen,
33    RParen,
34    LBracket,
35    RBracket,
36    LBrace,
37    RBrace,
38    Comma,
39    Colon,
40    Semicolon,
41    Dot,
42    Assign,
43    Plus,
44    Minus,
45    Star,
46    Slash,
47    Percent,
48    DoubleStar,
49    PlusAssign,
50    MinusAssign,
51    Increment,
52    Decrement,
53    StarAssign,
54    SlashAssign,
55    PercentAssign,
56    DoubleStarAssign,
57    Eq,
58    Ne,
59    Lt,
60    Le,
61    Gt,
62    Ge,
63    /// A bare `!` that is not `!=`; the parser rejects it as unsupported OPY.
64    LexBang,
65}
66
67/// One token with its source span and payload text.
68#[derive(Debug, Clone, PartialEq)]
69pub struct Token {
70    pub kind: TokenKind,
71    /// The source text of this token (numbers keep their spelling; strings
72    /// keep their unescaped value; identifiers keep their name).
73    pub text: String,
74    /// The exact characters between string quotes, before escape decoding.
75    /// Other token kinds leave this unset. It is retained so source-language
76    /// constructs such as f-string interpolations can recover expression
77    /// spans without losing provenance during preprocessing.
78    pub raw: Option<String>,
79    pub span: Span,
80}
81
82impl Token {
83    fn new(kind: TokenKind, text: impl Into<String>, span: Span) -> Token {
84        Token {
85            kind,
86            text: text.into(),
87            raw: None,
88            span,
89        }
90    }
91}
92
93/// The lexer input: one file's text with its file id.
94pub struct LexInput<'a> {
95    pub file_id: u32,
96    pub text: &'a str,
97}
98
99/// Lex one source file into a token stream.
100pub fn lex(input: LexInput<'_>) -> OpyResult<Vec<Token>> {
101    Lexer::new(input.file_id, input.text).run()
102}
103
104struct Lexer {
105    file_id: u32,
106    chars: Vec<char>,
107    pos: usize,
108    line: u32,
109    col: u32,
110    tokens: Vec<Token>,
111}
112
113impl Lexer {
114    fn new(file_id: u32, text: &str) -> Lexer {
115        Lexer {
116            file_id,
117            chars: text.chars().collect(),
118            pos: 0,
119            line: 1,
120            col: 1,
121            tokens: Vec::new(),
122        }
123    }
124
125    fn run(mut self) -> OpyResult<Vec<Token>> {
126        while self.pos < self.chars.len() {
127            let ch = self.chars[self.pos];
128            match ch {
129                '\n' => {
130                    self.tokens
131                        .push(Token::new(TokenKind::Newline, "\n", self.here(1)));
132                    self.advance();
133                    self.line += 1;
134                    self.col = 1;
135                }
136                ' ' | '\r' => {
137                    self.advance();
138                }
139                '\t' => {
140                    self.pos += 1;
141                    self.col += 4;
142                }
143                '\\' => {
144                    if !self.skip_line_continuation() {
145                        return Err(OpyError::at(
146                            "lex-error",
147                            "unexpected character '\\'",
148                            self.here(1),
149                        ));
150                    }
151                }
152                '#' => self.lex_hash()?,
153                '/' if self.peek(1) == Some('*') => self.skip_block_comment()?,
154                '"' | '\'' => self.lex_string(ch)?,
155                c if c.is_ascii_digit() => self.lex_number()?,
156                c if is_ident_start(c) => self.lex_ident(),
157                '(' => self.single(TokenKind::LParen),
158                ')' => self.single(TokenKind::RParen),
159                '[' => self.single(TokenKind::LBracket),
160                ']' => self.single(TokenKind::RBracket),
161                '{' => self.single(TokenKind::LBrace),
162                '}' => self.single(TokenKind::RBrace),
163                ',' => self.single(TokenKind::Comma),
164                ':' => self.single(TokenKind::Colon),
165                ';' => self.single(TokenKind::Semicolon),
166                '.' => self.single(TokenKind::Dot),
167                '@' => self.single(TokenKind::At),
168                '=' => self.two(TokenKind::Assign, TokenKind::Eq, '='),
169                '+' => {
170                    if self.peek(1) == Some('+') {
171                        self.lex_duplicate(TokenKind::Increment, "++");
172                    } else {
173                        self.lex_two(TokenKind::Plus, TokenKind::PlusAssign, '=');
174                    }
175                }
176                '-' => {
177                    if self.peek(1) == Some('-') {
178                        self.lex_duplicate(TokenKind::Decrement, "--");
179                    } else {
180                        self.lex_two(TokenKind::Minus, TokenKind::MinusAssign, '=');
181                    }
182                }
183                '*' => {
184                    if self.peek(1) == Some('*') {
185                        if self.peek(2) == Some('=') {
186                            let start = self.here(3);
187                            self.advance();
188                            self.advance();
189                            self.advance();
190                            let end = self.here(0);
191                            self.tokens.push(Token::new(
192                                TokenKind::DoubleStarAssign,
193                                "**=",
194                                Span::new(self.file_id, start.start, end.start),
195                            ));
196                        } else {
197                            self.advance();
198                            self.single(TokenKind::DoubleStar)
199                        }
200                    } else {
201                        self.lex_two(TokenKind::Star, TokenKind::StarAssign, '=')
202                    }
203                }
204                '/' => self.lex_two(TokenKind::Slash, TokenKind::SlashAssign, '='),
205                '%' => self.lex_two(TokenKind::Percent, TokenKind::PercentAssign, '='),
206                '<' => self.two(TokenKind::Lt, TokenKind::Le, '='),
207                '>' => self.two(TokenKind::Gt, TokenKind::Ge, '='),
208                '!' => self.two(TokenKind::LexBang, TokenKind::Ne, '='),
209                other => {
210                    return Err(OpyError::at(
211                        "lex-error",
212                        format!("unexpected character '{other}'"),
213                        self.here(1),
214                    ));
215                }
216            }
217        }
218        let here = self.here(0);
219        self.tokens.push(Token::new(TokenKind::Eof, "", here));
220        Ok(self.tokens)
221    }
222
223    /// `#` starts a `#!` directive (captured as one token) or a comment.
224    fn lex_hash(&mut self) -> OpyResult<()> {
225        if self.peek(1) == Some('!') {
226            let start = self.here(2);
227            self.advance();
228            self.advance();
229            let mut text = String::new();
230            while self.pos < self.chars.len() {
231                if self.chars[self.pos] == '\\' && self.skip_line_continuation() {
232                    text.push('\n');
233                    continue;
234                }
235                if self.chars[self.pos] == '\n' {
236                    break;
237                }
238                text.push(self.chars[self.pos]);
239                self.advance();
240            }
241            let end = self.here(0);
242            self.tokens.push(Token::new(
243                TokenKind::Directive,
244                text,
245                Span::new(self.file_id, start.start, end.start),
246            ));
247        } else {
248            while self.pos < self.chars.len() && self.chars[self.pos] != '\n' {
249                self.advance();
250            }
251        }
252        Ok(())
253    }
254
255    fn skip_block_comment(&mut self) -> OpyResult<()> {
256        let start = self.here(2);
257        self.advance();
258        self.advance();
259        while self.pos < self.chars.len() {
260            if self.chars[self.pos] == '*' && self.peek(1) == Some('/') {
261                self.advance();
262                self.advance();
263                return Ok(());
264            }
265            if self.chars[self.pos] == '\n' {
266                self.advance();
267                self.line += 1;
268                self.col = 1;
269            } else {
270                self.advance();
271            }
272        }
273        Err(OpyError::at(
274            "lex-error",
275            "unterminated block comment",
276            start,
277        ))
278    }
279
280    fn lex_string(&mut self, quote: char) -> OpyResult<()> {
281        let start = self.here(1);
282        self.advance();
283        let mut value = String::new();
284        let mut raw = String::new();
285        while self.pos < self.chars.len() {
286            let ch = self.chars[self.pos];
287            if ch == quote {
288                self.advance();
289                let end = self.here(0);
290                let mut token = Token::new(
291                    TokenKind::String,
292                    value,
293                    Span::new(self.file_id, start.start, end.start),
294                );
295                token.raw = Some(raw);
296                self.tokens.push(token);
297                return Ok(());
298            }
299            if ch == '\\' {
300                let escape_start = self.here(1);
301                raw.push(ch);
302                self.advance();
303                if self.pos >= self.chars.len() {
304                    break;
305                }
306                let escaped = self.chars[self.pos];
307                raw.push(escaped);
308                if escaped == '&' {
309                    self.advance();
310                    let mut entity_name = String::new();
311                    while let Some(character) = self.chars.get(self.pos).copied() {
312                        if character == ';' {
313                            break;
314                        }
315                        if !(character.is_ascii_alphanumeric() || character == '_') {
316                            return Err(OpyError::at(
317                                "invalid-string-entity",
318                                format!("invalid character '{character}' in string entity"),
319                                Span::new(self.file_id, escape_start.start, self.here(1).end),
320                            ));
321                        }
322                        entity_name.push(character);
323                        raw.push(character);
324                        self.advance();
325                    }
326                    if self.chars.get(self.pos) != Some(&';') {
327                        return Err(OpyError::at(
328                            "invalid-string-entity",
329                            "expected ';' to terminate string entity",
330                            Span::new(self.file_id, escape_start.start, self.here(0).start),
331                        ));
332                    }
333                    raw.push(';');
334                    self.advance();
335                    let Some(codepoint) = crate::string_entities::codepoint(&entity_name) else {
336                        return Err(OpyError::at(
337                            "unknown-string-entity",
338                            format!("unknown string entity '{entity_name}'"),
339                            Span::new(self.file_id, escape_start.start, self.here(0).start),
340                        ));
341                    };
342                    value.push(codepoint);
343                    continue;
344                }
345                if escaped == 'u' {
346                    self.advance();
347                    let mut codepoint = 0_u32;
348                    for _ in 0..4 {
349                        let Some(digit) = self.chars.get(self.pos).copied() else {
350                            return Err(OpyError::at(
351                                "lex-error",
352                                "Unicode escape requires four hexadecimal digits",
353                                Span::new(self.file_id, escape_start.start, self.here(0).start),
354                            ));
355                        };
356                        let Some(digit_value) = digit.to_digit(16) else {
357                            return Err(OpyError::at(
358                                "lex-error",
359                                "Unicode escape requires four hexadecimal digits",
360                                Span::new(self.file_id, escape_start.start, self.here(1).end),
361                            ));
362                        };
363                        raw.push(digit);
364                        codepoint = codepoint * 16 + digit_value;
365                        self.advance();
366                    }
367                    let Some(decoded) = char::from_u32(codepoint) else {
368                        return Err(OpyError::at(
369                            "lex-error",
370                            "Unicode escape does not name a Unicode scalar value",
371                            Span::new(self.file_id, escape_start.start, self.here(0).start),
372                        ));
373                    };
374                    value.push(decoded);
375                    continue;
376                }
377                value.push(match escaped {
378                    'n' => '\n',
379                    't' => '\t',
380                    'r' => '\r',
381                    '\\' => '\\',
382                    '"' => '"',
383                    '\'' => '\'',
384                    other => other,
385                });
386                self.advance();
387                continue;
388            }
389            if ch == '\n' {
390                return Err(OpyError::at(
391                    "lex-error",
392                    "unterminated string literal",
393                    start,
394                ));
395            }
396            raw.push(ch);
397            value.push(ch);
398            self.advance();
399        }
400        Err(OpyError::at(
401            "lex-error",
402            "unterminated string literal",
403            start,
404        ))
405    }
406
407    fn skip_line_continuation(&mut self) -> bool {
408        let mut offset = 1;
409        while matches!(self.peek(offset), Some(' ' | '\r')) {
410            offset += 1;
411        }
412        if self.peek(offset) != Some('\n') {
413            return false;
414        }
415        for _ in 0..=offset {
416            self.advance();
417        }
418        self.line += 1;
419        self.col = 1;
420        true
421    }
422
423    fn lex_number(&mut self) -> OpyResult<()> {
424        let start = self.here(1);
425        let mut text = String::new();
426        if self.chars[self.pos] == '0' && matches!(self.peek(1), Some('x' | 'X')) {
427            text.push('0');
428            self.advance();
429            text.push(self.chars[self.pos]);
430            self.advance();
431            let digits_start = self.pos;
432            while self.pos < self.chars.len() && self.chars[self.pos].is_ascii_hexdigit() {
433                text.push(self.chars[self.pos]);
434                self.advance();
435            }
436            if self.pos == digits_start {
437                return Err(OpyError::at(
438                    "lex-error",
439                    "hexadecimal literal requires at least one hexadecimal digit",
440                    Span::new(self.file_id, start.start, self.here(0).start),
441                ));
442            }
443            let end = self.here(0);
444            self.tokens.push(Token::new(
445                TokenKind::Number,
446                text,
447                Span::new(self.file_id, start.start, end.start),
448            ));
449            return Ok(());
450        }
451        while self.pos < self.chars.len() && self.chars[self.pos].is_ascii_digit() {
452            text.push(self.chars[self.pos]);
453            self.advance();
454        }
455        if self.pos < self.chars.len()
456            && self.chars[self.pos] == '.'
457            && self.peek(1).is_some_and(|c| c.is_ascii_digit())
458        {
459            text.push('.');
460            self.advance();
461            while self.pos < self.chars.len() && self.chars[self.pos].is_ascii_digit() {
462                text.push(self.chars[self.pos]);
463                self.advance();
464            }
465        }
466        // Optional exponent (not exercised by the corpus, supported for
467        // completeness of the number surface).
468        if self.pos < self.chars.len()
469            && (self.chars[self.pos] == 'e' || self.chars[self.pos] == 'E')
470        {
471            let mut lookahead = self.pos + 1;
472            if lookahead < self.chars.len()
473                && (self.chars[lookahead] == '+' || self.chars[lookahead] == '-')
474            {
475                lookahead += 1;
476            }
477            if lookahead < self.chars.len() && self.chars[lookahead].is_ascii_digit() {
478                text.push('e');
479                self.advance();
480                if self.pos < self.chars.len()
481                    && (self.chars[self.pos] == '+' || self.chars[self.pos] == '-')
482                {
483                    text.push(self.chars[self.pos]);
484                    self.advance();
485                }
486                while self.pos < self.chars.len() && self.chars[self.pos].is_ascii_digit() {
487                    text.push(self.chars[self.pos]);
488                    self.advance();
489                }
490            }
491        }
492        let end = self.here(0);
493        self.tokens.push(Token::new(
494            TokenKind::Number,
495            text,
496            Span::new(self.file_id, start.start, end.start),
497        ));
498        Ok(())
499    }
500
501    fn lex_ident(&mut self) {
502        let start = self.here(1);
503        let mut text = String::new();
504        while self.pos < self.chars.len() && is_ident_continue(self.chars[self.pos]) {
505            text.push(self.chars[self.pos]);
506            self.advance();
507        }
508        let end = self.here(0);
509        self.tokens.push(Token::new(
510            TokenKind::Ident,
511            text,
512            Span::new(self.file_id, start.start, end.start),
513        ));
514    }
515
516    fn single(&mut self, kind: TokenKind) {
517        let start = self.here(1);
518        let text = self.chars[self.pos].to_string();
519        self.advance();
520        let end = self.here(0);
521        self.tokens.push(Token::new(
522            kind,
523            text,
524            Span::new(self.file_id, start.start, end.start),
525        ));
526    }
527
528    /// Two-char operator where the second char may be `=`.
529    fn lex_two(&mut self, plain: TokenKind, assign: TokenKind, second: char) {
530        let start = self.here(1);
531        if self.peek(1) == Some(second) {
532            self.advance();
533            let text = format!("{}{}", self.chars[self.pos - 1], second);
534            self.advance();
535            let end = self.here(0);
536            self.tokens.push(Token::new(
537                assign,
538                text,
539                Span::new(self.file_id, start.start, end.start),
540            ));
541        } else {
542            let text = self.chars[self.pos].to_string();
543            self.advance();
544            let end = self.here(0);
545            self.tokens.push(Token::new(
546                plain,
547                text,
548                Span::new(self.file_id, start.start, end.start),
549            ));
550        }
551    }
552
553    fn lex_duplicate(&mut self, kind: TokenKind, text: &str) {
554        let start = self.here(1);
555        self.advance();
556        self.advance();
557        let end = self.here(0);
558        self.tokens.push(Token::new(
559            kind,
560            text,
561            Span::new(self.file_id, start.start, end.start),
562        ));
563    }
564
565    /// Two-char operator with a fixed second char (e.g. `==`, `<=`).
566    fn two(&mut self, plain: TokenKind, combined: TokenKind, second: char) {
567        let start = self.here(1);
568        let text = self.chars[self.pos].to_string();
569        if self.peek(1) == Some(second) {
570            self.advance();
571            let combined_text = format!("{}{}", text, second);
572            self.advance();
573            let end = self.here(0);
574            self.tokens.push(Token::new(
575                combined,
576                combined_text,
577                Span::new(self.file_id, start.start, end.start),
578            ));
579        } else {
580            self.advance();
581            let end = self.here(0);
582            self.tokens.push(Token::new(
583                plain,
584                text,
585                Span::new(self.file_id, start.start, end.start),
586            ));
587        }
588    }
589
590    fn here(&self, width: usize) -> Span {
591        Span::new(
592            self.file_id,
593            Position::new(self.line, self.col),
594            Position::new(self.line, self.col + width as u32),
595        )
596    }
597
598    fn peek(&self, offset: usize) -> Option<char> {
599        self.chars.get(self.pos + offset).copied()
600    }
601
602    fn advance(&mut self) {
603        self.pos += 1;
604        self.col += 1;
605    }
606}
607
608fn is_ident_start(c: char) -> bool {
609    c.is_ascii_alphabetic() || c == '_'
610}
611
612fn is_ident_continue(c: char) -> bool {
613    c.is_ascii_alphanumeric() || c == '_'
614}
615
616#[cfg(test)]
617mod tests {
618    use super::*;
619
620    fn lex_ok(text: &str) -> Vec<Token> {
621        lex(LexInput { file_id: 0, text }).unwrap()
622    }
623
624    #[test]
625    fn lexes_basic_rule() {
626        let tokens = lex_ok("rule \"setup\":\n    @Event global\n    disableInspector()\n");
627        let kinds: Vec<TokenKind> = tokens.iter().map(|t| t.kind).collect();
628        assert!(kinds.contains(&TokenKind::Ident));
629        assert!(kinds.contains(&TokenKind::String));
630        assert!(kinds.contains(&TokenKind::Colon));
631        assert!(kinds.contains(&TokenKind::At));
632        assert!(kinds.contains(&TokenKind::LParen));
633        assert!(kinds.contains(&TokenKind::Eof));
634    }
635
636    #[test]
637    fn numbers_preserve_text() {
638        let tokens = lex_ok("1 2.5 0.016 100");
639        let numbers: Vec<&str> = tokens
640            .iter()
641            .filter(|t| t.kind == TokenKind::Number)
642            .map(|t| t.text.as_str())
643            .collect();
644        assert_eq!(numbers, vec!["1", "2.5", "0.016", "100"]);
645    }
646
647    #[test]
648    fn directives_and_comments() {
649        let tokens = lex_ok("#!define X 1\n# comment\nrule \"r\":\n");
650        let directive = tokens
651            .iter()
652            .find(|t| t.kind == TokenKind::Directive)
653            .unwrap();
654        assert_eq!(directive.text, "define X 1");
655        assert!(!tokens.iter().any(|t| t.text == "comment"));
656    }
657
658    #[test]
659    fn directive_line_continuation_is_part_of_one_directive() {
660        let tokens = lex_ok("#!define X first + \\\n  second\n");
661        let directive = tokens
662            .iter()
663            .find(|token| token.kind == TokenKind::Directive)
664            .unwrap();
665        assert_eq!(directive.text, "define X first + \n  second");
666        assert_eq!(directive.span.start, Position::new(1, 1));
667        assert_eq!(directive.span.end, Position::new(2, 9));
668    }
669
670    #[test]
671    fn operators() {
672        let tokens = lex_ok("a += b == c <= d != e / f");
673        let kinds: Vec<TokenKind> = tokens.iter().map(|t| t.kind).collect();
674        for expected in [
675            TokenKind::PlusAssign,
676            TokenKind::Eq,
677            TokenKind::Le,
678            TokenKind::Ne,
679            TokenKind::Slash,
680        ] {
681            assert!(
682                kinds.contains(&expected),
683                "missing {expected:?} in {kinds:?}"
684            );
685        }
686    }
687
688    #[test]
689    fn power_operators_disambiguate() {
690        // The pinned OverPy 9.7.10 reference lexes `**=` as one power-assign
691        // operator distinct from `**` and `*=`.
692        let tokens = lex_ok("a **= b ** c *= d");
693        let kinds: Vec<TokenKind> = tokens.iter().map(|t| t.kind).collect();
694        assert_eq!(
695            kinds,
696            vec![
697                TokenKind::Ident,
698                TokenKind::DoubleStarAssign,
699                TokenKind::Ident,
700                TokenKind::DoubleStar,
701                TokenKind::Ident,
702                TokenKind::StarAssign,
703                TokenKind::Ident,
704                TokenKind::Eof,
705            ]
706        );
707        let assign = tokens
708            .iter()
709            .find(|t| t.kind == TokenKind::DoubleStarAssign)
710            .unwrap();
711        assert_eq!(assign.text, "**=");
712    }
713
714    #[test]
715    fn postfix_operators_are_single_tokens() {
716        let tokens = lex_ok("counter++ points--");
717        assert_eq!(
718            tokens.iter().map(|token| token.kind).collect::<Vec<_>>(),
719            vec![
720                TokenKind::Ident,
721                TokenKind::Increment,
722                TokenKind::Ident,
723                TokenKind::Decrement,
724                TokenKind::Eof,
725            ]
726        );
727        assert_eq!(tokens[1].span.start.col, 8);
728        assert_eq!(tokens[1].span.end.col, 10);
729    }
730
731    #[test]
732    fn unterminated_string_is_structured() {
733        let error = lex(LexInput {
734            file_id: 0,
735            text: "rule \"x\n",
736        })
737        .unwrap_err();
738        assert_eq!(error.code, "lex-error");
739        assert!(error.span.is_some());
740    }
741
742    #[test]
743    fn unicode_escapes_decode_at_the_string_boundary_and_keep_source_span() {
744        // Minimized from OWBastion/Bastion commit
745        // 44e12e08f046ed7fbf865237a29ddf2f272ec3c2,
746        // Bastion/src/utilities/system/savePlayerData.opy.
747        let tokens = lex_ok(r#"rule "pa\ufeffssed":"#);
748        let string = tokens
749            .iter()
750            .find(|token| token.kind == TokenKind::String)
751            .expect("rule name string token");
752        assert_eq!(string.text, "pa\u{feff}ssed");
753        assert_eq!(string.raw.as_deref(), Some(r"pa\ufeffssed"));
754        assert_eq!(string.span.start, Position::new(1, 6));
755        assert_eq!(string.span.end, Position::new(1, 20));
756    }
757
758    #[test]
759    fn existing_string_escapes_keep_their_decoded_values() {
760        let source = concat!("\"", "\\n", "\\t", "\\r", "\\\"", "\\\\", "\"");
761        let string = lex_ok(source)
762            .into_iter()
763            .find(|token| token.kind == TokenKind::String)
764            .expect("string token");
765        assert_eq!(string.text, "\n\t\r\"\\");
766    }
767
768    #[test]
769    fn named_string_entities_decode_with_raw_provenance() {
770        let string = lex_ok(r#""a\&black_square;b\&fullwidth_space;c""#)
771            .into_iter()
772            .find(|token| token.kind == TokenKind::String)
773            .expect("string token");
774        assert_eq!(string.text, "a■b　c");
775        assert_eq!(
776            string.raw.as_deref(),
777            Some(r"a\&black_square;b\&fullwidth_space;c")
778        );
779    }
780
781    #[test]
782    fn invalid_string_entities_are_structured_lex_errors() {
783        for (source, code) in [
784            (r#""\&missing;""#, "unknown-string-entity"),
785            (r#""\&black-square;""#, "invalid-string-entity"),
786            (r#""\&black_square""#, "invalid-string-entity"),
787        ] {
788            let error = lex(LexInput {
789                file_id: 0,
790                text: source,
791            })
792            .expect_err("invalid string entity unexpectedly lexed");
793            assert_eq!(error.code, code);
794            assert_eq!(error.span.unwrap().start, Position::new(1, 2));
795        }
796    }
797
798    #[test]
799    fn malformed_unicode_escapes_are_lex_errors() {
800        for source in [r#""\u"#, r#""\u12G4"#, r#""\u{1F600}"#] {
801            let error = lex(LexInput {
802                file_id: 0,
803                text: source,
804            })
805            .expect_err("malformed Unicode escape unexpectedly lexed");
806            assert_eq!(error.code, "lex-error");
807            assert!(error.message.contains("Unicode escape"));
808            assert_eq!(error.span.unwrap().start, Position::new(1, 2));
809        }
810    }
811
812    #[test]
813    fn backslash_line_continuation_is_not_a_token() {
814        let tokens = lex_ok("one \\\ntwo");
815        assert_eq!(
816            tokens.iter().map(|token| token.kind).collect::<Vec<_>>(),
817            vec![TokenKind::Ident, TokenKind::Ident, TokenKind::Eof]
818        );
819        assert_eq!(tokens[1].span.start.line, 2);
820        assert_eq!(tokens[1].span.start.col, 1);
821    }
822
823    #[test]
824    fn crlf_line_continuation_tracks_the_next_line() {
825        let tokens = lex_ok("one \\\r\ntwo");
826        assert_eq!(tokens[1].span.start, Position::new(2, 1));
827    }
828
829    #[test]
830    fn whitespace_before_line_ending_is_part_of_the_continuation() {
831        let tokens = lex_ok("one \\  \ntwo");
832        assert_eq!(tokens[1].span.start, Position::new(2, 1));
833    }
834
835    #[test]
836    fn non_newline_backslash_remains_a_lex_error() {
837        for text in ["one \\ two", "one \\", "one \\ \t\ntwo"] {
838            let error = lex(LexInput { file_id: 0, text }).unwrap_err();
839            assert_eq!(error.code, "lex-error");
840            assert_eq!(error.message, "unexpected character '\\'");
841            assert_eq!(error.span.unwrap().start, Position::new(1, 5));
842        }
843    }
844}
opy_rs/lexer.rs

opy_rs/
lexer.rs
