opy_rs/
lexer.rs

1//! The native `.opy` lexer.
2//!
3//! Produces a flat token stream (newlines and indentation included) from one
4//! source file. Comments (`#`, `/* */`) are skipped; `#!` directives are
5//! captured as a single directive token for the preprocessor. Positions are
6//! 1-based line/column, matching the Opy HIR protocol.
7
8use crate::diag::{OpyError, OpyResult, Position, Span};
9
10/// The kind of a token.
11#[derive(Debug, Clone, Copy, PartialEq, Eq)]
12pub enum TokenKind {
13    /// An identifier or keyword (keywords are resolved by the parser).
14    Ident,
15    /// A numeric literal (`text` holds the source spelling).
16    Number,
17    /// A string literal (`text` holds the unescaped value).
18    String,
19    /// A `#!` directive line (`text` holds everything after `#!`).
20    Directive,
21    /// A preprocessing marker carrying the rule-prefix state active at the
22    /// following top-level rule or subroutine.
23    RulePrefixMarker,
24    /// `@Event` / `@Condition` / other `@` directives.
25    At,
26    Newline,
27    /// Indentation change: the column of the current line.
28    Indent(u32),
29    /// End of file.
30    Eof,
31    // Punctuation and operators.
32    LParen,
33    RParen,
34    LBracket,
35    RBracket,
36    LBrace,
37    RBrace,
38    Comma,
39    Colon,
40    Semicolon,
41    Dot,
42    Assign,
43    Plus,
44    Minus,
45    Star,
46    Slash,
47    Percent,
48    DoubleStar,
49    PlusAssign,
50    MinusAssign,
51    Increment,
52    Decrement,
53    StarAssign,
54    SlashAssign,
55    PercentAssign,
56    DoubleStarAssign,
57    Eq,
58    Ne,
59    Lt,
60    Le,
61    Gt,
62    Ge,
63    /// A bare `!` that is not `!=`; the parser rejects it as unsupported OPY.
64    LexBang,
65}
66
67/// One token with its source span and payload text.
68#[derive(Debug, Clone, PartialEq)]
69pub struct Token {
70    pub kind: TokenKind,
71    /// The source text of this token (numbers keep their spelling; strings
72    /// keep their unescaped value; identifiers keep their name).
73    pub text: String,
74    /// The exact characters between string quotes, before escape decoding.
75    /// Other token kinds leave this unset. It is retained so source-language
76    /// constructs such as f-string interpolations can recover expression
77    /// spans without losing provenance during preprocessing.
78    pub raw: Option<String>,
79    /// Authored-source provenance. Tokens produced by macro or script
80    /// expansion carry the use-site span: the expanded text does not exist
81    /// in authored source, so the invocation is the only honest position.
82    pub span: Span,
83    /// The position the indentation-sensitive parser reads for layout
84    /// decisions. It equals `span` for authored tokens; expanded tokens keep
85    /// the expansion's relative line and column structure here so blocks and
86    /// statement boundaries still resolve (#506).
87    pub(crate) layout: Span,
88}
89
90impl Token {
91    fn new(kind: TokenKind, text: impl Into<String>, span: Span) -> Token {
92        Token {
93            kind,
94            text: text.into(),
95            raw: None,
96            span,
97            layout: span,
98        }
99    }
100}
101
102/// The lexer input: one file's text with its file id.
103pub struct LexInput<'a> {
104    pub file_id: u32,
105    pub text: &'a str,
106}
107
108/// Lex one source file into a token stream.
109pub fn lex(input: LexInput<'_>) -> OpyResult<Vec<Token>> {
110    Lexer::new(input.file_id, input.text).run()
111}
112
113struct Lexer {
114    file_id: u32,
115    chars: Vec<char>,
116    pos: usize,
117    line: u32,
118    col: u32,
119    tokens: Vec<Token>,
120}
121
122impl Lexer {
123    fn new(file_id: u32, text: &str) -> Lexer {
124        Lexer {
125            file_id,
126            chars: text.chars().collect(),
127            pos: 0,
128            line: 1,
129            col: 1,
130            tokens: Vec::new(),
131        }
132    }
133
134    fn run(mut self) -> OpyResult<Vec<Token>> {
135        while self.pos < self.chars.len() {
136            let ch = self.chars[self.pos];
137            match ch {
138                '\n' => {
139                    self.tokens
140                        .push(Token::new(TokenKind::Newline, "\n", self.here(1)));
141                    self.advance();
142                    self.line += 1;
143                    self.col = 1;
144                }
145                ' ' | '\r' => {
146                    self.advance();
147                }
148                '\t' => {
149                    self.pos += 1;
150                    self.col += 4;
151                }
152                '\\' => {
153                    if !self.skip_line_continuation() {
154                        return Err(OpyError::at(
155                            "lex-error",
156                            "unexpected character '\\'",
157                            self.here(1),
158                        ));
159                    }
160                }
161                '#' => self.lex_hash(),
162                '/' if self.peek(1) == Some('*') => self.skip_block_comment()?,
163                '"' | '\'' => self.lex_string(ch)?,
164                c if c.is_ascii_digit() => self.lex_number()?,
165                c if is_ident_start(c) => self.lex_ident(),
166                '(' => self.single(TokenKind::LParen),
167                ')' => self.single(TokenKind::RParen),
168                '[' => self.single(TokenKind::LBracket),
169                ']' => self.single(TokenKind::RBracket),
170                '{' => self.single(TokenKind::LBrace),
171                '}' => self.single(TokenKind::RBrace),
172                ',' => self.single(TokenKind::Comma),
173                ':' => self.single(TokenKind::Colon),
174                ';' => self.single(TokenKind::Semicolon),
175                '.' => self.single(TokenKind::Dot),
176                '@' => self.single(TokenKind::At),
177                '=' => self.lex_two(TokenKind::Assign, TokenKind::Eq, '='),
178                '+' => {
179                    if self.peek(1) == Some('+') {
180                        self.lex_duplicate(TokenKind::Increment, "++");
181                    } else {
182                        self.lex_two(TokenKind::Plus, TokenKind::PlusAssign, '=');
183                    }
184                }
185                '-' => {
186                    if self.peek(1) == Some('-') {
187                        self.lex_duplicate(TokenKind::Decrement, "--");
188                    } else {
189                        self.lex_two(TokenKind::Minus, TokenKind::MinusAssign, '=');
190                    }
191                }
192                '*' => {
193                    if self.peek(1) == Some('*') {
194                        if self.peek(2) == Some('=') {
195                            let start = self.here(3);
196                            self.advance();
197                            self.advance();
198                            self.advance();
199                            self.push_token(TokenKind::DoubleStarAssign, "**=", start);
200                        } else {
201                            self.advance();
202                            self.single(TokenKind::DoubleStar)
203                        }
204                    } else {
205                        self.lex_two(TokenKind::Star, TokenKind::StarAssign, '=')
206                    }
207                }
208                '/' => self.lex_two(TokenKind::Slash, TokenKind::SlashAssign, '='),
209                '%' => self.lex_two(TokenKind::Percent, TokenKind::PercentAssign, '='),
210                '<' => self.lex_two(TokenKind::Lt, TokenKind::Le, '='),
211                '>' => self.lex_two(TokenKind::Gt, TokenKind::Ge, '='),
212                '!' => self.lex_two(TokenKind::LexBang, TokenKind::Ne, '='),
213                other => {
214                    return Err(OpyError::at(
215                        "lex-error",
216                        format!("unexpected character '{other}'"),
217                        self.here(1),
218                    ));
219                }
220            }
221        }
222        let here = self.here(0);
223        self.push_token(TokenKind::Eof, "", here);
224        Ok(self.tokens)
225    }
226
227    /// `#` starts a `#!` directive (captured as one token) or a comment.
228    fn lex_hash(&mut self) {
229        if self.peek(1) == Some('!') {
230            let start = self.here(2);
231            self.advance();
232            self.advance();
233            let mut text = String::new();
234            while self.pos < self.chars.len() {
235                if self.chars[self.pos] == '\\' && self.skip_line_continuation() {
236                    text.push('\n');
237                    continue;
238                }
239                if self.chars[self.pos] == '\n' {
240                    break;
241                }
242                text.push(self.chars[self.pos]);
243                self.advance();
244            }
245            self.push_token(TokenKind::Directive, text, start);
246        } else {
247            while self.pos < self.chars.len() && self.chars[self.pos] != '\n' {
248                self.advance();
249            }
250        }
251    }
252
253    fn skip_block_comment(&mut self) -> OpyResult<()> {
254        let start = self.here(2);
255        self.advance();
256        self.advance();
257        while self.pos < self.chars.len() {
258            if self.chars[self.pos] == '*' && self.peek(1) == Some('/') {
259                self.advance();
260                self.advance();
261                return Ok(());
262            }
263            if self.chars[self.pos] == '\n' {
264                self.advance();
265                self.line += 1;
266                self.col = 1;
267            } else {
268                self.advance();
269            }
270        }
271        Err(OpyError::at(
272            "lex-error",
273            "unterminated block comment",
274            start,
275        ))
276    }
277
278    fn lex_string(&mut self, quote: char) -> OpyResult<()> {
279        let start = self.here(1);
280        self.advance();
281        let mut value = String::new();
282        let mut raw = String::new();
283        while self.pos < self.chars.len() {
284            let ch = self.chars[self.pos];
285            if ch == quote {
286                self.advance();
287                let mut token = self.make_token(TokenKind::String, value, start);
288                token.raw = Some(raw);
289                self.tokens.push(token);
290                return Ok(());
291            }
292            if ch == '\\' {
293                let escape_start = self.here(1);
294                raw.push(ch);
295                self.advance();
296                if self.pos >= self.chars.len() {
297                    break;
298                }
299                let escaped = self.chars[self.pos];
300                raw.push(escaped);
301                if escaped == '&' {
302                    self.advance();
303                    let mut entity_name = String::new();
304                    while let Some(character) = self.chars.get(self.pos).copied() {
305                        if character == ';' {
306                            break;
307                        }
308                        if !(character.is_ascii_alphanumeric() || character == '_') {
309                            return Err(OpyError::at(
310                                "invalid-string-entity",
311                                format!("invalid character '{character}' in string entity"),
312                                Span::new(self.file_id, escape_start.start, self.here(1).end),
313                            ));
314                        }
315                        entity_name.push(character);
316                        raw.push(character);
317                        self.advance();
318                    }
319                    if self.chars.get(self.pos) != Some(&';') {
320                        return Err(OpyError::at(
321                            "invalid-string-entity",
322                            "expected ';' to terminate string entity",
323                            Span::new(self.file_id, escape_start.start, self.here(0).start),
324                        ));
325                    }
326                    raw.push(';');
327                    self.advance();
328                    let Some(codepoint) = crate::string_entities::codepoint(&entity_name) else {
329                        return Err(OpyError::at(
330                            "unknown-string-entity",
331                            format!("unknown string entity '{entity_name}'"),
332                            Span::new(self.file_id, escape_start.start, self.here(0).start),
333                        ));
334                    };
335                    value.push(codepoint);
336                    continue;
337                }
338                if escaped == 'u' {
339                    self.advance();
340                    let mut codepoint = 0_u32;
341                    for _ in 0..4 {
342                        let Some(digit) = self.chars.get(self.pos).copied() else {
343                            return Err(OpyError::at(
344                                "lex-error",
345                                "Unicode escape requires four hexadecimal digits",
346                                Span::new(self.file_id, escape_start.start, self.here(0).start),
347                            ));
348                        };
349                        let Some(digit_value) = digit.to_digit(16) else {
350                            return Err(OpyError::at(
351                                "lex-error",
352                                "Unicode escape requires four hexadecimal digits",
353                                Span::new(self.file_id, escape_start.start, self.here(1).end),
354                            ));
355                        };
356                        raw.push(digit);
357                        codepoint = codepoint * 16 + digit_value;
358                        self.advance();
359                    }
360                    let Some(decoded) = char::from_u32(codepoint) else {
361                        return Err(OpyError::at(
362                            "lex-error",
363                            "Unicode escape does not name a Unicode scalar value",
364                            Span::new(self.file_id, escape_start.start, self.here(0).start),
365                        ));
366                    };
367                    value.push(decoded);
368                    continue;
369                }
370                value.push(decode_string_escape(escaped));
371                self.advance();
372                continue;
373            }
374            if ch == '\n' {
375                return Err(OpyError::at(
376                    "lex-error",
377                    "unterminated string literal",
378                    start,
379                ));
380            }
381            raw.push(ch);
382            value.push(ch);
383            self.advance();
384        }
385        Err(OpyError::at(
386            "lex-error",
387            "unterminated string literal",
388            start,
389        ))
390    }
391
392    fn skip_line_continuation(&mut self) -> bool {
393        let mut offset = 1;
394        while matches!(self.peek(offset), Some(' ' | '\r')) {
395            offset += 1;
396        }
397        if self.peek(offset) != Some('\n') {
398            return false;
399        }
400        for _ in 0..=offset {
401            self.advance();
402        }
403        self.line += 1;
404        self.col = 1;
405        true
406    }
407
408    fn lex_number(&mut self) -> OpyResult<()> {
409        let start = self.here(1);
410        let mut text = String::new();
411        if self.chars[self.pos] == '0'
412            && matches!(self.peek(1), Some('x' | 'X' | 'b' | 'B' | 'o' | 'O'))
413        {
414            text.push('0');
415            self.advance();
416            let letter = self.chars[self.pos];
417            text.push(letter);
418            self.advance();
419            let radix = match letter {
420                'x' | 'X' => 16,
421                'o' | 'O' => 8,
422                _ => 2,
423            };
424            let digits_start = self.pos;
425            while self.pos < self.chars.len() && self.chars[self.pos].is_digit(radix) {
426                text.push(self.chars[self.pos]);
427                self.advance();
428            }
429            if self.pos == digits_start {
430                return Err(OpyError::at(
431                    "lex-error",
432                    "integer radix literal requires at least one digit",
433                    Span::new(self.file_id, start.start, self.here(0).start),
434                ));
435            }
436            self.push_token(TokenKind::Number, text, start);
437            return Ok(());
438        }
439        while self.pos < self.chars.len() && self.chars[self.pos].is_ascii_digit() {
440            text.push(self.chars[self.pos]);
441            self.advance();
442        }
443        if self.pos < self.chars.len()
444            && self.chars[self.pos] == '.'
445            && self.peek(1).is_some_and(|c| c.is_ascii_digit())
446        {
447            text.push('.');
448            self.advance();
449            while self.pos < self.chars.len() && self.chars[self.pos].is_ascii_digit() {
450                text.push(self.chars[self.pos]);
451                self.advance();
452            }
453        }
454        // Optional exponent (not exercised by the corpus, supported for
455        // completeness of the number surface).
456        if self.pos < self.chars.len()
457            && (self.chars[self.pos] == 'e' || self.chars[self.pos] == 'E')
458        {
459            let mut lookahead = self.pos + 1;
460            if lookahead < self.chars.len()
461                && (self.chars[lookahead] == '+' || self.chars[lookahead] == '-')
462            {
463                lookahead += 1;
464            }
465            if lookahead < self.chars.len() && self.chars[lookahead].is_ascii_digit() {
466                text.push('e');
467                self.advance();
468                if self.pos < self.chars.len()
469                    && (self.chars[self.pos] == '+' || self.chars[self.pos] == '-')
470                {
471                    text.push(self.chars[self.pos]);
472                    self.advance();
473                }
474                while self.pos < self.chars.len() && self.chars[self.pos].is_ascii_digit() {
475                    text.push(self.chars[self.pos]);
476                    self.advance();
477                }
478            }
479        }
480        self.push_token(TokenKind::Number, text, start);
481        Ok(())
482    }
483
484    fn lex_ident(&mut self) {
485        let start = self.here(1);
486        let mut text = String::new();
487        while self.pos < self.chars.len() && is_ident_continue(self.chars[self.pos]) {
488            text.push(self.chars[self.pos]);
489            self.advance();
490        }
491        self.push_token(TokenKind::Ident, text, start);
492    }
493
494    fn single(&mut self, kind: TokenKind) {
495        let start = self.here(1);
496        let text = self.chars[self.pos].to_string();
497        self.advance();
498        self.push_token(kind, text, start);
499    }
500
501    /// Two-char operator where the second char may be `=`.
502    fn lex_two(&mut self, plain: TokenKind, assign: TokenKind, second: char) {
503        let start = self.here(1);
504        let (kind, text) = if self.peek(1) == Some(second) {
505            self.advance();
506            let text = format!("{}{}", self.chars[self.pos - 1], second);
507            self.advance();
508            (assign, text)
509        } else {
510            let text = self.chars[self.pos].to_string();
511            self.advance();
512            (plain, text)
513        };
514        self.push_token(kind, text, start);
515    }
516
517    fn lex_duplicate(&mut self, kind: TokenKind, text: &str) {
518        let start = self.here(1);
519        self.advance();
520        self.advance();
521        self.push_token(kind, text, start);
522    }
523
524    fn make_token(&self, kind: TokenKind, text: impl Into<String>, start: Span) -> Token {
525        Token::new(
526            kind,
527            text,
528            Span::new(self.file_id, start.start, self.here(0).start),
529        )
530    }
531
532    fn push_token(&mut self, kind: TokenKind, text: impl Into<String>, start: Span) {
533        let token = self.make_token(kind, text, start);
534        self.tokens.push(token);
535    }
536
537    fn here(&self, width: usize) -> Span {
538        Span::new(
539            self.file_id,
540            Position::new(self.line, self.col),
541            Position::new(self.line, self.col + width as u32),
542        )
543    }
544
545    fn peek(&self, offset: usize) -> Option<char> {
546        self.chars.get(self.pos + offset).copied()
547    }
548
549    fn advance(&mut self) {
550        self.pos += 1;
551        self.col += 1;
552    }
553}
554
555/// Whether `text` is a well-formed OPY identifier: an identifier start
556/// character followed by identifier continuation characters.
557pub fn is_identifier(text: &str) -> bool {
558    let mut characters = text.chars();
559    characters.next().is_some_and(is_ident_start) && characters.all(is_ident_continue)
560}
561
562pub(crate) fn decode_string_escape(character: char) -> char {
563    match character {
564        'n' => '\n',
565        't' => '\t',
566        'r' => '\r',
567        '\\' => '\\',
568        '"' => '"',
569        '\'' => '\'',
570        other => other,
571    }
572}
573
574/// Whether `c` may start an OPY identifier.
575pub fn is_ident_start(c: char) -> bool {
576    c.is_ascii_alphabetic() || c == '_'
577}
578
579/// Whether `c` may continue an OPY identifier.
580pub fn is_ident_continue(c: char) -> bool {
581    c.is_ascii_alphanumeric() || c == '_'
582}
583
584#[cfg(test)]
585mod tests {
586    use super::*;
587
588    fn lex_ok(text: &str) -> Vec<Token> {
589        lex(LexInput { file_id: 0, text }).unwrap()
590    }
591
592    #[test]
593    fn lexes_basic_rule() {
594        let tokens = lex_ok("rule \"setup\":\n    @Event global\n    disableInspector()\n");
595        let kinds: Vec<TokenKind> = tokens.iter().map(|t| t.kind).collect();
596        assert!(kinds.contains(&TokenKind::Ident));
597        assert!(kinds.contains(&TokenKind::String));
598        assert!(kinds.contains(&TokenKind::Colon));
599        assert!(kinds.contains(&TokenKind::At));
600        assert!(kinds.contains(&TokenKind::LParen));
601        assert!(kinds.contains(&TokenKind::Eof));
602    }
603
604    #[test]
605    fn numbers_preserve_text() {
606        let tokens = lex_ok("1 2.5 0.016 100");
607        let numbers: Vec<&str> = tokens
608            .iter()
609            .filter(|t| t.kind == TokenKind::Number)
610            .map(|t| t.text.as_str())
611            .collect();
612        assert_eq!(numbers, vec!["1", "2.5", "0.016", "100"]);
613    }
614
615    #[test]
616    fn radix_literals_lex_as_single_numbers() {
617        // The pinned OverPy tokenizer reads `0x`/`0b`/`0o` integer literals
618        // like JavaScript; they must stay one token so a settings block
619        // re-render keeps the spelling intact (opy-rs#496).
620        let tokens = lex_ok("0x1F 0b101 0o17 0X1f 0B101 0O17");
621        let numbers: Vec<&str> = tokens
622            .iter()
623            .filter(|t| t.kind == TokenKind::Number)
624            .map(|t| t.text.as_str())
625            .collect();
626        assert_eq!(
627            numbers,
628            vec!["0x1F", "0b101", "0o17", "0X1f", "0B101", "0O17"]
629        );
630    }
631
632    #[test]
633    fn radix_literal_without_digits_is_an_error() {
634        for text in ["0x", "0b", "0o"] {
635            let error = lex(LexInput { file_id: 0, text }).unwrap_err();
636            assert_eq!(error.code, "lex-error");
637        }
638    }
639
640    #[test]
641    fn directives_and_comments() {
642        let tokens = lex_ok("#!define X 1\n# comment\nrule \"r\":\n");
643        let directive = tokens
644            .iter()
645            .find(|t| t.kind == TokenKind::Directive)
646            .unwrap();
647        assert_eq!(directive.text, "define X 1");
648        assert!(!tokens.iter().any(|t| t.text == "comment"));
649    }
650
651    #[test]
652    fn directive_line_continuation_is_part_of_one_directive() {
653        let tokens = lex_ok("#!define X first + \\\n  second\n");
654        let directive = tokens
655            .iter()
656            .find(|token| token.kind == TokenKind::Directive)
657            .unwrap();
658        assert_eq!(directive.text, "define X first + \n  second");
659        assert_eq!(directive.span.start, Position::new(1, 1));
660        assert_eq!(directive.span.end, Position::new(2, 9));
661    }
662
663    #[test]
664    fn operators() {
665        let tokens = lex_ok("a += b == c <= d != e / f");
666        let kinds: Vec<TokenKind> = tokens.iter().map(|t| t.kind).collect();
667        for expected in [
668            TokenKind::PlusAssign,
669            TokenKind::Eq,
670            TokenKind::Le,
671            TokenKind::Ne,
672            TokenKind::Slash,
673        ] {
674            assert!(
675                kinds.contains(&expected),
676                "missing {expected:?} in {kinds:?}"
677            );
678        }
679    }
680
681    #[test]
682    fn power_operators_disambiguate() {
683        // The pinned OverPy 9.7.10 reference lexes `**=` as one power-assign
684        // operator distinct from `**` and `*=`.
685        let tokens = lex_ok("a **= b ** c *= d");
686        let kinds: Vec<TokenKind> = tokens.iter().map(|t| t.kind).collect();
687        assert_eq!(
688            kinds,
689            vec![
690                TokenKind::Ident,
691                TokenKind::DoubleStarAssign,
692                TokenKind::Ident,
693                TokenKind::DoubleStar,
694                TokenKind::Ident,
695                TokenKind::StarAssign,
696                TokenKind::Ident,
697                TokenKind::Eof,
698            ]
699        );
700        let assign = tokens
701            .iter()
702            .find(|t| t.kind == TokenKind::DoubleStarAssign)
703            .unwrap();
704        assert_eq!(assign.text, "**=");
705    }
706
707    #[test]
708    fn postfix_operators_are_single_tokens() {
709        let tokens = lex_ok("counter++ points--");
710        assert_eq!(
711            tokens.iter().map(|token| token.kind).collect::<Vec<_>>(),
712            vec![
713                TokenKind::Ident,
714                TokenKind::Increment,
715                TokenKind::Ident,
716                TokenKind::Decrement,
717                TokenKind::Eof,
718            ]
719        );
720        assert_eq!(tokens[1].span.start.col, 8);
721        assert_eq!(tokens[1].span.end.col, 10);
722    }
723
724    #[test]
725    fn unterminated_string_is_structured() {
726        let error = lex(LexInput {
727            file_id: 0,
728            text: "rule \"x\n",
729        })
730        .unwrap_err();
731        assert_eq!(error.code, "lex-error");
732        assert!(error.span.is_some());
733    }
734
735    #[test]
736    fn unicode_escapes_decode_at_the_string_boundary_and_keep_source_span() {
737        // Minimized from OWBastion/Bastion commit
738        // 44e12e08f046ed7fbf865237a29ddf2f272ec3c2,
739        // Bastion/src/utilities/system/savePlayerData.opy.
740        let tokens = lex_ok(r#"rule "pa\ufeffssed":"#);
741        let string = tokens
742            .iter()
743            .find(|token| token.kind == TokenKind::String)
744            .expect("rule name string token");
745        assert_eq!(string.text, "pa\u{feff}ssed");
746        assert_eq!(string.raw.as_deref(), Some(r"pa\ufeffssed"));
747        assert_eq!(string.span.start, Position::new(1, 6));
748        assert_eq!(string.span.end, Position::new(1, 20));
749    }
750
751    #[test]
752    fn existing_string_escapes_keep_their_decoded_values() {
753        let source = concat!("\"", "\\n", "\\t", "\\r", "\\\"", "\\\\", "\"");
754        let string = lex_ok(source)
755            .into_iter()
756            .find(|token| token.kind == TokenKind::String)
757            .expect("string token");
758        assert_eq!(string.text, "\n\t\r\"\\");
759    }
760
761    #[test]
762    fn named_string_entities_decode_with_raw_provenance() {
763        let string = lex_ok(r#""a\&black_square;b\&fullwidth_space;c""#)
764            .into_iter()
765            .find(|token| token.kind == TokenKind::String)
766            .expect("string token");
767        assert_eq!(string.text, "a■b　c");
768        assert_eq!(
769            string.raw.as_deref(),
770            Some(r"a\&black_square;b\&fullwidth_space;c")
771        );
772    }
773
774    #[test]
775    fn invalid_string_entities_are_structured_lex_errors() {
776        for (source, code) in [
777            (r#""\&missing;""#, "unknown-string-entity"),
778            (r#""\&black-square;""#, "invalid-string-entity"),
779            (r#""\&black_square""#, "invalid-string-entity"),
780        ] {
781            let error = lex(LexInput {
782                file_id: 0,
783                text: source,
784            })
785            .expect_err("invalid string entity unexpectedly lexed");
786            assert_eq!(error.code, code);
787            assert_eq!(error.span.unwrap().start, Position::new(1, 2));
788        }
789    }
790
791    #[test]
792    fn malformed_unicode_escapes_are_lex_errors() {
793        for source in [r#""\u"#, r#""\u12G4"#, r#""\u{1F600}"#] {
794            let error = lex(LexInput {
795                file_id: 0,
796                text: source,
797            })
798            .expect_err("malformed Unicode escape unexpectedly lexed");
799            assert_eq!(error.code, "lex-error");
800            assert!(error.message.contains("Unicode escape"));
801            assert_eq!(error.span.unwrap().start, Position::new(1, 2));
802        }
803    }
804
805    #[test]
806    fn backslash_line_continuation_is_not_a_token() {
807        let tokens = lex_ok("one \\\ntwo");
808        assert_eq!(
809            tokens.iter().map(|token| token.kind).collect::<Vec<_>>(),
810            vec![TokenKind::Ident, TokenKind::Ident, TokenKind::Eof]
811        );
812        assert_eq!(tokens[1].span.start.line, 2);
813        assert_eq!(tokens[1].span.start.col, 1);
814    }
815
816    #[test]
817    fn crlf_line_continuation_tracks_the_next_line() {
818        let tokens = lex_ok("one \\\r\ntwo");
819        assert_eq!(tokens[1].span.start, Position::new(2, 1));
820    }
821
822    #[test]
823    fn whitespace_before_line_ending_is_part_of_the_continuation() {
824        let tokens = lex_ok("one \\  \ntwo");
825        assert_eq!(tokens[1].span.start, Position::new(2, 1));
826    }
827
828    #[test]
829    fn non_newline_backslash_remains_a_lex_error() {
830        for text in ["one \\ two", "one \\", "one \\ \t\ntwo"] {
831            let error = lex(LexInput { file_id: 0, text }).unwrap_err();
832            assert_eq!(error.code, "lex-error");
833            assert_eq!(error.message, "unexpected character '\\'");
834            assert_eq!(error.span.unwrap().start, Position::new(1, 5));
835        }
836    }
837}
opy_rs/lexer.rs

opy_rs/
lexer.rs
