opy_rs/
lexer.rs

1//! The native `.opy` lexer.
2//!
3//! Produces a flat token stream (newlines and indentation included) from one
4//! source file. Comments (`#`, `/* */`) are skipped; `#!` directives are
5//! captured as a single directive token for the preprocessor. Positions are
6//! 1-based line/column, matching the Opy HIR protocol.
7
8use crate::diag::{OpyError, OpyResult, Position, Span};
9
10/// The kind of a token.
11#[derive(Debug, Clone, Copy, PartialEq, Eq)]
12pub enum TokenKind {
13    /// An identifier or keyword (keywords are resolved by the parser).
14    Ident,
15    /// A numeric literal (`text` holds the source spelling).
16    Number,
17    /// A string literal (`text` holds the unescaped value).
18    String,
19    /// A `#!` directive line (`text` holds everything after `#!`).
20    Directive,
21    /// A preprocessing marker carrying the rule-prefix state active at the
22    /// following top-level rule or subroutine.
23    RulePrefixMarker,
24    /// `@Event` / `@Condition` / other `@` directives.
25    At,
26    Newline,
27    /// Indentation change: the column of the current line.
28    Indent(u32),
29    /// End of file.
30    Eof,
31    // Punctuation and operators.
32    LParen,
33    RParen,
34    LBracket,
35    RBracket,
36    LBrace,
37    RBrace,
38    Comma,
39    Colon,
40    Semicolon,
41    Dot,
42    Assign,
43    Plus,
44    Minus,
45    Star,
46    Slash,
47    Percent,
48    DoubleStar,
49    PlusAssign,
50    MinusAssign,
51    Increment,
52    Decrement,
53    StarAssign,
54    SlashAssign,
55    PercentAssign,
56    DoubleStarAssign,
57    Eq,
58    Ne,
59    Lt,
60    Le,
61    Gt,
62    Ge,
63    /// A bare `!` that is not `!=`; the parser rejects it as unsupported OPY.
64    LexBang,
65}
66
67/// One token with its source span and payload text.
68#[derive(Debug, Clone, PartialEq)]
69pub struct Token {
70    pub kind: TokenKind,
71    /// The source text of this token (numbers keep their spelling; strings
72    /// keep their unescaped value; identifiers keep their name).
73    pub text: String,
74    /// The exact characters between string quotes, before escape decoding.
75    /// Other token kinds leave this unset. It is retained so source-language
76    /// constructs such as f-string interpolations can recover expression
77    /// spans without losing provenance during preprocessing.
78    pub raw: Option<String>,
79    /// Authored-source provenance. Tokens produced by macro or script
80    /// expansion carry the use-site span: the expanded text does not exist
81    /// in authored source, so the invocation is the only honest position.
82    pub span: Span,
83    /// The position the indentation-sensitive parser reads for layout
84    /// decisions. It equals `span` for authored tokens; expanded tokens keep
85    /// the expansion's relative line and column structure here so blocks and
86    /// statement boundaries still resolve (#506).
87    pub(crate) layout: Span,
88}
89
90impl Token {
91    fn new(kind: TokenKind, text: impl Into<String>, span: Span) -> Token {
92        Token {
93            kind,
94            text: text.into(),
95            raw: None,
96            span,
97            layout: span,
98        }
99    }
100}
101
102/// The lexer input: one file's text with its file id.
103pub struct LexInput<'a> {
104    pub file_id: u32,
105    pub text: &'a str,
106}
107
108/// Lex one source file into a token stream.
109pub fn lex(input: LexInput<'_>) -> OpyResult<Vec<Token>> {
110    Lexer::new(input.file_id, input.text).run()
111}
112
113struct Lexer {
114    file_id: u32,
115    chars: Vec<char>,
116    pos: usize,
117    line: u32,
118    col: u32,
119    tokens: Vec<Token>,
120}
121
122impl Lexer {
123    fn new(file_id: u32, text: &str) -> Lexer {
124        Lexer {
125            file_id,
126            chars: text.chars().collect(),
127            pos: 0,
128            line: 1,
129            col: 1,
130            tokens: Vec::new(),
131        }
132    }
133
134    fn run(mut self) -> OpyResult<Vec<Token>> {
135        while self.pos < self.chars.len() {
136            let ch = self.chars[self.pos];
137            match ch {
138                '\n' => {
139                    self.tokens
140                        .push(Token::new(TokenKind::Newline, "\n", self.here(1)));
141                    self.advance();
142                    self.line += 1;
143                    self.col = 1;
144                }
145                ' ' | '\r' => {
146                    self.advance();
147                }
148                '\t' => {
149                    self.pos += 1;
150                    self.col += 4;
151                }
152                '\\' => {
153                    if !self.skip_line_continuation() {
154                        return Err(OpyError::at(
155                            "lex-error",
156                            "unexpected character '\\'",
157                            self.here(1),
158                        ));
159                    }
160                }
161                '#' => self.lex_hash(),
162                '/' if self.peek(1) == Some('*') => self.skip_block_comment()?,
163                '"' | '\'' => self.lex_string(ch)?,
164                c if c.is_ascii_digit() => self.lex_number()?,
165                c if is_ident_start(c) => self.lex_ident(),
166                '(' => self.single(TokenKind::LParen),
167                ')' => self.single(TokenKind::RParen),
168                '[' => self.single(TokenKind::LBracket),
169                ']' => self.single(TokenKind::RBracket),
170                '{' => self.single(TokenKind::LBrace),
171                '}' => self.single(TokenKind::RBrace),
172                ',' => self.single(TokenKind::Comma),
173                ':' => self.single(TokenKind::Colon),
174                ';' => self.single(TokenKind::Semicolon),
175                '.' => self.single(TokenKind::Dot),
176                '@' => self.single(TokenKind::At),
177                '=' => self.lex_two(TokenKind::Assign, TokenKind::Eq, '='),
178                '+' => {
179                    if self.peek(1) == Some('+') {
180                        self.lex_duplicate(TokenKind::Increment, "++");
181                    } else {
182                        self.lex_two(TokenKind::Plus, TokenKind::PlusAssign, '=');
183                    }
184                }
185                '-' => {
186                    if self.peek(1) == Some('-') {
187                        self.lex_duplicate(TokenKind::Decrement, "--");
188                    } else {
189                        self.lex_two(TokenKind::Minus, TokenKind::MinusAssign, '=');
190                    }
191                }
192                '*' => {
193                    if self.peek(1) == Some('*') {
194                        if self.peek(2) == Some('=') {
195                            let start = self.here(3);
196                            self.advance();
197                            self.advance();
198                            self.advance();
199                            self.push_token(TokenKind::DoubleStarAssign, "**=", start);
200                        } else {
201                            let start = self.here(2);
202                            self.advance();
203                            self.advance();
204                            self.push_token(TokenKind::DoubleStar, "**", start);
205                        }
206                    } else {
207                        self.lex_two(TokenKind::Star, TokenKind::StarAssign, '=')
208                    }
209                }
210                '/' => self.lex_two(TokenKind::Slash, TokenKind::SlashAssign, '='),
211                '%' => self.lex_two(TokenKind::Percent, TokenKind::PercentAssign, '='),
212                '<' => self.lex_two(TokenKind::Lt, TokenKind::Le, '='),
213                '>' => self.lex_two(TokenKind::Gt, TokenKind::Ge, '='),
214                '!' => self.lex_two(TokenKind::LexBang, TokenKind::Ne, '='),
215                other => {
216                    return Err(OpyError::at(
217                        "lex-error",
218                        format!("unexpected character '{other}'"),
219                        self.here(1),
220                    ));
221                }
222            }
223        }
224        let here = self.here(0);
225        self.push_token(TokenKind::Eof, "", here);
226        Ok(self.tokens)
227    }
228
229    /// `#` starts a `#!` directive (captured as one token) or a comment.
230    fn lex_hash(&mut self) {
231        if self.peek(1) == Some('!') {
232            let start = self.here(2);
233            self.advance();
234            self.advance();
235            let mut text = String::new();
236            while self.pos < self.chars.len() {
237                if self.chars[self.pos] == '\\' && self.skip_line_continuation() {
238                    text.push('\n');
239                    continue;
240                }
241                if self.chars[self.pos] == '\n' {
242                    break;
243                }
244                text.push(self.chars[self.pos]);
245                self.advance();
246            }
247            self.push_token(TokenKind::Directive, text, start);
248        } else {
249            while self.pos < self.chars.len() && self.chars[self.pos] != '\n' {
250                self.advance();
251            }
252        }
253    }
254
255    fn skip_block_comment(&mut self) -> OpyResult<()> {
256        let start = self.here(2);
257        self.advance();
258        self.advance();
259        while self.pos < self.chars.len() {
260            if self.chars[self.pos] == '*' && self.peek(1) == Some('/') {
261                self.advance();
262                self.advance();
263                return Ok(());
264            }
265            if self.chars[self.pos] == '\n' {
266                self.advance();
267                self.line += 1;
268                self.col = 1;
269            } else {
270                self.advance();
271            }
272        }
273        Err(OpyError::at(
274            "lex-error",
275            "unterminated block comment",
276            start,
277        ))
278    }
279
280    fn lex_string(&mut self, quote: char) -> OpyResult<()> {
281        let start = self.here(1);
282        self.advance();
283        let mut value = String::new();
284        let mut raw = String::new();
285        while self.pos < self.chars.len() {
286            let ch = self.chars[self.pos];
287            if ch == quote {
288                self.advance();
289                let mut token = self.make_token(TokenKind::String, value, start);
290                token.raw = Some(raw);
291                self.tokens.push(token);
292                return Ok(());
293            }
294            if ch == '\\' {
295                let escape_start = self.here(1);
296                raw.push(ch);
297                self.advance();
298                if self.pos >= self.chars.len() {
299                    break;
300                }
301                let escaped = self.chars[self.pos];
302                raw.push(escaped);
303                if escaped == '&' {
304                    self.advance();
305                    let mut entity_name = String::new();
306                    while let Some(character) = self.chars.get(self.pos).copied() {
307                        if character == ';' {
308                            break;
309                        }
310                        if !(character.is_ascii_alphanumeric() || character == '_') {
311                            return Err(OpyError::at(
312                                "invalid-string-entity",
313                                format!("invalid character '{character}' in string entity"),
314                                Span::new(self.file_id, escape_start.start, self.here(1).end),
315                            ));
316                        }
317                        entity_name.push(character);
318                        raw.push(character);
319                        self.advance();
320                    }
321                    if self.chars.get(self.pos) != Some(&';') {
322                        return Err(OpyError::at(
323                            "invalid-string-entity",
324                            "expected ';' to terminate string entity",
325                            Span::new(self.file_id, escape_start.start, self.here(0).start),
326                        ));
327                    }
328                    raw.push(';');
329                    self.advance();
330                    let Some(codepoint) = crate::string_entities::codepoint(&entity_name) else {
331                        return Err(OpyError::at(
332                            "unknown-string-entity",
333                            format!("unknown string entity '{entity_name}'"),
334                            Span::new(self.file_id, escape_start.start, self.here(0).start),
335                        ));
336                    };
337                    value.push(codepoint);
338                    continue;
339                }
340                if escaped == 'u' {
341                    self.advance();
342                    let mut codepoint = 0_u32;
343                    for _ in 0..4 {
344                        let Some(digit) = self.chars.get(self.pos).copied() else {
345                            return Err(OpyError::at(
346                                "lex-error",
347                                "Unicode escape requires four hexadecimal digits",
348                                Span::new(self.file_id, escape_start.start, self.here(0).start),
349                            ));
350                        };
351                        let Some(digit_value) = digit.to_digit(16) else {
352                            return Err(OpyError::at(
353                                "lex-error",
354                                "Unicode escape requires four hexadecimal digits",
355                                Span::new(self.file_id, escape_start.start, self.here(1).end),
356                            ));
357                        };
358                        raw.push(digit);
359                        codepoint = codepoint * 16 + digit_value;
360                        self.advance();
361                    }
362                    // JS strings are UTF-16 and hold lone surrogates;
363                    // every observable downstream use (indexing, emission)
364                    // sees them as U+FFFD, so decode them to it directly.
365                    value.push(char::from_u32(codepoint).unwrap_or('\u{FFFD}'));
366                    continue;
367                }
368                value.push(decode_string_escape(escaped));
369                self.advance();
370                continue;
371            }
372            if ch == '\n' {
373                return Err(OpyError::at(
374                    "lex-error",
375                    "unterminated string literal",
376                    start,
377                ));
378            }
379            raw.push(ch);
380            value.push(ch);
381            self.advance();
382        }
383        Err(OpyError::at(
384            "lex-error",
385            "unterminated string literal",
386            start,
387        ))
388    }
389
390    fn skip_line_continuation(&mut self) -> bool {
391        let mut offset = 1;
392        while matches!(self.peek(offset), Some(' ' | '\r')) {
393            offset += 1;
394        }
395        if self.peek(offset) != Some('\n') {
396            return false;
397        }
398        for _ in 0..=offset {
399            self.advance();
400        }
401        self.line += 1;
402        self.col = 1;
403        true
404    }
405
406    fn lex_number(&mut self) -> OpyResult<()> {
407        let start = self.here(1);
408        let mut text = String::new();
409        let mut signed_exponent = false;
410        if self.chars[self.pos] == '0'
411            && matches!(self.peek(1), Some('x' | 'X' | 'b' | 'B' | 'o' | 'O'))
412        {
413            text.push('0');
414            self.advance();
415            let letter = self.chars[self.pos];
416            text.push(letter);
417            self.advance();
418            let radix = match letter {
419                'x' | 'X' => 16,
420                'o' | 'O' => 8,
421                _ => 2,
422            };
423            while self.pos < self.chars.len() && self.chars[self.pos].is_digit(radix) {
424                text.push(self.chars[self.pos]);
425                self.advance();
426            }
427        } else {
428            while self.pos < self.chars.len() && self.chars[self.pos].is_ascii_digit() {
429                text.push(self.chars[self.pos]);
430                self.advance();
431            }
432            if self.pos < self.chars.len()
433                && self.chars[self.pos] == '.'
434                && self.peek(1).is_some_and(|c| c.is_ascii_digit())
435            {
436                text.push('.');
437                self.advance();
438                while self.pos < self.chars.len() && self.chars[self.pos].is_ascii_digit() {
439                    text.push(self.chars[self.pos]);
440                    self.advance();
441                }
442            }
443            // Optional exponent (not exercised by the corpus, supported for
444            // completeness of the number surface).
445            if self.pos < self.chars.len()
446                && (self.chars[self.pos] == 'e' || self.chars[self.pos] == 'E')
447            {
448                let mut lookahead = self.pos + 1;
449                if lookahead < self.chars.len()
450                    && (self.chars[lookahead] == '+' || self.chars[lookahead] == '-')
451                {
452                    lookahead += 1;
453                }
454                if lookahead < self.chars.len() && self.chars[lookahead].is_ascii_digit() {
455                    text.push('e');
456                    self.advance();
457                    if self.pos < self.chars.len()
458                        && (self.chars[self.pos] == '+' || self.chars[self.pos] == '-')
459                    {
460                        text.push(self.chars[self.pos]);
461                        self.advance();
462                        signed_exponent = true;
463                    }
464                    while self.pos < self.chars.len() && self.chars[self.pos].is_ascii_digit() {
465                        text.push(self.chars[self.pos]);
466                        self.advance();
467                    }
468                }
469            }
470        }
471        // The pinned tokenizer reads a digit-led run of word characters as
472        // one token — `1_000`, `12abc`, `5e`, `0xZZ`, `0o18`, `12@x` — and
473        // `Number` converts the whole text later, so those spell `NaN`
474        // there (`isVarChar` admits `@`/`$` too; #512). A sign ends the
475        // word upstream, so `1e-7` stays split.
476        if !signed_exponent {
477            while self.pos < self.chars.len()
478                && (self.chars[self.pos].is_ascii_alphanumeric()
479                    || matches!(self.chars[self.pos], '_' | '@' | '$'))
480            {
481                text.push(self.chars[self.pos]);
482                self.advance();
483            }
484        }
485        self.push_token(TokenKind::Number, text, start);
486        Ok(())
487    }
488
489    fn lex_ident(&mut self) {
490        let start = self.here(1);
491        let mut text = String::new();
492        while self.pos < self.chars.len() && is_ident_continue(self.chars[self.pos]) {
493            text.push(self.chars[self.pos]);
494            self.advance();
495        }
496        self.push_token(TokenKind::Ident, text, start);
497    }
498
499    fn single(&mut self, kind: TokenKind) {
500        let start = self.here(1);
501        let text = self.chars[self.pos].to_string();
502        self.advance();
503        self.push_token(kind, text, start);
504    }
505
506    /// Two-char operator where the second char may be `=`.
507    fn lex_two(&mut self, plain: TokenKind, assign: TokenKind, second: char) {
508        let start = self.here(1);
509        let (kind, text) = if self.peek(1) == Some(second) {
510            self.advance();
511            let text = format!("{}{}", self.chars[self.pos - 1], second);
512            self.advance();
513            (assign, text)
514        } else {
515            let text = self.chars[self.pos].to_string();
516            self.advance();
517            (plain, text)
518        };
519        self.push_token(kind, text, start);
520    }
521
522    fn lex_duplicate(&mut self, kind: TokenKind, text: &str) {
523        let start = self.here(1);
524        self.advance();
525        self.advance();
526        self.push_token(kind, text, start);
527    }
528
529    fn make_token(&self, kind: TokenKind, text: impl Into<String>, start: Span) -> Token {
530        Token::new(
531            kind,
532            text,
533            Span::new(self.file_id, start.start, self.here(0).start),
534        )
535    }
536
537    fn push_token(&mut self, kind: TokenKind, text: impl Into<String>, start: Span) {
538        let token = self.make_token(kind, text, start);
539        self.tokens.push(token);
540    }
541
542    fn here(&self, width: usize) -> Span {
543        Span::new(
544            self.file_id,
545            Position::new(self.line, self.col),
546            Position::new(self.line, self.col + width as u32),
547        )
548    }
549
550    fn peek(&self, offset: usize) -> Option<char> {
551        self.chars.get(self.pos + offset).copied()
552    }
553
554    fn advance(&mut self) {
555        self.pos += 1;
556        self.col += 1;
557    }
558}
559
560/// Whether `text` is a well-formed OPY identifier: an identifier start
561/// character followed by identifier continuation characters.
562pub fn is_identifier(text: &str) -> bool {
563    let mut characters = text.chars();
564    characters.next().is_some_and(is_ident_start) && characters.all(is_ident_continue)
565}
566
567pub(crate) fn decode_string_escape(character: char) -> char {
568    match character {
569        'n' => '\n',
570        't' => '\t',
571        'r' => '\r',
572        '\\' => '\\',
573        '"' => '"',
574        '\'' => '\'',
575        other => other,
576    }
577}
578
579/// Whether `c` may start an OPY identifier.
580pub fn is_ident_start(c: char) -> bool {
581    c.is_ascii_alphabetic() || c == '_'
582}
583
584/// Whether `c` may continue an OPY identifier.
585pub fn is_ident_continue(c: char) -> bool {
586    c.is_ascii_alphanumeric() || c == '_'
587}
588
589#[cfg(test)]
590mod tests {
591    use super::*;
592
593    fn lex_ok(text: &str) -> Vec<Token> {
594        lex(LexInput { file_id: 0, text }).unwrap()
595    }
596
597    #[test]
598    fn lexes_basic_rule() {
599        let tokens = lex_ok("rule \"setup\":\n    @Event global\n    disableInspector()\n");
600        let kinds: Vec<TokenKind> = tokens.iter().map(|t| t.kind).collect();
601        assert!(kinds.contains(&TokenKind::Ident));
602        assert!(kinds.contains(&TokenKind::String));
603        assert!(kinds.contains(&TokenKind::Colon));
604        assert!(kinds.contains(&TokenKind::At));
605        assert!(kinds.contains(&TokenKind::LParen));
606        assert!(kinds.contains(&TokenKind::Eof));
607    }
608
609    #[test]
610    fn numbers_preserve_text() {
611        let tokens = lex_ok("1 2.5 0.016 100");
612        let numbers: Vec<&str> = tokens
613            .iter()
614            .filter(|t| t.kind == TokenKind::Number)
615            .map(|t| t.text.as_str())
616            .collect();
617        assert_eq!(numbers, vec!["1", "2.5", "0.016", "100"]);
618    }
619
620    #[test]
621    fn radix_literals_lex_as_single_numbers() {
622        // The pinned OverPy tokenizer reads `0x`/`0b`/`0o` integer literals
623        // like JavaScript; they must stay one token so a settings block
624        // re-render keeps the spelling intact (opy-rs#496).
625        let tokens = lex_ok("0x1F 0b101 0o17 0X1f 0B101 0O17");
626        let numbers: Vec<&str> = tokens
627            .iter()
628            .filter(|t| t.kind == TokenKind::Number)
629            .map(|t| t.text.as_str())
630            .collect();
631        assert_eq!(
632            numbers,
633            vec!["0x1F", "0b101", "0o17", "0X1f", "0B101", "0O17"]
634        );
635    }
636
637    #[test]
638    fn radix_literal_without_digits_is_a_word_token() {
639        // The pinned tokenizer reads `0x`/`0b`/`0o` as single word tokens
640        // that `Number` converts to `NaN`; they are not lexer errors.
641        for text in ["0x", "0b", "0o"] {
642            let tokens = lex_ok(text);
643            assert_eq!(tokens[0].kind, TokenKind::Number);
644            assert_eq!(tokens[0].text, text);
645        }
646    }
647
648    #[test]
649    fn directives_and_comments() {
650        let tokens = lex_ok("#!define X 1\n# comment\nrule \"r\":\n");
651        let directive = tokens
652            .iter()
653            .find(|t| t.kind == TokenKind::Directive)
654            .unwrap();
655        assert_eq!(directive.text, "define X 1");
656        assert!(!tokens.iter().any(|t| t.text == "comment"));
657    }
658
659    #[test]
660    fn directive_line_continuation_is_part_of_one_directive() {
661        let tokens = lex_ok("#!define X first + \\\n  second\n");
662        let directive = tokens
663            .iter()
664            .find(|token| token.kind == TokenKind::Directive)
665            .unwrap();
666        assert_eq!(directive.text, "define X first + \n  second");
667        assert_eq!(directive.span.start, Position::new(1, 1));
668        assert_eq!(directive.span.end, Position::new(2, 9));
669    }
670
671    #[test]
672    fn operators() {
673        let tokens = lex_ok("a += b == c <= d != e / f");
674        let kinds: Vec<TokenKind> = tokens.iter().map(|t| t.kind).collect();
675        for expected in [
676            TokenKind::PlusAssign,
677            TokenKind::Eq,
678            TokenKind::Le,
679            TokenKind::Ne,
680            TokenKind::Slash,
681        ] {
682            assert!(
683                kinds.contains(&expected),
684                "missing {expected:?} in {kinds:?}"
685            );
686        }
687    }
688
689    #[test]
690    fn power_operators_disambiguate() {
691        // The pinned OverPy 9.7.10 reference lexes `**=` as one power-assign
692        // operator distinct from `**` and `*=`.
693        let tokens = lex_ok("a **= b ** c *= d");
694        let kinds: Vec<TokenKind> = tokens.iter().map(|t| t.kind).collect();
695        assert_eq!(
696            kinds,
697            vec![
698                TokenKind::Ident,
699                TokenKind::DoubleStarAssign,
700                TokenKind::Ident,
701                TokenKind::DoubleStar,
702                TokenKind::Ident,
703                TokenKind::StarAssign,
704                TokenKind::Ident,
705                TokenKind::Eof,
706            ]
707        );
708        let assign = tokens
709            .iter()
710            .find(|t| t.kind == TokenKind::DoubleStarAssign)
711            .unwrap();
712        assert_eq!(assign.text, "**=");
713    }
714
715    #[test]
716    fn postfix_operators_are_single_tokens() {
717        let tokens = lex_ok("counter++ points--");
718        assert_eq!(
719            tokens.iter().map(|token| token.kind).collect::<Vec<_>>(),
720            vec![
721                TokenKind::Ident,
722                TokenKind::Increment,
723                TokenKind::Ident,
724                TokenKind::Decrement,
725                TokenKind::Eof,
726            ]
727        );
728        assert_eq!(tokens[1].span.start.col, 8);
729        assert_eq!(tokens[1].span.end.col, 10);
730    }
731
732    #[test]
733    fn unterminated_string_is_structured() {
734        let error = lex(LexInput {
735            file_id: 0,
736            text: "rule \"x\n",
737        })
738        .unwrap_err();
739        assert_eq!(error.code, "lex-error");
740        assert!(error.span.is_some());
741    }
742
743    #[test]
744    fn unicode_escapes_decode_at_the_string_boundary_and_keep_source_span() {
745        // Minimized from OWBastion/Bastion commit
746        // 44e12e08f046ed7fbf865237a29ddf2f272ec3c2,
747        // Bastion/src/utilities/system/savePlayerData.opy.
748        let tokens = lex_ok(r#"rule "pa\ufeffssed":"#);
749        let string = tokens
750            .iter()
751            .find(|token| token.kind == TokenKind::String)
752            .expect("rule name string token");
753        assert_eq!(string.text, "pa\u{feff}ssed");
754        assert_eq!(string.raw.as_deref(), Some(r"pa\ufeffssed"));
755        assert_eq!(string.span.start, Position::new(1, 6));
756        assert_eq!(string.span.end, Position::new(1, 20));
757    }
758
759    #[test]
760    fn existing_string_escapes_keep_their_decoded_values() {
761        let source = concat!("\"", "\\n", "\\t", "\\r", "\\\"", "\\\\", "\"");
762        let string = lex_ok(source)
763            .into_iter()
764            .find(|token| token.kind == TokenKind::String)
765            .expect("string token");
766        assert_eq!(string.text, "\n\t\r\"\\");
767    }
768
769    #[test]
770    fn named_string_entities_decode_with_raw_provenance() {
771        let string = lex_ok(r#""a\&black_square;b\&fullwidth_space;c""#)
772            .into_iter()
773            .find(|token| token.kind == TokenKind::String)
774            .expect("string token");
775        assert_eq!(string.text, "a■b　c");
776        assert_eq!(
777            string.raw.as_deref(),
778            Some(r"a\&black_square;b\&fullwidth_space;c")
779        );
780    }
781
782    #[test]
783    fn invalid_string_entities_are_structured_lex_errors() {
784        for (source, code) in [
785            (r#""\&missing;""#, "unknown-string-entity"),
786            (r#""\&black-square;""#, "invalid-string-entity"),
787            (r#""\&black_square""#, "invalid-string-entity"),
788        ] {
789            let error = lex(LexInput {
790                file_id: 0,
791                text: source,
792            })
793            .expect_err("invalid string entity unexpectedly lexed");
794            assert_eq!(error.code, code);
795            assert_eq!(error.span.unwrap().start, Position::new(1, 2));
796        }
797    }
798
799    #[test]
800    fn malformed_unicode_escapes_are_lex_errors() {
801        for source in [r#""\u"#, r#""\u12G4"#, r#""\u{1F600}"#] {
802            let error = lex(LexInput {
803                file_id: 0,
804                text: source,
805            })
806            .expect_err("malformed Unicode escape unexpectedly lexed");
807            assert_eq!(error.code, "lex-error");
808            assert!(error.message.contains("Unicode escape"));
809            assert_eq!(error.span.unwrap().start, Position::new(1, 2));
810        }
811    }
812
813    #[test]
814    fn backslash_line_continuation_is_not_a_token() {
815        let tokens = lex_ok("one \\\ntwo");
816        assert_eq!(
817            tokens.iter().map(|token| token.kind).collect::<Vec<_>>(),
818            vec![TokenKind::Ident, TokenKind::Ident, TokenKind::Eof]
819        );
820        assert_eq!(tokens[1].span.start.line, 2);
821        assert_eq!(tokens[1].span.start.col, 1);
822    }
823
824    #[test]
825    fn crlf_line_continuation_tracks_the_next_line() {
826        let tokens = lex_ok("one \\\r\ntwo");
827        assert_eq!(tokens[1].span.start, Position::new(2, 1));
828    }
829
830    #[test]
831    fn whitespace_before_line_ending_is_part_of_the_continuation() {
832        let tokens = lex_ok("one \\  \ntwo");
833        assert_eq!(tokens[1].span.start, Position::new(2, 1));
834    }
835
836    #[test]
837    fn non_newline_backslash_remains_a_lex_error() {
838        for text in ["one \\ two", "one \\", "one \\ \t\ntwo"] {
839            let error = lex(LexInput { file_id: 0, text }).unwrap_err();
840            assert_eq!(error.code, "lex-error");
841            assert_eq!(error.message, "unexpected character '\\'");
842            assert_eq!(error.span.unwrap().start, Position::new(1, 5));
843        }
844    }
845}
opy_rs/lexer.rs

opy_rs/
lexer.rs
