Skip to main content

runec/
lexer.rs

1use crate::error::LexingError;
2use crate::token::{ESCAPED_LT, Kind, Token};
3use crate::types::Type;
4use std::path::PathBuf;
5
6pub struct Lexer<'a> {
7    source: &'a [u8],
8    file_name: &'a PathBuf,
9    pos: usize,
10    line: usize,
11    col: usize,
12}
13
14impl<'a> Lexer<'a> {
15    pub fn new(input: &'a str, file_name: &'a PathBuf) -> Self {
16        Self {
17            source: input.as_bytes(),
18            file_name,
19            pos: 0,
20            line: 1,
21            col: 1,
22        }
23    }
24
25    fn peek(&self) -> Option<u8> {
26        self.source.get(self.pos).copied()
27    }
28
29    fn peek_ahead(&self, offset: usize) -> Option<u8> {
30        self.source.get(self.pos + offset).copied()
31    }
32
33    fn advance(&mut self) -> Option<u8> {
34        let ch = self.source.get(self.pos).copied()?;
35        self.pos += 1;
36        if ch == b'\n' {
37            self.line += 1;
38            self.col = 1;
39        } else {
40            self.col += 1;
41        }
42        Some(ch)
43    }
44
45    fn token(&self, kind: Kind, value: String, line: usize, col: usize) -> Token {
46        Token::new(kind, value, line, col)
47    }
48
49    fn skip_whitespace(&mut self) {
50        while let Some(ch) = self.peek() {
51            if ch.is_ascii_whitespace() {
52                self.advance();
53            } else {
54                break;
55            }
56        }
57    }
58
59    fn read_string_literal(&mut self, raw: bool) -> Result<Token, LexingError> {
60        let start_line = self.line;
61        let start_col = self.col;
62        self.advance(); // consume opening quote
63        let mut value = String::new();
64        let mut interp_depth: u32 = 0; // depth inside <...> interpolation blocks
65        loop {
66            match self.advance() {
67                Some(b'"') => {
68                    // Inside an interpolation block, embedded quotes are allowed
69                    if interp_depth > 0 {
70                        value.push('"');
71                    } else {
72                        break;
73                    }
74                }
75                Some(b'<') if !raw => {
76                    interp_depth += 1;
77                    value.push('<');
78                }
79                Some(b'>') if !raw => {
80                    interp_depth = interp_depth.saturating_sub(1);
81                    value.push('>');
82                }
83                Some(b'\\') => match self.advance() {
84                    // RuneScript only escapes \\, \", and \< (matching ANTLR grammar)
85                    Some(b'\\') => value.push('\\'),
86                    Some(b'"') => value.push('"'),
87                    Some(b'<') => value.push(if raw { '<' } else { ESCAPED_LT }),
88                    Some(ch) => {
89                        value.push('\\');
90                        value.push(ch as char);
91                    }
92                    None => {
93                        return Err(LexingError::new(
94                            self.file_name.clone(),
95                            "Unterminated string literal".to_string(),
96                            start_line,
97                            start_col,
98                        ));
99                    }
100                },
101                Some(ch) => {
102                    // Handle multi-byte UTF-8 sequences
103                    if ch < 0x80 {
104                        value.push(ch as char);
105                    } else if ch >= 0xC0 {
106                        // Start of multi-byte UTF-8: decode properly
107                        let start = self.pos - 1;
108                        while let Some(&next) = self.source.get(self.pos) {
109                            if (0x80..0xC0).contains(&next) {
110                                self.pos += 1;
111                                self.col += 1;
112                            } else {
113                                break;
114                            }
115                        }
116                        let bytes = &self.source[start..self.pos];
117                        if let Ok(s) = std::str::from_utf8(bytes) {
118                            value.push_str(s);
119                        } else {
120                            // Fallback: push as Latin-1
121                            for &b in bytes {
122                                value.push(b as char);
123                            }
124                        }
125                    } else {
126                        // Continuation byte without start — push as-is
127                        value.push(ch as char);
128                    }
129                }
130                None => {
131                    return Err(LexingError::new(
132                        self.file_name.clone(),
133                        "Unterminated string literal".to_string(),
134                        start_line,
135                        start_col,
136                    ));
137                }
138            }
139        }
140        Ok(self.token(
141            if raw {
142                Kind::RawStringLiteral
143            } else {
144                Kind::StringLiteral
145            },
146            value,
147            start_line,
148            start_col,
149        ))
150    }
151
152    fn read_char_literal(&mut self) -> Result<Token, LexingError> {
153        let start_line = self.line;
154        let start_col = self.col;
155        self.advance(); // consume opening quote
156        let ch = match self.advance() {
157            Some(b'\\') => match self.advance() {
158                Some(b'n') => '\n',
159                Some(b't') => '\t',
160                Some(b'\\') => '\\',
161                Some(b'\'') => '\'',
162                Some(c) => c as char,
163                None => {
164                    return Err(LexingError::new(
165                        self.file_name.clone(),
166                        "Unterminated character literal".to_string(),
167                        start_line,
168                        start_col,
169                    ));
170                }
171            },
172            Some(c) => c as char,
173            None => {
174                return Err(LexingError::new(
175                    self.file_name.clone(),
176                    "Unterminated character literal".to_string(),
177                    start_line,
178                    start_col,
179                ));
180            }
181        };
182        match self.advance() {
183            Some(b'\'') => {}
184            _ => {
185                return Err(LexingError::new(
186                    self.file_name.clone(),
187                    "Expected closing single quote".to_string(),
188                    self.line,
189                    self.col,
190                ));
191            }
192        }
193        Ok(self.token(Kind::CharLiteral, ch.to_string(), start_line, start_col))
194    }
195
196    fn read_identifier(&mut self) -> String {
197        let start = self.pos;
198        while let Some(ch) = self.peek() {
199            if ch.is_ascii_alphanumeric() || ch == b'_' {
200                self.advance();
201            } else {
202                break;
203            }
204        }
205        String::from_utf8_lossy(&self.source[start..self.pos]).to_string()
206    }
207
208    fn read_number(&mut self) -> (String, bool) {
209        let start = self.pos;
210        let mut is_long = false;
211        while let Some(ch) = self.peek() {
212            if ch.is_ascii_digit() {
213                self.advance();
214            } else {
215                break;
216            }
217        }
218        // Check for long suffix
219        if self.peek() == Some(b'L') || self.peek() == Some(b'l') {
220            is_long = true;
221            self.advance();
222        }
223        let number = String::from_utf8_lossy(&self.source[start..self.pos]).to_string();
224        // Strip the L suffix from the value
225        let value = if is_long {
226            number.trim_end_matches(['L', 'l']).to_string()
227        } else {
228            number
229        };
230        (value, is_long)
231    }
232
233    fn read_single_line_comment(&mut self) -> String {
234        let start = self.pos;
235        while let Some(ch) = self.peek() {
236            if ch == b'\n' {
237                break;
238            }
239            self.advance();
240        }
241        String::from_utf8_lossy(&self.source[start..self.pos]).to_string()
242    }
243
244    fn read_multi_line_comment(&mut self) -> Result<String, LexingError> {
245        let start_line = self.line;
246        let start_col = self.col;
247        let mut comment = String::new();
248        let mut depth: u32 = 1;
249        while depth > 0 {
250            match self.advance() {
251                Some(b'/') => {
252                    if self.peek() == Some(b'*') {
253                        self.advance();
254                        comment.push_str("/*");
255                        depth += 1;
256                    } else {
257                        comment.push('/');
258                    }
259                }
260                Some(b'*') => {
261                    if self.peek() == Some(b'/') {
262                        self.advance();
263                        depth -= 1;
264                        if depth > 0 {
265                            comment.push_str("*/");
266                        }
267                    } else {
268                        comment.push('*');
269                    }
270                }
271                Some(ch) => comment.push(ch as char),
272                None => {
273                    return Err(LexingError::new(
274                        self.file_name.clone(),
275                        "Unterminated multi-line comment".to_string(),
276                        start_line,
277                        start_col,
278                    ));
279                }
280            }
281        }
282        Ok(comment)
283    }
284
285    fn classify_identifier(&self, ident: &str) -> Kind {
286        match ident {
287            "proc" | "clientscript" | "label" | "debugproc" | "walktrigger" | "queue" | "timer"
288            | "softtimer" | "ai_queue" | "ai_timer" | "opnpc" | "opnpc1" | "opnpc2" | "opnpc3"
289            | "opnpc4" | "opnpc5" | "opobj" | "opobj1" | "opobj2" | "opobj3" | "opobj4"
290            | "opobj5" | "oploc" | "oploc1" | "oploc2" | "oploc3" | "oploc4" | "oploc5"
291            | "opplayer" | "opplayer1" | "opplayer2" | "opplayer3" | "opplayer4" | "opplayer5"
292            | "opheld" | "opheld1" | "opheld2" | "opheld3" | "opheld4" | "opheld5" | "oplocu"
293            | "opobju" | "opnpcu" | "opplayeru" | "opheldu" | "opnpct" | "oploct" | "opplayert"
294            | "opheldt" | "opobjt" | "apnpc" | "apnpc1" | "apnpc2" | "apnpc3" | "apnpc4"
295            | "apnpc5" | "aploc" | "aploc1" | "aploc2" | "aploc3" | "aploc4" | "aploc5"
296            | "apobj" | "apobj1" | "apobj2" | "apobj3" | "apobj4" | "apobj5" | "applayer"
297            | "applayer1" | "applayer2" | "applayer3" | "applayer4" | "applayer5" | "if_button"
298            | "if_close" | "if_button1" | "if_button2" | "if_button3" | "if_button4"
299            | "if_button5" | "if_button6" | "if_button7" | "if_button8" | "if_button9"
300            | "if_button10" | "if_buttond" | "login" | "logout" | "inv_button1" | "inv_button2"
301            | "inv_button3" | "inv_button4" | "inv_button5" | "inv_buttond" | "mapzone"
302            | "maplength" | "mapenter" => Kind::Trigger,
303
304            "if" => Kind::If,
305            "else" => Kind::Else,
306            "while" => Kind::While,
307            "return" => Kind::Return,
308            "calc" => Kind::Calc,
309            "null" => Kind::Null,
310            "true" => Kind::BooleanTrue,
311            "false" => Kind::BooleanFalse,
312            "case" => Kind::Case,
313            "default" => Kind::Default,
314
315            s if s.starts_with("switch_") => Kind::Switch,
316
317            s if s.starts_with("def_") => {
318                if Type::from_def_str(s).is_some() {
319                    Kind::Def
320                } else {
321                    Kind::Identifier
322                }
323            }
324
325            _ => Kind::Identifier,
326        }
327    }
328
329    pub fn tokenize(&mut self) -> Result<Vec<Token>, LexingError> {
330        let mut tokens = Vec::new();
331        self.tokenize_into(&mut tokens, &mut None)?;
332        Ok(tokens)
333    }
334
335    /// Recover tokens and byte ranges for editor documents, including after
336    /// invalid characters. Ranges refer to the original, unescaped source.
337    pub fn tokenize_spanned(
338        &mut self,
339    ) -> (Vec<Token>, Vec<std::ops::Range<usize>>, Vec<LexingError>) {
340        let mut tokens = Vec::new();
341        let mut spans = Some(Vec::new());
342        let mut errors = Vec::new();
343        loop {
344            let before = self.pos;
345            match self.tokenize_into(&mut tokens, &mut spans) {
346                Ok(()) => break,
347                Err(error) => {
348                    errors.push(error);
349                    if self.pos == before || self.peek().is_some() {
350                        self.advance();
351                        while self.peek().is_some_and(|b| (0x80..0xc0).contains(&b)) {
352                            self.advance();
353                        }
354                    }
355                }
356            }
357        }
358        (tokens, spans.unwrap(), errors)
359    }
360
361    fn tokenize_into(
362        &mut self,
363        tokens: &mut Vec<Token>,
364        spans: &mut Option<Vec<std::ops::Range<usize>>>,
365    ) -> Result<(), LexingError> {
366        loop {
367            self.skip_whitespace();
368
369            let Some(ch) = self.peek() else {
370                break;
371            };
372
373            let start_line = self.line;
374            let start_col = self.col;
375            let start = self.pos;
376            let count = tokens.len();
377
378            match ch {
379                b'[' => {
380                    self.advance();
381                    tokens.push(self.token(Kind::LBracket, "[".into(), start_line, start_col));
382                }
383                b']' => {
384                    self.advance();
385                    tokens.push(self.token(Kind::RBracket, "]".into(), start_line, start_col));
386                }
387                b'(' => {
388                    self.advance();
389                    tokens.push(self.token(Kind::LParen, "(".into(), start_line, start_col));
390                }
391                b')' => {
392                    self.advance();
393                    tokens.push(self.token(Kind::RParen, ")".into(), start_line, start_col));
394                }
395                b'{' => {
396                    self.advance();
397                    tokens.push(self.token(Kind::LBrace, "{".into(), start_line, start_col));
398                }
399                b'}' => {
400                    self.advance();
401                    tokens.push(self.token(Kind::RBrace, "}".into(), start_line, start_col));
402                }
403                b';' => {
404                    self.advance();
405                    tokens.push(self.token(Kind::Semicolon, ";".into(), start_line, start_col));
406                }
407                b',' => {
408                    self.advance();
409                    tokens.push(self.token(Kind::Comma, ",".into(), start_line, start_col));
410                }
411                b':' => {
412                    self.advance();
413                    tokens.push(self.token(Kind::Colon, ":".into(), start_line, start_col));
414                }
415                b'&' => {
416                    self.advance();
417                    tokens.push(self.token(Kind::LogicalAnd, "&".into(), start_line, start_col));
418                }
419                b'|' => {
420                    self.advance();
421                    tokens.push(self.token(Kind::LogicalOr, "|".into(), start_line, start_col));
422                }
423                b'!' => {
424                    self.advance();
425                    tokens.push(self.token(Kind::Not, "!".into(), start_line, start_col));
426                }
427                b'#' => {
428                    self.advance();
429                    while let Some(c) = self.peek() {
430                        if c == b'\n' {
431                            break;
432                        }
433                        self.advance();
434                    }
435                }
436
437                b'.' => {
438                    self.advance();
439                    tokens.push(self.token(Kind::Dot, ".".into(), start_line, start_col));
440                }
441                b'~' => {
442                    self.advance();
443                    tokens.push(self.token(Kind::ScriptCall, "~".into(), start_line, start_col));
444                }
445                b'@' => {
446                    self.advance();
447                    tokens.push(self.token(Kind::JumpCall, "@".into(), start_line, start_col));
448                }
449
450                b'$' => {
451                    self.advance(); // consume $
452                    tokens.push(self.token(Kind::LocalVar, "$".into(), start_line, start_col));
453                }
454
455                b'%' => {
456                    self.advance(); // consume %
457                    // Disambiguate GameVar prefix vs modulo operator by the
458                    // following byte. `%name` is a varp/varn reference;
459                    // `% 4` / `%4` after a value is modulo. If the next byte
460                    // is whitespace or a digit (or EOF / operator), the token
461                    // is binary `%`; only a letter/underscore kicks off a
462                    // GameVar identifier.
463                    let is_gamevar = match self.peek() {
464                        Some(c) => c.is_ascii_alphabetic() || c == b'_',
465                        None => false,
466                    };
467                    if is_gamevar {
468                        tokens.push(self.token(Kind::GameVar, "%".into(), start_line, start_col));
469                    } else {
470                        tokens.push(self.token(
471                            Kind::BinaryOperator,
472                            "%".into(),
473                            start_line,
474                            start_col,
475                        ));
476                    }
477                }
478
479                b'^' => {
480                    self.advance(); // consume ^
481                    tokens.push(self.token(Kind::ConstantVar, "^".into(), start_line, start_col));
482                }
483
484                b'=' => {
485                    self.advance();
486                    tokens.push(self.token(Kind::Equals, "=".into(), start_line, start_col));
487                }
488
489                b'<' => {
490                    self.advance();
491                    if self.peek() == Some(b'=') {
492                        self.advance();
493                        tokens.push(self.token(
494                            Kind::ComparisonOperator,
495                            "<=".into(),
496                            start_line,
497                            start_col,
498                        ));
499                    } else {
500                        tokens.push(self.token(
501                            Kind::ComparisonOperator,
502                            "<".into(),
503                            start_line,
504                            start_col,
505                        ));
506                    }
507                }
508                b'>' => {
509                    self.advance();
510                    if self.peek() == Some(b'=') {
511                        self.advance();
512                        tokens.push(self.token(
513                            Kind::ComparisonOperator,
514                            ">=".into(),
515                            start_line,
516                            start_col,
517                        ));
518                    } else {
519                        tokens.push(self.token(
520                            Kind::ComparisonOperator,
521                            ">".into(),
522                            start_line,
523                            start_col,
524                        ));
525                    }
526                }
527
528                b'+' | b'-' | b'*' => {
529                    self.advance();
530                    // Handle negative numbers: if '-' follows certain tokens, it's unary
531                    tokens.push(self.token(
532                        Kind::BinaryOperator,
533                        (ch as char).to_string(),
534                        start_line,
535                        start_col,
536                    ));
537                }
538
539                b'/' => {
540                    self.advance();
541                    match self.peek() {
542                        Some(b'/') => {
543                            self.advance(); // consume second /
544                            let comment = self.read_single_line_comment();
545                            tokens.push(self.token(
546                                Kind::SingleLineComment,
547                                comment,
548                                start_line,
549                                start_col,
550                            ));
551                        }
552                        Some(b'*') => {
553                            self.advance(); // consume *
554                            let comment = self.read_multi_line_comment()?;
555                            tokens.push(self.token(
556                                Kind::MultiLineComment,
557                                comment,
558                                start_line,
559                                start_col,
560                            ));
561                        }
562                        _ => {
563                            tokens.push(self.token(
564                                Kind::BinaryOperator,
565                                "/".into(),
566                                start_line,
567                                start_col,
568                            ));
569                        }
570                    }
571                }
572
573                b'"' => {
574                    let tok = self.read_string_literal(false)?;
575                    tokens.push(tok);
576                }
577
578                b'r' if self.peek_ahead(1) == Some(b'"') => {
579                    self.advance();
580                    let tok = self.read_string_literal(true)?;
581                    tokens.push(tok);
582                }
583
584                b'\'' => {
585                    let tok = self.read_char_literal()?;
586                    tokens.push(tok);
587                }
588
589                b'_' => {
590                    // Could be standalone underscore or start of identifier
591                    if self
592                        .peek_ahead(1)
593                        .is_some_and(|c| c.is_ascii_alphanumeric() || c == b'_')
594                    {
595                        let ident = self.read_identifier();
596                        let kind = self.classify_identifier(&ident);
597                        tokens.push(self.token(kind, ident, start_line, start_col));
598                    } else {
599                        self.advance();
600                        tokens.push(self.token(
601                            Kind::Underscore,
602                            "_".into(),
603                            start_line,
604                            start_col,
605                        ));
606                    }
607                }
608
609                c if c.is_ascii_alphabetic() => {
610                    let ident = self.read_identifier();
611                    let kind = self.classify_identifier(&ident);
612                    tokens.push(self.token(kind, ident, start_line, start_col));
613                }
614
615                c if c.is_ascii_digit() => {
616                    // Check for hex literal 0x...
617                    let (value, is_long) = if c == b'0'
618                        && self.source.get(self.pos + 1).copied() == Some(b'x')
619                    {
620                        self.advance(); // consume '0'
621                        self.advance(); // consume 'x'
622                        let hex_start = self.pos;
623                        while let Some(ch) = self.peek() {
624                            if ch.is_ascii_hexdigit() {
625                                self.advance();
626                            } else {
627                                break;
628                            }
629                        }
630                        let hex_str =
631                            String::from_utf8_lossy(&self.source[hex_start..self.pos]).to_string();
632                        // Parse as u32 first to handle 0xFFFFFFFF correctly, then reinterpret as i32
633                        let val = u32::from_str_radix(&hex_str, 16).unwrap_or(0) as i32;
634                        (val.to_string(), false)
635                    } else {
636                        self.read_number()
637                    };
638                    // Check if this number is immediately followed by more identifier chars
639                    // (e.g., "2dose1strength" or "0_48_48_newbiefishing") → lex as Identifier
640                    let next_is_ident = self.peek().is_some_and(|c| c.is_ascii_alphabetic());
641                    if !is_long && next_is_ident {
642                        // Number immediately followed by letters — treat as identifier
643                        let mut ident = value;
644                        let rest = self.read_identifier();
645                        ident.push_str(&rest);
646                        tokens.push(self.token(Kind::Identifier, ident, start_line, start_col));
647                    } else if !is_long && self.peek() == Some(b'_') {
648                        // Possibly a coord literal like 0_39_48_41_13
649                        let saved_pos = self.pos;
650                        let saved_line = self.line;
651                        let saved_col = self.col;
652                        let mut coord = value.clone();
653                        let mut is_coord = true;
654                        for _ in 0..4 {
655                            if self.peek() == Some(b'_') {
656                                self.advance(); // consume _
657                                coord.push('_');
658                                if self.peek().is_some_and(|c| c.is_ascii_digit()) {
659                                    let (part, part_long) = self.read_number();
660                                    if part_long {
661                                        is_coord = false;
662                                        break;
663                                    }
664                                    coord.push_str(&part);
665                                } else {
666                                    // Non-numeric after underscore — could be identifier suffix
667                                    // e.g., 0_48_48_newbiefishing: read the rest as identifier
668                                    is_coord = false;
669                                    break;
670                                }
671                            } else {
672                                is_coord = false;
673                                break;
674                            }
675                        }
676                        if is_coord
677                            && !self
678                                .peek()
679                                .is_some_and(|c| c == b'_' || c.is_ascii_alphanumeric())
680                        {
681                            tokens.push(self.token(
682                                Kind::CoordLiteral,
683                                coord,
684                                start_line,
685                                start_col,
686                            ));
687                        } else {
688                            // Not a coord literal; restore and re-read as identifier
689                            self.pos = saved_pos;
690                            self.line = saved_line;
691                            self.col = saved_col;
692                            // If next is '_' followed by alpha, read the whole thing as identifier
693                            if self.peek() == Some(b'_') {
694                                let mut ident = value;
695                                // read_identifier starts from current pos which is '_'
696                                let rest = self.read_identifier();
697                                ident.push_str(&rest);
698                                tokens.push(self.token(
699                                    Kind::Identifier,
700                                    ident,
701                                    start_line,
702                                    start_col,
703                                ));
704                            } else {
705                                tokens.push(self.token(Kind::Number, value, start_line, start_col));
706                            }
707                        }
708                    } else {
709                        let kind = if is_long {
710                            Kind::LongLiteral
711                        } else {
712                            Kind::Number
713                        };
714                        tokens.push(self.token(kind, value, start_line, start_col));
715                    }
716                }
717
718                _ => {
719                    return Err(LexingError::new(
720                        self.file_name.clone(),
721                        format!("Unrecognized character '{}'", ch as char),
722                        self.line,
723                        self.col,
724                    ));
725                }
726            }
727            if let Some(spans) = spans {
728                spans.extend((count..tokens.len()).map(|_| start..self.pos));
729            }
730        }
731
732        tokens.push(Token::new(
733            Kind::EndOfFile,
734            "EOF".into(),
735            self.line,
736            self.col,
737        ));
738        if let Some(spans) = spans {
739            spans.push(self.pos..self.pos);
740        }
741        Ok(())
742    }
743}