Skip to main content

runec/
lexer.rs

1use crate::error::LexingError;
2use crate::token::{Kind, Token};
3use crate::types::Type;
4use std::path::PathBuf;
5
6pub struct Lexer<'a> {
7    source: &'a [u8],
8    file_name: &'a PathBuf,
9    pos: usize,
10    line: usize,
11    col: usize,
12}
13
14impl<'a> Lexer<'a> {
15    pub fn new(input: &'a str, file_name: &'a PathBuf) -> Self {
16        Self {
17            source: input.as_bytes(),
18            file_name,
19            pos: 0,
20            line: 1,
21            col: 1,
22        }
23    }
24
25    fn peek(&self) -> Option<u8> {
26        self.source.get(self.pos).copied()
27    }
28
29    fn peek_ahead(&self, offset: usize) -> Option<u8> {
30        self.source.get(self.pos + offset).copied()
31    }
32
33    fn advance(&mut self) -> Option<u8> {
34        let ch = self.source.get(self.pos).copied()?;
35        self.pos += 1;
36        if ch == b'\n' {
37            self.line += 1;
38            self.col = 1;
39        } else {
40            self.col += 1;
41        }
42        Some(ch)
43    }
44
45    fn token(&self, kind: Kind, value: String, line: usize, col: usize) -> Token {
46        Token::new(kind, value, line, col)
47    }
48
49    fn skip_whitespace(&mut self) {
50        while let Some(ch) = self.peek() {
51            if ch.is_ascii_whitespace() {
52                self.advance();
53            } else {
54                break;
55            }
56        }
57    }
58
59    fn read_string_literal(&mut self) -> Result<Token, LexingError> {
60        let start_line = self.line;
61        let start_col = self.col;
62        self.advance(); // consume opening quote
63        let mut value = String::new();
64        let mut interp_depth: u32 = 0; // depth inside <...> interpolation blocks
65        loop {
66            match self.advance() {
67                Some(b'"') => {
68                    // Inside an interpolation block, embedded quotes are allowed
69                    if interp_depth > 0 {
70                        value.push('"');
71                    } else {
72                        break;
73                    }
74                }
75                Some(b'<') => {
76                    interp_depth += 1;
77                    value.push('<');
78                }
79                Some(b'>') => {
80                    interp_depth = interp_depth.saturating_sub(1);
81                    value.push('>');
82                }
83                Some(b'\\') => match self.advance() {
84                    // RuneScript only escapes \\, \", and \< (matching ANTLR grammar)
85                    Some(b'\\') => value.push('\\'),
86                    Some(b'"') => value.push('"'),
87                    Some(b'<') => value.push('<'),
88                    Some(ch) => {
89                        value.push('\\');
90                        value.push(ch as char);
91                    }
92                    None => {
93                        return Err(LexingError::new(
94                            self.file_name.clone(),
95                            "Unterminated string literal".to_string(),
96                            start_line,
97                            start_col,
98                        ));
99                    }
100                },
101                Some(ch) => {
102                    // Handle multi-byte UTF-8 sequences
103                    if ch < 0x80 {
104                        value.push(ch as char);
105                    } else if ch >= 0xC0 {
106                        // Start of multi-byte UTF-8: decode properly
107                        let mut bytes = vec![ch];
108                        while let Some(&next) = self.source.get(self.pos) {
109                            if (0x80..0xC0).contains(&next) {
110                                bytes.push(next);
111                                self.pos += 1;
112                                self.col += 1;
113                            } else {
114                                break;
115                            }
116                        }
117                        if let Ok(s) = std::str::from_utf8(&bytes) {
118                            value.push_str(s);
119                        } else {
120                            // Fallback: push as Latin-1
121                            for b in bytes {
122                                value.push(b as char);
123                            }
124                        }
125                    } else {
126                        // Continuation byte without start — push as-is
127                        value.push(ch as char);
128                    }
129                }
130                None => {
131                    return Err(LexingError::new(
132                        self.file_name.clone(),
133                        "Unterminated string literal".to_string(),
134                        start_line,
135                        start_col,
136                    ));
137                }
138            }
139        }
140        Ok(self.token(Kind::StringLiteral, value, start_line, start_col))
141    }
142
143    fn read_char_literal(&mut self) -> Result<Token, LexingError> {
144        let start_line = self.line;
145        let start_col = self.col;
146        self.advance(); // consume opening quote
147        let ch = match self.advance() {
148            Some(b'\\') => match self.advance() {
149                Some(b'n') => '\n',
150                Some(b't') => '\t',
151                Some(b'\\') => '\\',
152                Some(b'\'') => '\'',
153                Some(c) => c as char,
154                None => {
155                    return Err(LexingError::new(
156                        self.file_name.clone(),
157                        "Unterminated character literal".to_string(),
158                        start_line,
159                        start_col,
160                    ));
161                }
162            },
163            Some(c) => c as char,
164            None => {
165                return Err(LexingError::new(
166                    self.file_name.clone(),
167                    "Unterminated character literal".to_string(),
168                    start_line,
169                    start_col,
170                ));
171            }
172        };
173        match self.advance() {
174            Some(b'\'') => {}
175            _ => {
176                return Err(LexingError::new(
177                    self.file_name.clone(),
178                    "Expected closing single quote".to_string(),
179                    self.line,
180                    self.col,
181                ));
182            }
183        }
184        Ok(self.token(Kind::CharLiteral, ch.to_string(), start_line, start_col))
185    }
186
187    fn read_identifier(&mut self) -> String {
188        let start = self.pos;
189        while let Some(ch) = self.peek() {
190            if ch.is_ascii_alphanumeric() || ch == b'_' {
191                self.advance();
192            } else {
193                break;
194            }
195        }
196        String::from_utf8_lossy(&self.source[start..self.pos]).to_string()
197    }
198
199    fn read_number(&mut self) -> (String, bool) {
200        let start = self.pos;
201        let mut is_long = false;
202        while let Some(ch) = self.peek() {
203            if ch.is_ascii_digit() {
204                self.advance();
205            } else {
206                break;
207            }
208        }
209        // Check for long suffix
210        if self.peek() == Some(b'L') || self.peek() == Some(b'l') {
211            is_long = true;
212            self.advance();
213        }
214        let number = String::from_utf8_lossy(&self.source[start..self.pos]).to_string();
215        // Strip the L suffix from the value
216        let value = if is_long {
217            number.trim_end_matches(['L', 'l']).to_string()
218        } else {
219            number
220        };
221        (value, is_long)
222    }
223
224    fn read_single_line_comment(&mut self) -> String {
225        let start = self.pos;
226        while let Some(ch) = self.peek() {
227            if ch == b'\n' {
228                break;
229            }
230            self.advance();
231        }
232        String::from_utf8_lossy(&self.source[start..self.pos]).to_string()
233    }
234
235    fn read_multi_line_comment(&mut self) -> Result<String, LexingError> {
236        let start_line = self.line;
237        let start_col = self.col;
238        let mut comment = String::new();
239        let mut depth: u32 = 1;
240        while depth > 0 {
241            match self.advance() {
242                Some(b'/') => {
243                    if self.peek() == Some(b'*') {
244                        self.advance();
245                        comment.push_str("/*");
246                        depth += 1;
247                    } else {
248                        comment.push('/');
249                    }
250                }
251                Some(b'*') => {
252                    if self.peek() == Some(b'/') {
253                        self.advance();
254                        depth -= 1;
255                        if depth > 0 {
256                            comment.push_str("*/");
257                        }
258                    } else {
259                        comment.push('*');
260                    }
261                }
262                Some(ch) => comment.push(ch as char),
263                None => {
264                    return Err(LexingError::new(
265                        self.file_name.clone(),
266                        "Unterminated multi-line comment".to_string(),
267                        start_line,
268                        start_col,
269                    ));
270                }
271            }
272        }
273        Ok(comment)
274    }
275
276    fn classify_identifier(&self, ident: &str) -> Kind {
277        match ident {
278            "proc" | "clientscript" | "label" | "debugproc" | "walktrigger" | "queue" | "timer"
279            | "softtimer" | "ai_queue" | "ai_timer" | "opnpc" | "opnpc1" | "opnpc2" | "opnpc3"
280            | "opnpc4" | "opnpc5" | "opobj" | "opobj1" | "opobj2" | "opobj3" | "opobj4"
281            | "opobj5" | "oploc" | "oploc1" | "oploc2" | "oploc3" | "oploc4" | "oploc5"
282            | "opplayer" | "opplayer1" | "opplayer2" | "opplayer3" | "opplayer4" | "opplayer5"
283            | "opheld" | "opheld1" | "opheld2" | "opheld3" | "opheld4" | "opheld5" | "oplocu"
284            | "opobju" | "opnpcu" | "opplayeru" | "opheldu" | "opnpct" | "oploct" | "opplayert"
285            | "opheldt" | "opobjt" | "apnpc" | "apnpc1" | "apnpc2" | "apnpc3" | "apnpc4"
286            | "apnpc5" | "aploc" | "aploc1" | "aploc2" | "aploc3" | "aploc4" | "aploc5"
287            | "apobj" | "apobj1" | "apobj2" | "apobj3" | "apobj4" | "apobj5" | "applayer"
288            | "applayer1" | "applayer2" | "applayer3" | "applayer4" | "applayer5" | "if_button"
289            | "if_close" | "if_button1" | "if_button2" | "if_button3" | "if_button4"
290            | "if_button5" | "if_button6" | "if_button7" | "if_button8" | "if_button9"
291            | "if_button10" | "if_buttond" | "login" | "logout" | "inv_button1" | "inv_button2"
292            | "inv_button3" | "inv_button4" | "inv_button5" | "inv_buttond" | "mapzone"
293            | "maplength" | "mapenter" => Kind::Trigger,
294
295            "if" => Kind::If,
296            "else" => Kind::Else,
297            "while" => Kind::While,
298            "return" => Kind::Return,
299            "calc" => Kind::Calc,
300            "null" => Kind::Null,
301            "true" => Kind::BooleanTrue,
302            "false" => Kind::BooleanFalse,
303            "case" => Kind::Case,
304            "default" => Kind::Default,
305
306            s if s.starts_with("switch_") => Kind::Switch,
307
308            s if s.starts_with("def_") => {
309                if Type::from_def_str(s).is_some() {
310                    Kind::Def
311                } else {
312                    Kind::Identifier
313                }
314            }
315
316            _ => Kind::Identifier,
317        }
318    }
319
320    pub fn tokenize(&mut self) -> Result<Vec<Token>, LexingError> {
321        let mut tokens = Vec::new();
322
323        loop {
324            self.skip_whitespace();
325
326            let Some(ch) = self.peek() else {
327                break;
328            };
329
330            let start_line = self.line;
331            let start_col = self.col;
332
333            match ch {
334                b'[' => {
335                    self.advance();
336                    tokens.push(self.token(Kind::LBracket, "[".into(), start_line, start_col));
337                }
338                b']' => {
339                    self.advance();
340                    tokens.push(self.token(Kind::RBracket, "]".into(), start_line, start_col));
341                }
342                b'(' => {
343                    self.advance();
344                    tokens.push(self.token(Kind::LParen, "(".into(), start_line, start_col));
345                }
346                b')' => {
347                    self.advance();
348                    tokens.push(self.token(Kind::RParen, ")".into(), start_line, start_col));
349                }
350                b'{' => {
351                    self.advance();
352                    tokens.push(self.token(Kind::LBrace, "{".into(), start_line, start_col));
353                }
354                b'}' => {
355                    self.advance();
356                    tokens.push(self.token(Kind::RBrace, "}".into(), start_line, start_col));
357                }
358                b';' => {
359                    self.advance();
360                    tokens.push(self.token(Kind::Semicolon, ";".into(), start_line, start_col));
361                }
362                b',' => {
363                    self.advance();
364                    tokens.push(self.token(Kind::Comma, ",".into(), start_line, start_col));
365                }
366                b':' => {
367                    self.advance();
368                    tokens.push(self.token(Kind::Colon, ":".into(), start_line, start_col));
369                }
370                b'&' => {
371                    self.advance();
372                    tokens.push(self.token(Kind::LogicalAnd, "&".into(), start_line, start_col));
373                }
374                b'|' => {
375                    self.advance();
376                    tokens.push(self.token(Kind::LogicalOr, "|".into(), start_line, start_col));
377                }
378                b'!' => {
379                    self.advance();
380                    tokens.push(self.token(Kind::Not, "!".into(), start_line, start_col));
381                }
382                b'#' => {
383                    self.advance();
384                    while let Some(c) = self.peek() {
385                        if c == b'\n' {
386                            break;
387                        }
388                        self.advance();
389                    }
390                }
391
392                b'.' => {
393                    self.advance();
394                    tokens.push(self.token(Kind::Dot, ".".into(), start_line, start_col));
395                }
396                b'~' => {
397                    self.advance();
398                    tokens.push(self.token(Kind::ScriptCall, "~".into(), start_line, start_col));
399                }
400                b'@' => {
401                    self.advance();
402                    tokens.push(self.token(Kind::JumpCall, "@".into(), start_line, start_col));
403                }
404
405                b'$' => {
406                    self.advance(); // consume $
407                    tokens.push(self.token(Kind::LocalVar, "$".into(), start_line, start_col));
408                }
409
410                b'%' => {
411                    self.advance(); // consume %
412                    // Disambiguate GameVar prefix vs modulo operator by the
413                    // following byte. `%name` is a varp/varn reference;
414                    // `% 4` / `%4` after a value is modulo. If the next byte
415                    // is whitespace or a digit (or EOF / operator), the token
416                    // is binary `%`; only a letter/underscore kicks off a
417                    // GameVar identifier.
418                    let is_gamevar = match self.peek() {
419                        Some(c) => c.is_ascii_alphabetic() || c == b'_',
420                        None => false,
421                    };
422                    if is_gamevar {
423                        tokens.push(self.token(Kind::GameVar, "%".into(), start_line, start_col));
424                    } else {
425                        tokens.push(self.token(
426                            Kind::BinaryOperator,
427                            "%".into(),
428                            start_line,
429                            start_col,
430                        ));
431                    }
432                }
433
434                b'^' => {
435                    self.advance(); // consume ^
436                    tokens.push(self.token(Kind::ConstantVar, "^".into(), start_line, start_col));
437                }
438
439                b'=' => {
440                    self.advance();
441                    tokens.push(self.token(Kind::Equals, "=".into(), start_line, start_col));
442                }
443
444                b'<' => {
445                    self.advance();
446                    if self.peek() == Some(b'=') {
447                        self.advance();
448                        tokens.push(self.token(
449                            Kind::ComparisonOperator,
450                            "<=".into(),
451                            start_line,
452                            start_col,
453                        ));
454                    } else {
455                        tokens.push(self.token(
456                            Kind::ComparisonOperator,
457                            "<".into(),
458                            start_line,
459                            start_col,
460                        ));
461                    }
462                }
463                b'>' => {
464                    self.advance();
465                    if self.peek() == Some(b'=') {
466                        self.advance();
467                        tokens.push(self.token(
468                            Kind::ComparisonOperator,
469                            ">=".into(),
470                            start_line,
471                            start_col,
472                        ));
473                    } else {
474                        tokens.push(self.token(
475                            Kind::ComparisonOperator,
476                            ">".into(),
477                            start_line,
478                            start_col,
479                        ));
480                    }
481                }
482
483                b'+' | b'-' | b'*' => {
484                    self.advance();
485                    // Handle negative numbers: if '-' follows certain tokens, it's unary
486                    tokens.push(self.token(
487                        Kind::BinaryOperator,
488                        (ch as char).to_string(),
489                        start_line,
490                        start_col,
491                    ));
492                }
493
494                b'/' => {
495                    self.advance();
496                    match self.peek() {
497                        Some(b'/') => {
498                            self.advance(); // consume second /
499                            let comment = self.read_single_line_comment();
500                            tokens.push(self.token(
501                                Kind::SingleLineComment,
502                                comment,
503                                start_line,
504                                start_col,
505                            ));
506                        }
507                        Some(b'*') => {
508                            self.advance(); // consume *
509                            let comment = self.read_multi_line_comment()?;
510                            tokens.push(self.token(
511                                Kind::MultiLineComment,
512                                comment,
513                                start_line,
514                                start_col,
515                            ));
516                        }
517                        _ => {
518                            tokens.push(self.token(
519                                Kind::BinaryOperator,
520                                "/".into(),
521                                start_line,
522                                start_col,
523                            ));
524                        }
525                    }
526                }
527
528                b'"' => {
529                    let tok = self.read_string_literal()?;
530                    tokens.push(tok);
531                }
532
533                b'\'' => {
534                    let tok = self.read_char_literal()?;
535                    tokens.push(tok);
536                }
537
538                b'_' => {
539                    // Could be standalone underscore or start of identifier
540                    if self
541                        .peek_ahead(1)
542                        .is_some_and(|c| c.is_ascii_alphanumeric() || c == b'_')
543                    {
544                        let ident = self.read_identifier();
545                        let kind = self.classify_identifier(&ident);
546                        tokens.push(self.token(kind, ident, start_line, start_col));
547                    } else {
548                        self.advance();
549                        tokens.push(self.token(
550                            Kind::Underscore,
551                            "_".into(),
552                            start_line,
553                            start_col,
554                        ));
555                    }
556                }
557
558                c if c.is_ascii_alphabetic() => {
559                    let ident = self.read_identifier();
560                    let kind = self.classify_identifier(&ident);
561                    tokens.push(self.token(kind, ident, start_line, start_col));
562                }
563
564                c if c.is_ascii_digit() => {
565                    // Check for hex literal 0x...
566                    let (value, is_long) = if c == b'0'
567                        && self.source.get(self.pos + 1).copied() == Some(b'x')
568                    {
569                        self.advance(); // consume '0'
570                        self.advance(); // consume 'x'
571                        let hex_start = self.pos;
572                        while let Some(ch) = self.peek() {
573                            if ch.is_ascii_hexdigit() {
574                                self.advance();
575                            } else {
576                                break;
577                            }
578                        }
579                        let hex_str =
580                            String::from_utf8_lossy(&self.source[hex_start..self.pos]).to_string();
581                        // Parse as u32 first to handle 0xFFFFFFFF correctly, then reinterpret as i32
582                        let val = u32::from_str_radix(&hex_str, 16).unwrap_or(0) as i32;
583                        (val.to_string(), false)
584                    } else {
585                        self.read_number()
586                    };
587                    // Check if this number is immediately followed by more identifier chars
588                    // (e.g., "2dose1strength" or "0_48_48_newbiefishing") → lex as Identifier
589                    let next_is_ident = self.peek().is_some_and(|c| c.is_ascii_alphabetic());
590                    if !is_long && next_is_ident {
591                        // Number immediately followed by letters — treat as identifier
592                        let mut ident = value;
593                        let rest = self.read_identifier();
594                        ident.push_str(&rest);
595                        tokens.push(self.token(Kind::Identifier, ident, start_line, start_col));
596                    } else if !is_long && self.peek() == Some(b'_') {
597                        // Possibly a coord literal like 0_39_48_41_13
598                        let saved_pos = self.pos;
599                        let saved_line = self.line;
600                        let saved_col = self.col;
601                        let mut coord = value.clone();
602                        let mut is_coord = true;
603                        for _ in 0..4 {
604                            if self.peek() == Some(b'_') {
605                                self.advance(); // consume _
606                                coord.push('_');
607                                if self.peek().is_some_and(|c| c.is_ascii_digit()) {
608                                    let (part, part_long) = self.read_number();
609                                    if part_long {
610                                        is_coord = false;
611                                        break;
612                                    }
613                                    coord.push_str(&part);
614                                } else {
615                                    // Non-numeric after underscore — could be identifier suffix
616                                    // e.g., 0_48_48_newbiefishing: read the rest as identifier
617                                    is_coord = false;
618                                    break;
619                                }
620                            } else {
621                                is_coord = false;
622                                break;
623                            }
624                        }
625                        if is_coord
626                            && !self
627                                .peek()
628                                .is_some_and(|c| c == b'_' || c.is_ascii_alphanumeric())
629                        {
630                            tokens.push(self.token(
631                                Kind::CoordLiteral,
632                                coord,
633                                start_line,
634                                start_col,
635                            ));
636                        } else {
637                            // Not a coord literal; restore and re-read as identifier
638                            self.pos = saved_pos;
639                            self.line = saved_line;
640                            self.col = saved_col;
641                            // If next is '_' followed by alpha, read the whole thing as identifier
642                            if self.peek() == Some(b'_') {
643                                let mut ident = value;
644                                // read_identifier starts from current pos which is '_'
645                                let rest = self.read_identifier();
646                                ident.push_str(&rest);
647                                tokens.push(self.token(
648                                    Kind::Identifier,
649                                    ident,
650                                    start_line,
651                                    start_col,
652                                ));
653                            } else {
654                                tokens.push(self.token(Kind::Number, value, start_line, start_col));
655                            }
656                        }
657                    } else {
658                        let kind = if is_long {
659                            Kind::LongLiteral
660                        } else {
661                            Kind::Number
662                        };
663                        tokens.push(self.token(kind, value, start_line, start_col));
664                    }
665                }
666
667                _ => {
668                    return Err(LexingError::new(
669                        self.file_name.clone(),
670                        format!("Unrecognized character '{}'", ch as char),
671                        self.line,
672                        self.col,
673                    ));
674                }
675            }
676        }
677
678        tokens.push(Token::new(
679            Kind::EndOfFile,
680            "EOF".into(),
681            self.line,
682            self.col,
683        ));
684        Ok(tokens)
685    }
686}