1use crate::error::LexingError;
2use crate::token::{ESCAPED_LT, Kind, Token};
3use crate::types::Type;
4use std::path::PathBuf;
5
6pub struct Lexer<'a> {
7 source: &'a [u8],
8 file_name: &'a PathBuf,
9 pos: usize,
10 line: usize,
11 col: usize,
12}
13
14impl<'a> Lexer<'a> {
15 pub fn new(input: &'a str, file_name: &'a PathBuf) -> Self {
16 Self {
17 source: input.as_bytes(),
18 file_name,
19 pos: 0,
20 line: 1,
21 col: 1,
22 }
23 }
24
25 fn peek(&self) -> Option<u8> {
26 self.source.get(self.pos).copied()
27 }
28
29 fn peek_ahead(&self, offset: usize) -> Option<u8> {
30 self.source.get(self.pos + offset).copied()
31 }
32
33 fn advance(&mut self) -> Option<u8> {
34 let ch = self.source.get(self.pos).copied()?;
35 self.pos += 1;
36 if ch == b'\n' {
37 self.line += 1;
38 self.col = 1;
39 } else {
40 self.col += 1;
41 }
42 Some(ch)
43 }
44
45 fn token(&self, kind: Kind, value: String, line: usize, col: usize) -> Token {
46 Token::new(kind, value, line, col)
47 }
48
49 fn skip_whitespace(&mut self) {
50 while let Some(ch) = self.peek() {
51 if ch.is_ascii_whitespace() {
52 self.advance();
53 } else {
54 break;
55 }
56 }
57 }
58
59 fn read_string_literal(&mut self, raw: bool) -> Result<Token, LexingError> {
60 let start_line = self.line;
61 let start_col = self.col;
62 self.advance(); let mut value = String::new();
64 let mut interp_depth: u32 = 0; loop {
66 match self.advance() {
67 Some(b'"') => {
68 if interp_depth > 0 {
70 value.push('"');
71 } else {
72 break;
73 }
74 }
75 Some(b'<') if !raw => {
76 interp_depth += 1;
77 value.push('<');
78 }
79 Some(b'>') if !raw => {
80 interp_depth = interp_depth.saturating_sub(1);
81 value.push('>');
82 }
83 Some(b'\\') => match self.advance() {
84 Some(b'\\') => value.push('\\'),
86 Some(b'"') => value.push('"'),
87 Some(b'<') => value.push(if raw { '<' } else { ESCAPED_LT }),
88 Some(ch) => {
89 value.push('\\');
90 value.push(ch as char);
91 }
92 None => {
93 return Err(LexingError::new(
94 self.file_name.clone(),
95 "Unterminated string literal".to_string(),
96 start_line,
97 start_col,
98 ));
99 }
100 },
101 Some(ch) => {
102 if ch < 0x80 {
104 value.push(ch as char);
105 } else if ch >= 0xC0 {
106 let start = self.pos - 1;
108 while let Some(&next) = self.source.get(self.pos) {
109 if (0x80..0xC0).contains(&next) {
110 self.pos += 1;
111 self.col += 1;
112 } else {
113 break;
114 }
115 }
116 let bytes = &self.source[start..self.pos];
117 if let Ok(s) = std::str::from_utf8(bytes) {
118 value.push_str(s);
119 } else {
120 for &b in bytes {
122 value.push(b as char);
123 }
124 }
125 } else {
126 value.push(ch as char);
128 }
129 }
130 None => {
131 return Err(LexingError::new(
132 self.file_name.clone(),
133 "Unterminated string literal".to_string(),
134 start_line,
135 start_col,
136 ));
137 }
138 }
139 }
140 Ok(self.token(
141 if raw {
142 Kind::RawStringLiteral
143 } else {
144 Kind::StringLiteral
145 },
146 value,
147 start_line,
148 start_col,
149 ))
150 }
151
152 fn read_char_literal(&mut self) -> Result<Token, LexingError> {
153 let start_line = self.line;
154 let start_col = self.col;
155 self.advance(); let ch = match self.advance() {
157 Some(b'\\') => match self.advance() {
158 Some(b'n') => '\n',
159 Some(b't') => '\t',
160 Some(b'\\') => '\\',
161 Some(b'\'') => '\'',
162 Some(c) => c as char,
163 None => {
164 return Err(LexingError::new(
165 self.file_name.clone(),
166 "Unterminated character literal".to_string(),
167 start_line,
168 start_col,
169 ));
170 }
171 },
172 Some(c) => c as char,
173 None => {
174 return Err(LexingError::new(
175 self.file_name.clone(),
176 "Unterminated character literal".to_string(),
177 start_line,
178 start_col,
179 ));
180 }
181 };
182 match self.advance() {
183 Some(b'\'') => {}
184 _ => {
185 return Err(LexingError::new(
186 self.file_name.clone(),
187 "Expected closing single quote".to_string(),
188 self.line,
189 self.col,
190 ));
191 }
192 }
193 Ok(self.token(Kind::CharLiteral, ch.to_string(), start_line, start_col))
194 }
195
196 fn read_identifier(&mut self) -> String {
197 let start = self.pos;
198 while let Some(ch) = self.peek() {
199 if ch.is_ascii_alphanumeric() || ch == b'_' {
200 self.advance();
201 } else {
202 break;
203 }
204 }
205 String::from_utf8_lossy(&self.source[start..self.pos]).to_string()
206 }
207
208 fn read_number(&mut self) -> (String, bool) {
209 let start = self.pos;
210 let mut is_long = false;
211 while let Some(ch) = self.peek() {
212 if ch.is_ascii_digit() {
213 self.advance();
214 } else {
215 break;
216 }
217 }
218 if self.peek() == Some(b'L') || self.peek() == Some(b'l') {
220 is_long = true;
221 self.advance();
222 }
223 let number = String::from_utf8_lossy(&self.source[start..self.pos]).to_string();
224 let value = if is_long {
226 number.trim_end_matches(['L', 'l']).to_string()
227 } else {
228 number
229 };
230 (value, is_long)
231 }
232
233 fn read_single_line_comment(&mut self) -> String {
234 let start = self.pos;
235 while let Some(ch) = self.peek() {
236 if ch == b'\n' {
237 break;
238 }
239 self.advance();
240 }
241 String::from_utf8_lossy(&self.source[start..self.pos]).to_string()
242 }
243
244 fn read_multi_line_comment(&mut self) -> Result<String, LexingError> {
245 let start_line = self.line;
246 let start_col = self.col;
247 let mut comment = String::new();
248 let mut depth: u32 = 1;
249 while depth > 0 {
250 match self.advance() {
251 Some(b'/') => {
252 if self.peek() == Some(b'*') {
253 self.advance();
254 comment.push_str("/*");
255 depth += 1;
256 } else {
257 comment.push('/');
258 }
259 }
260 Some(b'*') => {
261 if self.peek() == Some(b'/') {
262 self.advance();
263 depth -= 1;
264 if depth > 0 {
265 comment.push_str("*/");
266 }
267 } else {
268 comment.push('*');
269 }
270 }
271 Some(ch) => comment.push(ch as char),
272 None => {
273 return Err(LexingError::new(
274 self.file_name.clone(),
275 "Unterminated multi-line comment".to_string(),
276 start_line,
277 start_col,
278 ));
279 }
280 }
281 }
282 Ok(comment)
283 }
284
285 fn classify_identifier(&self, ident: &str) -> Kind {
286 match ident {
287 "proc" | "clientscript" | "label" | "debugproc" | "walktrigger" | "queue" | "timer"
288 | "softtimer" | "ai_queue" | "ai_timer" | "opnpc" | "opnpc1" | "opnpc2" | "opnpc3"
289 | "opnpc4" | "opnpc5" | "opobj" | "opobj1" | "opobj2" | "opobj3" | "opobj4"
290 | "opobj5" | "oploc" | "oploc1" | "oploc2" | "oploc3" | "oploc4" | "oploc5"
291 | "opplayer" | "opplayer1" | "opplayer2" | "opplayer3" | "opplayer4" | "opplayer5"
292 | "opheld" | "opheld1" | "opheld2" | "opheld3" | "opheld4" | "opheld5" | "oplocu"
293 | "opobju" | "opnpcu" | "opplayeru" | "opheldu" | "opnpct" | "oploct" | "opplayert"
294 | "opheldt" | "opobjt" | "apnpc" | "apnpc1" | "apnpc2" | "apnpc3" | "apnpc4"
295 | "apnpc5" | "aploc" | "aploc1" | "aploc2" | "aploc3" | "aploc4" | "aploc5"
296 | "apobj" | "apobj1" | "apobj2" | "apobj3" | "apobj4" | "apobj5" | "applayer"
297 | "applayer1" | "applayer2" | "applayer3" | "applayer4" | "applayer5" | "if_button"
298 | "if_close" | "if_button1" | "if_button2" | "if_button3" | "if_button4"
299 | "if_button5" | "if_button6" | "if_button7" | "if_button8" | "if_button9"
300 | "if_button10" | "if_buttond" | "login" | "logout" | "inv_button1" | "inv_button2"
301 | "inv_button3" | "inv_button4" | "inv_button5" | "inv_buttond" | "mapzone"
302 | "maplength" | "mapenter" => Kind::Trigger,
303
304 "if" => Kind::If,
305 "else" => Kind::Else,
306 "while" => Kind::While,
307 "return" => Kind::Return,
308 "calc" => Kind::Calc,
309 "null" => Kind::Null,
310 "true" => Kind::BooleanTrue,
311 "false" => Kind::BooleanFalse,
312 "case" => Kind::Case,
313 "default" => Kind::Default,
314
315 s if s.starts_with("switch_") => Kind::Switch,
316
317 s if s.starts_with("def_") => {
318 if Type::from_def_str(s).is_some() {
319 Kind::Def
320 } else {
321 Kind::Identifier
322 }
323 }
324
325 _ => Kind::Identifier,
326 }
327 }
328
329 pub fn tokenize(&mut self) -> Result<Vec<Token>, LexingError> {
330 let mut tokens = Vec::new();
331 self.tokenize_into(&mut tokens, &mut None)?;
332 Ok(tokens)
333 }
334
335 pub fn tokenize_spanned(
338 &mut self,
339 ) -> (Vec<Token>, Vec<std::ops::Range<usize>>, Vec<LexingError>) {
340 let mut tokens = Vec::new();
341 let mut spans = Some(Vec::new());
342 let mut errors = Vec::new();
343 loop {
344 let before = self.pos;
345 match self.tokenize_into(&mut tokens, &mut spans) {
346 Ok(()) => break,
347 Err(error) => {
348 errors.push(error);
349 if self.pos == before || self.peek().is_some() {
350 self.advance();
351 while self.peek().is_some_and(|b| (0x80..0xc0).contains(&b)) {
352 self.advance();
353 }
354 }
355 }
356 }
357 }
358 (tokens, spans.unwrap(), errors)
359 }
360
361 fn tokenize_into(
362 &mut self,
363 tokens: &mut Vec<Token>,
364 spans: &mut Option<Vec<std::ops::Range<usize>>>,
365 ) -> Result<(), LexingError> {
366 loop {
367 self.skip_whitespace();
368
369 let Some(ch) = self.peek() else {
370 break;
371 };
372
373 let start_line = self.line;
374 let start_col = self.col;
375 let start = self.pos;
376 let count = tokens.len();
377
378 match ch {
379 b'[' => {
380 self.advance();
381 tokens.push(self.token(Kind::LBracket, "[".into(), start_line, start_col));
382 }
383 b']' => {
384 self.advance();
385 tokens.push(self.token(Kind::RBracket, "]".into(), start_line, start_col));
386 }
387 b'(' => {
388 self.advance();
389 tokens.push(self.token(Kind::LParen, "(".into(), start_line, start_col));
390 }
391 b')' => {
392 self.advance();
393 tokens.push(self.token(Kind::RParen, ")".into(), start_line, start_col));
394 }
395 b'{' => {
396 self.advance();
397 tokens.push(self.token(Kind::LBrace, "{".into(), start_line, start_col));
398 }
399 b'}' => {
400 self.advance();
401 tokens.push(self.token(Kind::RBrace, "}".into(), start_line, start_col));
402 }
403 b';' => {
404 self.advance();
405 tokens.push(self.token(Kind::Semicolon, ";".into(), start_line, start_col));
406 }
407 b',' => {
408 self.advance();
409 tokens.push(self.token(Kind::Comma, ",".into(), start_line, start_col));
410 }
411 b':' => {
412 self.advance();
413 tokens.push(self.token(Kind::Colon, ":".into(), start_line, start_col));
414 }
415 b'&' => {
416 self.advance();
417 tokens.push(self.token(Kind::LogicalAnd, "&".into(), start_line, start_col));
418 }
419 b'|' => {
420 self.advance();
421 tokens.push(self.token(Kind::LogicalOr, "|".into(), start_line, start_col));
422 }
423 b'!' => {
424 self.advance();
425 tokens.push(self.token(Kind::Not, "!".into(), start_line, start_col));
426 }
427 b'#' => {
428 self.advance();
429 while let Some(c) = self.peek() {
430 if c == b'\n' {
431 break;
432 }
433 self.advance();
434 }
435 }
436
437 b'.' => {
438 self.advance();
439 tokens.push(self.token(Kind::Dot, ".".into(), start_line, start_col));
440 }
441 b'~' => {
442 self.advance();
443 tokens.push(self.token(Kind::ScriptCall, "~".into(), start_line, start_col));
444 }
445 b'@' => {
446 self.advance();
447 tokens.push(self.token(Kind::JumpCall, "@".into(), start_line, start_col));
448 }
449
450 b'$' => {
451 self.advance(); tokens.push(self.token(Kind::LocalVar, "$".into(), start_line, start_col));
453 }
454
455 b'%' => {
456 self.advance(); let is_gamevar = match self.peek() {
464 Some(c) => c.is_ascii_alphabetic() || c == b'_',
465 None => false,
466 };
467 if is_gamevar {
468 tokens.push(self.token(Kind::GameVar, "%".into(), start_line, start_col));
469 } else {
470 tokens.push(self.token(
471 Kind::BinaryOperator,
472 "%".into(),
473 start_line,
474 start_col,
475 ));
476 }
477 }
478
479 b'^' => {
480 self.advance(); tokens.push(self.token(Kind::ConstantVar, "^".into(), start_line, start_col));
482 }
483
484 b'=' => {
485 self.advance();
486 tokens.push(self.token(Kind::Equals, "=".into(), start_line, start_col));
487 }
488
489 b'<' => {
490 self.advance();
491 if self.peek() == Some(b'=') {
492 self.advance();
493 tokens.push(self.token(
494 Kind::ComparisonOperator,
495 "<=".into(),
496 start_line,
497 start_col,
498 ));
499 } else {
500 tokens.push(self.token(
501 Kind::ComparisonOperator,
502 "<".into(),
503 start_line,
504 start_col,
505 ));
506 }
507 }
508 b'>' => {
509 self.advance();
510 if self.peek() == Some(b'=') {
511 self.advance();
512 tokens.push(self.token(
513 Kind::ComparisonOperator,
514 ">=".into(),
515 start_line,
516 start_col,
517 ));
518 } else {
519 tokens.push(self.token(
520 Kind::ComparisonOperator,
521 ">".into(),
522 start_line,
523 start_col,
524 ));
525 }
526 }
527
528 b'+' | b'-' | b'*' => {
529 self.advance();
530 tokens.push(self.token(
532 Kind::BinaryOperator,
533 (ch as char).to_string(),
534 start_line,
535 start_col,
536 ));
537 }
538
539 b'/' => {
540 self.advance();
541 match self.peek() {
542 Some(b'/') => {
543 self.advance(); let comment = self.read_single_line_comment();
545 tokens.push(self.token(
546 Kind::SingleLineComment,
547 comment,
548 start_line,
549 start_col,
550 ));
551 }
552 Some(b'*') => {
553 self.advance(); let comment = self.read_multi_line_comment()?;
555 tokens.push(self.token(
556 Kind::MultiLineComment,
557 comment,
558 start_line,
559 start_col,
560 ));
561 }
562 _ => {
563 tokens.push(self.token(
564 Kind::BinaryOperator,
565 "/".into(),
566 start_line,
567 start_col,
568 ));
569 }
570 }
571 }
572
573 b'"' => {
574 let tok = self.read_string_literal(false)?;
575 tokens.push(tok);
576 }
577
578 b'r' if self.peek_ahead(1) == Some(b'"') => {
579 self.advance();
580 let tok = self.read_string_literal(true)?;
581 tokens.push(tok);
582 }
583
584 b'\'' => {
585 let tok = self.read_char_literal()?;
586 tokens.push(tok);
587 }
588
589 b'_' => {
590 if self
592 .peek_ahead(1)
593 .is_some_and(|c| c.is_ascii_alphanumeric() || c == b'_')
594 {
595 let ident = self.read_identifier();
596 let kind = self.classify_identifier(&ident);
597 tokens.push(self.token(kind, ident, start_line, start_col));
598 } else {
599 self.advance();
600 tokens.push(self.token(
601 Kind::Underscore,
602 "_".into(),
603 start_line,
604 start_col,
605 ));
606 }
607 }
608
609 c if c.is_ascii_alphabetic() => {
610 let ident = self.read_identifier();
611 let kind = self.classify_identifier(&ident);
612 tokens.push(self.token(kind, ident, start_line, start_col));
613 }
614
615 c if c.is_ascii_digit() => {
616 let (value, is_long) = if c == b'0'
618 && self.source.get(self.pos + 1).copied() == Some(b'x')
619 {
620 self.advance(); self.advance(); let hex_start = self.pos;
623 while let Some(ch) = self.peek() {
624 if ch.is_ascii_hexdigit() {
625 self.advance();
626 } else {
627 break;
628 }
629 }
630 let hex_str =
631 String::from_utf8_lossy(&self.source[hex_start..self.pos]).to_string();
632 let val = u32::from_str_radix(&hex_str, 16).unwrap_or(0) as i32;
634 (val.to_string(), false)
635 } else {
636 self.read_number()
637 };
638 let next_is_ident = self.peek().is_some_and(|c| c.is_ascii_alphabetic());
641 if !is_long && next_is_ident {
642 let mut ident = value;
644 let rest = self.read_identifier();
645 ident.push_str(&rest);
646 tokens.push(self.token(Kind::Identifier, ident, start_line, start_col));
647 } else if !is_long && self.peek() == Some(b'_') {
648 let saved_pos = self.pos;
650 let saved_line = self.line;
651 let saved_col = self.col;
652 let mut coord = value.clone();
653 let mut is_coord = true;
654 for _ in 0..4 {
655 if self.peek() == Some(b'_') {
656 self.advance(); coord.push('_');
658 if self.peek().is_some_and(|c| c.is_ascii_digit()) {
659 let (part, part_long) = self.read_number();
660 if part_long {
661 is_coord = false;
662 break;
663 }
664 coord.push_str(&part);
665 } else {
666 is_coord = false;
669 break;
670 }
671 } else {
672 is_coord = false;
673 break;
674 }
675 }
676 if is_coord
677 && !self
678 .peek()
679 .is_some_and(|c| c == b'_' || c.is_ascii_alphanumeric())
680 {
681 tokens.push(self.token(
682 Kind::CoordLiteral,
683 coord,
684 start_line,
685 start_col,
686 ));
687 } else {
688 self.pos = saved_pos;
690 self.line = saved_line;
691 self.col = saved_col;
692 if self.peek() == Some(b'_') {
694 let mut ident = value;
695 let rest = self.read_identifier();
697 ident.push_str(&rest);
698 tokens.push(self.token(
699 Kind::Identifier,
700 ident,
701 start_line,
702 start_col,
703 ));
704 } else {
705 tokens.push(self.token(Kind::Number, value, start_line, start_col));
706 }
707 }
708 } else {
709 let kind = if is_long {
710 Kind::LongLiteral
711 } else {
712 Kind::Number
713 };
714 tokens.push(self.token(kind, value, start_line, start_col));
715 }
716 }
717
718 _ => {
719 return Err(LexingError::new(
720 self.file_name.clone(),
721 format!("Unrecognized character '{}'", ch as char),
722 self.line,
723 self.col,
724 ));
725 }
726 }
727 if let Some(spans) = spans {
728 spans.extend((count..tokens.len()).map(|_| start..self.pos));
729 }
730 }
731
732 tokens.push(Token::new(
733 Kind::EndOfFile,
734 "EOF".into(),
735 self.line,
736 self.col,
737 ));
738 if let Some(spans) = spans {
739 spans.push(self.pos..self.pos);
740 }
741 Ok(())
742 }
743}