1use crate::error::LexingError;
2use crate::token::{Kind, Token};
3use crate::types::Type;
4use std::path::PathBuf;
5
6pub struct Lexer<'a> {
7 source: &'a [u8],
8 file_name: &'a PathBuf,
9 pos: usize,
10 line: usize,
11 col: usize,
12}
13
14impl<'a> Lexer<'a> {
15 pub fn new(input: &'a str, file_name: &'a PathBuf) -> Self {
16 Self {
17 source: input.as_bytes(),
18 file_name,
19 pos: 0,
20 line: 1,
21 col: 1,
22 }
23 }
24
25 fn peek(&self) -> Option<u8> {
26 self.source.get(self.pos).copied()
27 }
28
29 fn peek_ahead(&self, offset: usize) -> Option<u8> {
30 self.source.get(self.pos + offset).copied()
31 }
32
33 fn advance(&mut self) -> Option<u8> {
34 let ch = self.source.get(self.pos).copied()?;
35 self.pos += 1;
36 if ch == b'\n' {
37 self.line += 1;
38 self.col = 1;
39 } else {
40 self.col += 1;
41 }
42 Some(ch)
43 }
44
45 fn token(&self, kind: Kind, value: String, line: usize, col: usize) -> Token {
46 Token::new(kind, value, line, col)
47 }
48
49 fn skip_whitespace(&mut self) {
50 while let Some(ch) = self.peek() {
51 if ch.is_ascii_whitespace() {
52 self.advance();
53 } else {
54 break;
55 }
56 }
57 }
58
59 fn read_string_literal(&mut self) -> Result<Token, LexingError> {
60 let start_line = self.line;
61 let start_col = self.col;
62 self.advance(); let mut value = String::new();
64 let mut interp_depth: u32 = 0; loop {
66 match self.advance() {
67 Some(b'"') => {
68 if interp_depth > 0 {
70 value.push('"');
71 } else {
72 break;
73 }
74 }
75 Some(b'<') => {
76 interp_depth += 1;
77 value.push('<');
78 }
79 Some(b'>') => {
80 interp_depth = interp_depth.saturating_sub(1);
81 value.push('>');
82 }
83 Some(b'\\') => match self.advance() {
84 Some(b'\\') => value.push('\\'),
86 Some(b'"') => value.push('"'),
87 Some(b'<') => value.push('<'),
88 Some(ch) => {
89 value.push('\\');
90 value.push(ch as char);
91 }
92 None => {
93 return Err(LexingError::new(
94 self.file_name.clone(),
95 "Unterminated string literal".to_string(),
96 start_line,
97 start_col,
98 ));
99 }
100 },
101 Some(ch) => {
102 if ch < 0x80 {
104 value.push(ch as char);
105 } else if ch >= 0xC0 {
106 let mut bytes = vec![ch];
108 while let Some(&next) = self.source.get(self.pos) {
109 if (0x80..0xC0).contains(&next) {
110 bytes.push(next);
111 self.pos += 1;
112 self.col += 1;
113 } else {
114 break;
115 }
116 }
117 if let Ok(s) = std::str::from_utf8(&bytes) {
118 value.push_str(s);
119 } else {
120 for b in bytes {
122 value.push(b as char);
123 }
124 }
125 } else {
126 value.push(ch as char);
128 }
129 }
130 None => {
131 return Err(LexingError::new(
132 self.file_name.clone(),
133 "Unterminated string literal".to_string(),
134 start_line,
135 start_col,
136 ));
137 }
138 }
139 }
140 Ok(self.token(Kind::StringLiteral, value, start_line, start_col))
141 }
142
143 fn read_char_literal(&mut self) -> Result<Token, LexingError> {
144 let start_line = self.line;
145 let start_col = self.col;
146 self.advance(); let ch = match self.advance() {
148 Some(b'\\') => match self.advance() {
149 Some(b'n') => '\n',
150 Some(b't') => '\t',
151 Some(b'\\') => '\\',
152 Some(b'\'') => '\'',
153 Some(c) => c as char,
154 None => {
155 return Err(LexingError::new(
156 self.file_name.clone(),
157 "Unterminated character literal".to_string(),
158 start_line,
159 start_col,
160 ));
161 }
162 },
163 Some(c) => c as char,
164 None => {
165 return Err(LexingError::new(
166 self.file_name.clone(),
167 "Unterminated character literal".to_string(),
168 start_line,
169 start_col,
170 ));
171 }
172 };
173 match self.advance() {
174 Some(b'\'') => {}
175 _ => {
176 return Err(LexingError::new(
177 self.file_name.clone(),
178 "Expected closing single quote".to_string(),
179 self.line,
180 self.col,
181 ));
182 }
183 }
184 Ok(self.token(Kind::CharLiteral, ch.to_string(), start_line, start_col))
185 }
186
187 fn read_identifier(&mut self) -> String {
188 let start = self.pos;
189 while let Some(ch) = self.peek() {
190 if ch.is_ascii_alphanumeric() || ch == b'_' {
191 self.advance();
192 } else {
193 break;
194 }
195 }
196 String::from_utf8_lossy(&self.source[start..self.pos]).to_string()
197 }
198
199 fn read_number(&mut self) -> (String, bool) {
200 let start = self.pos;
201 let mut is_long = false;
202 while let Some(ch) = self.peek() {
203 if ch.is_ascii_digit() {
204 self.advance();
205 } else {
206 break;
207 }
208 }
209 if self.peek() == Some(b'L') || self.peek() == Some(b'l') {
211 is_long = true;
212 self.advance();
213 }
214 let number = String::from_utf8_lossy(&self.source[start..self.pos]).to_string();
215 let value = if is_long {
217 number.trim_end_matches(['L', 'l']).to_string()
218 } else {
219 number
220 };
221 (value, is_long)
222 }
223
224 fn read_single_line_comment(&mut self) -> String {
225 let start = self.pos;
226 while let Some(ch) = self.peek() {
227 if ch == b'\n' {
228 break;
229 }
230 self.advance();
231 }
232 String::from_utf8_lossy(&self.source[start..self.pos]).to_string()
233 }
234
235 fn read_multi_line_comment(&mut self) -> Result<String, LexingError> {
236 let start_line = self.line;
237 let start_col = self.col;
238 let mut comment = String::new();
239 let mut depth: u32 = 1;
240 while depth > 0 {
241 match self.advance() {
242 Some(b'/') => {
243 if self.peek() == Some(b'*') {
244 self.advance();
245 comment.push_str("/*");
246 depth += 1;
247 } else {
248 comment.push('/');
249 }
250 }
251 Some(b'*') => {
252 if self.peek() == Some(b'/') {
253 self.advance();
254 depth -= 1;
255 if depth > 0 {
256 comment.push_str("*/");
257 }
258 } else {
259 comment.push('*');
260 }
261 }
262 Some(ch) => comment.push(ch as char),
263 None => {
264 return Err(LexingError::new(
265 self.file_name.clone(),
266 "Unterminated multi-line comment".to_string(),
267 start_line,
268 start_col,
269 ));
270 }
271 }
272 }
273 Ok(comment)
274 }
275
276 fn classify_identifier(&self, ident: &str) -> Kind {
277 match ident {
278 "proc" | "clientscript" | "label" | "debugproc" | "walktrigger" | "queue" | "timer"
279 | "softtimer" | "ai_queue" | "ai_timer" | "opnpc" | "opnpc1" | "opnpc2" | "opnpc3"
280 | "opnpc4" | "opnpc5" | "opobj" | "opobj1" | "opobj2" | "opobj3" | "opobj4"
281 | "opobj5" | "oploc" | "oploc1" | "oploc2" | "oploc3" | "oploc4" | "oploc5"
282 | "opplayer" | "opplayer1" | "opplayer2" | "opplayer3" | "opplayer4" | "opplayer5"
283 | "opheld" | "opheld1" | "opheld2" | "opheld3" | "opheld4" | "opheld5" | "oplocu"
284 | "opobju" | "opnpcu" | "opplayeru" | "opheldu" | "opnpct" | "oploct" | "opplayert"
285 | "opheldt" | "opobjt" | "apnpc" | "apnpc1" | "apnpc2" | "apnpc3" | "apnpc4"
286 | "apnpc5" | "aploc" | "aploc1" | "aploc2" | "aploc3" | "aploc4" | "aploc5"
287 | "apobj" | "apobj1" | "apobj2" | "apobj3" | "apobj4" | "apobj5" | "applayer"
288 | "applayer1" | "applayer2" | "applayer3" | "applayer4" | "applayer5" | "if_button"
289 | "if_close" | "if_button1" | "if_button2" | "if_button3" | "if_button4"
290 | "if_button5" | "if_button6" | "if_button7" | "if_button8" | "if_button9"
291 | "if_button10" | "if_buttond" | "login" | "logout" | "inv_button1" | "inv_button2"
292 | "inv_button3" | "inv_button4" | "inv_button5" | "inv_buttond" | "mapzone"
293 | "maplength" | "mapenter" => Kind::Trigger,
294
295 "if" => Kind::If,
296 "else" => Kind::Else,
297 "while" => Kind::While,
298 "return" => Kind::Return,
299 "calc" => Kind::Calc,
300 "null" => Kind::Null,
301 "true" => Kind::BooleanTrue,
302 "false" => Kind::BooleanFalse,
303 "case" => Kind::Case,
304 "default" => Kind::Default,
305
306 s if s.starts_with("switch_") => Kind::Switch,
307
308 s if s.starts_with("def_") => {
309 if Type::from_def_str(s).is_some() {
310 Kind::Def
311 } else {
312 Kind::Identifier
313 }
314 }
315
316 _ => Kind::Identifier,
317 }
318 }
319
320 pub fn tokenize(&mut self) -> Result<Vec<Token>, LexingError> {
321 let mut tokens = Vec::new();
322
323 loop {
324 self.skip_whitespace();
325
326 let Some(ch) = self.peek() else {
327 break;
328 };
329
330 let start_line = self.line;
331 let start_col = self.col;
332
333 match ch {
334 b'[' => {
335 self.advance();
336 tokens.push(self.token(Kind::LBracket, "[".into(), start_line, start_col));
337 }
338 b']' => {
339 self.advance();
340 tokens.push(self.token(Kind::RBracket, "]".into(), start_line, start_col));
341 }
342 b'(' => {
343 self.advance();
344 tokens.push(self.token(Kind::LParen, "(".into(), start_line, start_col));
345 }
346 b')' => {
347 self.advance();
348 tokens.push(self.token(Kind::RParen, ")".into(), start_line, start_col));
349 }
350 b'{' => {
351 self.advance();
352 tokens.push(self.token(Kind::LBrace, "{".into(), start_line, start_col));
353 }
354 b'}' => {
355 self.advance();
356 tokens.push(self.token(Kind::RBrace, "}".into(), start_line, start_col));
357 }
358 b';' => {
359 self.advance();
360 tokens.push(self.token(Kind::Semicolon, ";".into(), start_line, start_col));
361 }
362 b',' => {
363 self.advance();
364 tokens.push(self.token(Kind::Comma, ",".into(), start_line, start_col));
365 }
366 b':' => {
367 self.advance();
368 tokens.push(self.token(Kind::Colon, ":".into(), start_line, start_col));
369 }
370 b'&' => {
371 self.advance();
372 tokens.push(self.token(Kind::LogicalAnd, "&".into(), start_line, start_col));
373 }
374 b'|' => {
375 self.advance();
376 tokens.push(self.token(Kind::LogicalOr, "|".into(), start_line, start_col));
377 }
378 b'!' => {
379 self.advance();
380 tokens.push(self.token(Kind::Not, "!".into(), start_line, start_col));
381 }
382 b'#' => {
383 self.advance();
384 while let Some(c) = self.peek() {
385 if c == b'\n' {
386 break;
387 }
388 self.advance();
389 }
390 }
391
392 b'.' => {
393 self.advance();
394 tokens.push(self.token(Kind::Dot, ".".into(), start_line, start_col));
395 }
396 b'~' => {
397 self.advance();
398 tokens.push(self.token(Kind::ScriptCall, "~".into(), start_line, start_col));
399 }
400 b'@' => {
401 self.advance();
402 tokens.push(self.token(Kind::JumpCall, "@".into(), start_line, start_col));
403 }
404
405 b'$' => {
406 self.advance(); tokens.push(self.token(Kind::LocalVar, "$".into(), start_line, start_col));
408 }
409
410 b'%' => {
411 self.advance(); let is_gamevar = match self.peek() {
419 Some(c) => c.is_ascii_alphabetic() || c == b'_',
420 None => false,
421 };
422 if is_gamevar {
423 tokens.push(self.token(Kind::GameVar, "%".into(), start_line, start_col));
424 } else {
425 tokens.push(self.token(
426 Kind::BinaryOperator,
427 "%".into(),
428 start_line,
429 start_col,
430 ));
431 }
432 }
433
434 b'^' => {
435 self.advance(); tokens.push(self.token(Kind::ConstantVar, "^".into(), start_line, start_col));
437 }
438
439 b'=' => {
440 self.advance();
441 tokens.push(self.token(Kind::Equals, "=".into(), start_line, start_col));
442 }
443
444 b'<' => {
445 self.advance();
446 if self.peek() == Some(b'=') {
447 self.advance();
448 tokens.push(self.token(
449 Kind::ComparisonOperator,
450 "<=".into(),
451 start_line,
452 start_col,
453 ));
454 } else {
455 tokens.push(self.token(
456 Kind::ComparisonOperator,
457 "<".into(),
458 start_line,
459 start_col,
460 ));
461 }
462 }
463 b'>' => {
464 self.advance();
465 if self.peek() == Some(b'=') {
466 self.advance();
467 tokens.push(self.token(
468 Kind::ComparisonOperator,
469 ">=".into(),
470 start_line,
471 start_col,
472 ));
473 } else {
474 tokens.push(self.token(
475 Kind::ComparisonOperator,
476 ">".into(),
477 start_line,
478 start_col,
479 ));
480 }
481 }
482
483 b'+' | b'-' | b'*' => {
484 self.advance();
485 tokens.push(self.token(
487 Kind::BinaryOperator,
488 (ch as char).to_string(),
489 start_line,
490 start_col,
491 ));
492 }
493
494 b'/' => {
495 self.advance();
496 match self.peek() {
497 Some(b'/') => {
498 self.advance(); let comment = self.read_single_line_comment();
500 tokens.push(self.token(
501 Kind::SingleLineComment,
502 comment,
503 start_line,
504 start_col,
505 ));
506 }
507 Some(b'*') => {
508 self.advance(); let comment = self.read_multi_line_comment()?;
510 tokens.push(self.token(
511 Kind::MultiLineComment,
512 comment,
513 start_line,
514 start_col,
515 ));
516 }
517 _ => {
518 tokens.push(self.token(
519 Kind::BinaryOperator,
520 "/".into(),
521 start_line,
522 start_col,
523 ));
524 }
525 }
526 }
527
528 b'"' => {
529 let tok = self.read_string_literal()?;
530 tokens.push(tok);
531 }
532
533 b'\'' => {
534 let tok = self.read_char_literal()?;
535 tokens.push(tok);
536 }
537
538 b'_' => {
539 if self
541 .peek_ahead(1)
542 .is_some_and(|c| c.is_ascii_alphanumeric() || c == b'_')
543 {
544 let ident = self.read_identifier();
545 let kind = self.classify_identifier(&ident);
546 tokens.push(self.token(kind, ident, start_line, start_col));
547 } else {
548 self.advance();
549 tokens.push(self.token(
550 Kind::Underscore,
551 "_".into(),
552 start_line,
553 start_col,
554 ));
555 }
556 }
557
558 c if c.is_ascii_alphabetic() => {
559 let ident = self.read_identifier();
560 let kind = self.classify_identifier(&ident);
561 tokens.push(self.token(kind, ident, start_line, start_col));
562 }
563
564 c if c.is_ascii_digit() => {
565 let (value, is_long) = if c == b'0'
567 && self.source.get(self.pos + 1).copied() == Some(b'x')
568 {
569 self.advance(); self.advance(); let hex_start = self.pos;
572 while let Some(ch) = self.peek() {
573 if ch.is_ascii_hexdigit() {
574 self.advance();
575 } else {
576 break;
577 }
578 }
579 let hex_str =
580 String::from_utf8_lossy(&self.source[hex_start..self.pos]).to_string();
581 let val = u32::from_str_radix(&hex_str, 16).unwrap_or(0) as i32;
583 (val.to_string(), false)
584 } else {
585 self.read_number()
586 };
587 let next_is_ident = self.peek().is_some_and(|c| c.is_ascii_alphabetic());
590 if !is_long && next_is_ident {
591 let mut ident = value;
593 let rest = self.read_identifier();
594 ident.push_str(&rest);
595 tokens.push(self.token(Kind::Identifier, ident, start_line, start_col));
596 } else if !is_long && self.peek() == Some(b'_') {
597 let saved_pos = self.pos;
599 let saved_line = self.line;
600 let saved_col = self.col;
601 let mut coord = value.clone();
602 let mut is_coord = true;
603 for _ in 0..4 {
604 if self.peek() == Some(b'_') {
605 self.advance(); coord.push('_');
607 if self.peek().is_some_and(|c| c.is_ascii_digit()) {
608 let (part, part_long) = self.read_number();
609 if part_long {
610 is_coord = false;
611 break;
612 }
613 coord.push_str(&part);
614 } else {
615 is_coord = false;
618 break;
619 }
620 } else {
621 is_coord = false;
622 break;
623 }
624 }
625 if is_coord
626 && !self
627 .peek()
628 .is_some_and(|c| c == b'_' || c.is_ascii_alphanumeric())
629 {
630 tokens.push(self.token(
631 Kind::CoordLiteral,
632 coord,
633 start_line,
634 start_col,
635 ));
636 } else {
637 self.pos = saved_pos;
639 self.line = saved_line;
640 self.col = saved_col;
641 if self.peek() == Some(b'_') {
643 let mut ident = value;
644 let rest = self.read_identifier();
646 ident.push_str(&rest);
647 tokens.push(self.token(
648 Kind::Identifier,
649 ident,
650 start_line,
651 start_col,
652 ));
653 } else {
654 tokens.push(self.token(Kind::Number, value, start_line, start_col));
655 }
656 }
657 } else {
658 let kind = if is_long {
659 Kind::LongLiteral
660 } else {
661 Kind::Number
662 };
663 tokens.push(self.token(kind, value, start_line, start_col));
664 }
665 }
666
667 _ => {
668 return Err(LexingError::new(
669 self.file_name.clone(),
670 format!("Unrecognized character '{}'", ch as char),
671 self.line,
672 self.col,
673 ));
674 }
675 }
676 }
677
678 tokens.push(Token::new(
679 Kind::EndOfFile,
680 "EOF".into(),
681 self.line,
682 self.col,
683 ));
684 Ok(tokens)
685 }
686}