1use crate::lex::lexing::{Token, TokenType};
2use crate::utils::location::Location;
3
4#[derive(Debug, Clone)]
5pub enum LexError {
6 UnexpectedEof {
7 context: &'static str,
8 location: Location,
9 },
10 UnknownEscapeSequence {
11 ch: char,
12 location: Location,
13 },
14 UnterminatedCharLiteral {
15 location: Location,
16 },
17 UnterminatedComment {
18 location: Location,
19 },
20 UnknownCharacter {
21 location: Location,
22 },
23}
24
25impl std::fmt::Display for LexError {
26 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
27 match self {
28 LexError::UnexpectedEof { context, location } => {
29 write!(
30 f,
31 "Unexpected EOF while lexing {} at {:?}",
32 context, location
33 )
34 }
35 LexError::UnknownEscapeSequence { ch, location } => {
36 write!(f, "Unknown escape sequence '\\{}' at {:?}", ch, location)
37 }
38 LexError::UnterminatedCharLiteral { location } => {
39 write!(f, "Unterminated character literal at {:?}", location)
40 }
41 LexError::UnterminatedComment { location } => {
42 write!(f, "Unterminated multi-line comment at {:?}", location)
43 }
44 LexError::UnknownCharacter { location } => {
45 write!(f, "Unknown Character at {:?}", location)
46 }
47 }
48 }
49}
50impl std::error::Error for LexError {}
51
52pub struct Lexer {
53 pub source: String,
54 pub token_idx: usize,
55 pub line: usize,
56 pub col: usize,
57 pub tokens: Vec<Token>,
58 pub file: std::rc::Rc<str>,
59}
60
61impl Lexer {
62 pub fn new(source: String, file: impl Into<std::rc::Rc<str>>) -> Self {
63 Self {
64 source,
65 token_idx: 0,
66 tokens: Vec::new(),
67 line: 0,
68 col: 0,
69 file: file.into(),
70 }
71 }
72
73 fn current_location(&self) -> Location {
74 Location::new_with_file(self.line, self.col, self.file.clone())
75 }
76
77 fn advance(&mut self) {
78 if let Some(ch) = self.get_char() {
79 if ch == '\n' {
80 self.line += 1;
81 self.col = 0;
82 } else {
83 self.col += 1;
84 }
85 }
86 self.token_idx += 1;
87 }
88
89 fn peek(&self, offset: i32) -> Option<char> {
90 let true_offset = self.token_idx + offset as usize;
91 self.source
92 .get(true_offset..)
93 .and_then(|s| s.chars().next())
94 }
95
96 pub fn get_char(&self) -> Option<char> {
97 self.peek(0)
98 }
99
100 fn add_token(&mut self, token: Token) {
101 self.tokens.push(token);
102 }
103
104 fn single_char(&mut self, ttype: TokenType) -> Result<(), LexError> {
105 let ch = self.get_char().ok_or(LexError::UnexpectedEof {
106 context: "single char token",
107 location: self.current_location(),
108 })?;
109
110 let t = Token {
111 ttype,
112 location: self.current_location(),
113 value: ch.to_string(),
114 };
115
116 self.add_token(t);
117 self.advance();
118 Ok(())
119 }
120
121 pub fn lex(&mut self) -> Result<(), LexError> {
122 while let Some(ch) = self.get_char() {
123 if char::is_numeric(ch) {
124 let t = self.lex_numeric();
125 self.add_token(t);
126 } else if char::is_alphabetic(ch) || ch == '_' {
127 let t = self.lex_identifier_and_keyword();
128 self.add_token(t);
129 } else {
130 match ch {
131 ' ' | '\n' | '\r' | '\t' => self.advance(),
132 '=' => {
133 let t = self.lex_assign()?;
134 self.add_token(t);
135 }
136 '"' => {
137 let t = self.lex_string();
138 self.add_token(t);
139 }
140 '\'' => {
141 let t = self.lex_char()?;
142 self.add_token(t);
143 }
144 '!' => {
145 let t = self.lex_not()?;
146 self.add_token(t);
147 }
148 '|' => {
149 let t = self.lex_pipe()?;
150 self.add_token(t);
151 }
152 '>' => {
153 let t = self.lex_gt()?;
154 self.add_token(t);
155 }
156 '<' => {
157 let t = self.lex_lt()?;
158 self.add_token(t);
159 }
160 '.' => self.single_char(TokenType::Period)?,
161 '&' => {
162 let t = self.lex_amp()?;
163 self.add_token(t);
164 }
165 '^' => self.single_char(TokenType::Star)?,
166 ':' => {
167 let t = self.lex_colon()?;
168 self.add_token(t);
169 }
170 ';' => self.single_char(TokenType::SemiColon)?,
171 '(' => self.single_char(TokenType::LParen)?,
172 ')' => self.single_char(TokenType::RParen)?,
173 '{' => self.single_char(TokenType::LBrace)?,
174 '}' => self.single_char(TokenType::RBrace)?,
175 ',' => self.single_char(TokenType::Comma)?,
176 '[' => self.single_char(TokenType::LBracket)?,
177 ']' => self.single_char(TokenType::RBracket)?,
178 '+' => self.single_char(TokenType::Add)?,
179 '-' => self.single_char(TokenType::Minus)?,
180 '*' => self.single_char(TokenType::Multiply)?,
181 '/' => {
182 if let Some(t) = self.lex_slash()? {
183 self.add_token(t);
184 }
185 }
186 '%' => self.single_char(TokenType::Modulo)?,
187 _ => {
188 return Err(LexError::UnknownCharacter {
189 location: self.current_location(),
190 });
191 }
192 }
193 }
194 }
195 Ok(())
196 }
197
198 fn lex_slash(&mut self) -> Result<Option<Token>, LexError> {
199 let loc = self.current_location();
200 let current = self.get_char().ok_or(LexError::UnexpectedEof {
201 context: "'/'",
202 location: loc.clone(),
203 })?;
204
205 if self.peek(1) == Some('/') {
206 if self.peek(2) == Some('\'') {
207 self.advance(); self.advance(); self.advance(); loop {
213 match self.get_char() {
214 None => {
215 return Err(LexError::UnterminatedComment {
216 location: self.current_location(),
217 });
218 }
219 Some('\'') if self.peek(1) == Some('/') && self.peek(2) == Some('/') => {
220 self.advance();
221 self.advance();
222 self.advance();
223 break;
224 }
225 Some(_) => self.advance(),
226 }
227 }
228 } else {
229 self.advance();
230 self.advance();
231 while let Some(ch) = self.get_char() {
232 if ch == '\n' {
233 break;
234 }
235 self.advance();
236 }
237 }
238 return Ok(None);
239 }
240
241 self.advance();
242
243 Ok(Some(Token {
244 ttype: TokenType::Divide,
245 location: loc,
246 value: current.to_string(),
247 }))
248 }
249
250 fn lex_colon(&mut self) -> Result<Token, LexError> {
251 let loc = self.current_location();
252 let current = self.get_char().ok_or(LexError::UnexpectedEof {
253 context: "':'",
254 location: loc.clone(),
255 })?;
256
257 if self.peek(1) == Some(':') {
258 let next = self.peek(1).unwrap();
259 self.advance();
260 self.advance();
261 return Ok(Token {
262 ttype: TokenType::DoubleColon,
263 location: loc,
264 value: format!("{}{}", current, next),
265 });
266 }
267
268 self.advance();
269 Ok(Token {
270 ttype: TokenType::Colon,
271 location: loc,
272 value: current.to_string(),
273 })
274 }
275
276 fn lex_gt(&mut self) -> Result<Token, LexError> {
277 let loc = self.current_location();
278 let current = self.get_char().ok_or(LexError::UnexpectedEof {
279 context: "'>'",
280 location: loc.clone(),
281 })?;
282
283 if self.peek(1) == Some('=') {
284 let next = self.peek(1).unwrap();
285 self.advance();
286 self.advance();
287 return Ok(Token {
288 ttype: TokenType::GreaterThanEquals,
289 location: loc,
290 value: format!("{}{}", current, next),
291 });
292 }
293
294 self.advance();
295 Ok(Token {
296 ttype: TokenType::GreaterThan,
297 location: loc,
298 value: current.to_string(),
299 })
300 }
301
302 fn lex_amp(&mut self) -> Result<Token, LexError> {
303 let loc = self.current_location();
304 let current = self.get_char().ok_or(LexError::UnexpectedEof {
305 context: "'&'",
306 location: loc.clone(),
307 })?;
308
309 if self.peek(1) == Some('&') {
310 let next = self.peek(1).unwrap();
311 self.advance();
312 self.advance();
313 return Ok(Token {
314 ttype: TokenType::And,
315 location: loc,
316 value: format!("{}{}", current, next),
317 });
318 }
319
320 self.advance();
321 Ok(Token {
322 ttype: TokenType::Ampersand,
323 location: loc,
324 value: current.to_string(),
325 })
326 }
327
328 fn lex_pipe(&mut self) -> Result<Token, LexError> {
329 let loc = self.current_location();
330 let current = self.get_char().ok_or(LexError::UnexpectedEof {
331 context: "'|'",
332 location: loc.clone(),
333 })?;
334
335 if self.peek(1) == Some('|') {
336 let next = self.peek(1).unwrap();
337 self.advance();
338 self.advance();
339 return Ok(Token {
340 ttype: TokenType::Or,
341 location: loc,
342 value: format!("{}{}", current, next),
343 });
344 }
345
346 self.advance();
347 Err(LexError::UnknownCharacter { location: loc })
348 }
349
350 fn lex_lt(&mut self) -> Result<Token, LexError> {
351 let loc = self.current_location();
352 let current = self.get_char().ok_or(LexError::UnexpectedEof {
353 context: "'<'",
354 location: loc.clone(),
355 })?;
356
357 if self.peek(1) == Some('=') {
358 let next = self.peek(1).unwrap();
359 self.advance();
360 self.advance();
361 return Ok(Token {
362 ttype: TokenType::LessThanEquals,
363 location: loc,
364 value: format!("{}{}", current, next),
365 });
366 }
367
368 self.advance();
369 Ok(Token {
370 ttype: TokenType::LessThan,
371 location: loc,
372 value: current.to_string(),
373 })
374 }
375
376 fn lex_not(&mut self) -> Result<Token, LexError> {
377 let loc = self.current_location();
378 let current = self.get_char().ok_or(LexError::UnexpectedEof {
379 context: "'!'",
380 location: loc.clone(),
381 })?;
382
383 if self.peek(1) == Some('=') {
384 let next = self.peek(1).unwrap();
385 self.advance();
386 self.advance();
387 return Ok(Token {
388 ttype: TokenType::NotEquals,
389 location: loc,
390 value: format!("{}{}", current, next),
391 });
392 }
393
394 self.advance();
395 Ok(Token {
396 ttype: TokenType::Not,
397 location: loc,
398 value: current.to_string(),
399 })
400 }
401
402 fn lex_assign(&mut self) -> Result<Token, LexError> {
403 let loc = self.current_location();
404 let current = self.get_char().ok_or(LexError::UnexpectedEof {
405 context: "'='",
406 location: loc.clone(),
407 })?;
408
409 if self.peek(1) == Some('=') {
410 let next = self.peek(1).unwrap();
411 self.advance();
412 self.advance();
413 return Ok(Token {
414 ttype: TokenType::Equals,
415 location: loc,
416 value: format!("{}{}", current, next),
417 });
418 }
419
420 self.advance();
421 Ok(Token {
422 ttype: TokenType::Assign,
423 location: loc,
424 value: current.to_string(),
425 })
426 }
427
428 fn lex_char(&mut self) -> Result<Token, LexError> {
429 let loc = self.current_location();
430
431 self.advance(); let next_char = self.get_char();
434 self.advance();
435
436 let final_char = match next_char {
437 Some('\\') => {
438 let escape_type = self.get_char();
439 self.advance();
440
441 match escape_type {
442 Some('n') => '\n',
443 Some('t') => '\t',
444 Some('r') => '\r',
445 Some('\\') => '\\',
446 Some('\'') => '\'',
447 Some('0') => '\0',
448 Some(other) => {
449 return Err(LexError::UnknownEscapeSequence {
450 ch: other,
451 location: loc,
452 });
453 }
454 None => {
455 return Err(LexError::UnexpectedEof {
456 context: "escape sequence",
457 location: loc,
458 });
459 }
460 }
461 }
462 Some(ch) => ch,
463 None => {
464 return Err(LexError::UnexpectedEof {
465 context: "character literal",
466 location: loc,
467 });
468 }
469 };
470
471 if self.get_char() != Some('\'') {
472 return Err(LexError::UnterminatedCharLiteral { location: loc });
473 }
474 self.advance();
475
476 Ok(Token {
477 ttype: TokenType::CharLiteral,
478 location: loc,
479 value: final_char.to_string(),
480 })
481 }
482
483 fn escapers(&self, v: String) -> String {
484 let nl_replace = v.replace("\\n", "\n");
485 let tb_replace = nl_replace.replace("\\t", "\t");
486 let rb_replace = tb_replace.replace("\\r", "\r");
487 let c0_replace = rb_replace.replace("\\0", "\0");
488 c0_replace.replace("\\\"", "\"")
489 }
490
491 fn lex_string(&mut self) -> Token {
492 let loc = self.current_location();
493 let mut string: Vec<char> = Vec::new();
494 self.advance(); while let Some(ch) = self.get_char() {
497 if ch != '"' {
498 if ch == '\\' && self.peek(1).is_some() && self.peek(1).unwrap() == '"' {
499 string.push('\\');
500 string.push('"');
501 self.advance();
502 self.advance();
503 } else {
504 string.push(ch);
505 self.advance();
506 }
507 } else {
508 self.advance();
509 break;
510 }
511 }
512
513 let value: String = self.escapers(string.into_iter().collect());
514
515 Token {
516 ttype: TokenType::StringLiteral,
517 location: loc,
518 value,
519 }
520 }
521
522 fn lex_numeric(&mut self) -> Token {
523 let loc = self.current_location();
524 let mut numstring: Vec<char> = Vec::new();
525
526 while let Some(ch) = self.get_char() {
527 if ch.is_numeric() {
528 numstring.push(ch);
529 self.advance();
530 } else {
531 break;
532 }
533 }
534
535 Token {
536 ttype: TokenType::IntLiteral,
537 location: loc,
538 value: numstring.into_iter().collect(),
539 }
540 }
541
542 fn lex_identifier_and_keyword(&mut self) -> Token {
543 let loc = self.current_location();
544 let mut buf: Vec<char> = Vec::new();
545
546 while let Some(ch) = self.get_char() {
547 if ch.is_alphanumeric() || ch == '_' {
548 buf.push(ch);
549 self.advance();
550 } else {
551 break;
552 }
553 }
554
555 let value: String = buf.into_iter().collect();
556 let ttype = match value.as_str() {
557 "var" => TokenType::VarKeyword,
558 "if" => TokenType::IfKeyword,
559 "else" => TokenType::ElseKeyword,
560 "elseif" => TokenType::ElseIfKeyword,
561 "while" => TokenType::WhileKeyword,
562 "fn" => TokenType::FnKeyword,
563 "pub" => TokenType::PubKeyword,
564 "use" => TokenType::UseKeyword,
565 "for" => TokenType::ForKeyword,
566 "return" => TokenType::ReturnKeyword,
567 "extern" => TokenType::ExternKeyword,
568 "true" => TokenType::True,
569 "false" => TokenType::False,
570 "struct" => TokenType::StructKeyword,
571 "sizeof" => TokenType::SizeOfKeyword,
572 "break" => TokenType::BreakKeyword,
573 "const" => TokenType::ConstKeyword,
574 "as" => TokenType::AsKeyword,
575 _ => TokenType::Identifier,
576 };
577
578 Token {
579 ttype,
580 location: loc,
581 value,
582 }
583 }
584}