1use crate::error::{Error, Result};
2use crate::token::{keyword_or_identifier, Token, TokenKind};
3
4#[derive(Clone, Debug)]
5pub struct Lexer<'a> {
6 input: &'a str,
7 offset: usize,
8}
9
10impl<'a> Lexer<'a> {
11 pub fn new(input: &'a str) -> Self {
12 Self { input, offset: 0 }
13 }
14
15 fn peek_char(&self) -> Option<char> {
16 self.input[self.offset..].chars().next()
17 }
18
19 fn peek_next_char(&self) -> Option<char> {
20 let mut chars = self.input[self.offset..].chars();
21 chars.next()?;
22 chars.next()
23 }
24
25 fn bump(&mut self) -> Option<char> {
26 let ch = self.peek_char()?;
27 self.offset += ch.len_utf8();
28 Some(ch)
29 }
30
31 fn token_from(&self, kind: TokenKind, start: usize) -> Token {
32 Token::new(kind, &self.input[start..self.offset], start)
33 }
34
35 fn eat_while(&mut self, pred: impl Fn(char) -> bool) {
36 while matches!(self.peek_char(), Some(ch) if pred(ch)) {
37 self.bump();
38 }
39 }
40
41 fn skip_ws_and_comments(&mut self) -> Result<()> {
42 loop {
43 self.eat_while(char::is_whitespace);
44
45 if self.input[self.offset..].starts_with("//")
46 || self.input[self.offset..].starts_with("--")
47 {
48 self.eat_while(|ch| ch != '\n');
49 continue;
50 }
51
52 if self.input[self.offset..].starts_with("/*") {
53 let start = self.offset;
54 self.offset += 2;
55 while self.offset < self.input.len() && !self.input[self.offset..].starts_with("*/")
56 {
57 self.bump();
58 }
59 if self.input[self.offset..].starts_with("*/") {
60 self.offset += 2;
61 } else {
62 return Err(Error::UnterminatedBlockComment {
63 offset: start,
64 token_text: self.input[start..].to_owned(),
65 });
66 }
67 continue;
68 }
69
70 break;
71 }
72 Ok(())
73 }
74
75 fn identifier(&mut self, start: usize) -> Token {
76 self.eat_while(is_identifier_continue);
77 let text = &self.input[start..self.offset];
78 Token::new(keyword_or_identifier(text), text, start)
79 }
80
81 fn quoted_identifier(&mut self, start: usize, quote: char) -> Result<Token> {
82 self.quoted_identifier_with_escape(start, quote, true)
83 }
84
85 fn quoted_identifier_no_escape(&mut self, start: usize) -> Result<Token> {
86 self.bump();
87 let quote = self.peek_char().ok_or_else(|| Error::UnrecognizedToken {
88 offset: start,
89 token_text: self.input[start..self.offset].to_owned(),
90 })?;
91 self.quoted_identifier_with_escape(start, quote, false)
92 }
93
94 fn quoted_identifier_with_escape(
95 &mut self,
96 start: usize,
97 quote: char,
98 escaping_enabled: bool,
99 ) -> Result<Token> {
100 let value = self.quoted_sequence(start, quote, escaping_enabled)?;
101 let kind = if quote == '"' {
102 TokenKind::DoubleQuotedString
103 } else {
104 TokenKind::QuotedString
105 };
106 Ok(Token::new(kind, value, start))
107 }
108
109 fn quoted_sequence(
110 &mut self,
111 start: usize,
112 quote: char,
113 escaping_enabled: bool,
114 ) -> Result<String> {
115 if self.bump() != Some(quote) {
116 return Err(Error::UnrecognizedToken {
117 offset: start,
118 token_text: self.input[start..self.offset].to_owned(),
119 });
120 }
121 let mut value = String::new();
122 loop {
123 match self.bump() {
124 Some(ch) if ch == quote => {
125 if self.peek_char() == Some(quote) {
126 self.bump();
127 value.push(quote);
128 } else {
129 return Ok(value);
130 }
131 }
132 Some('\\') if escaping_enabled => value.push(self.escaped_character(start)?),
133 Some(ch) => value.push(ch),
134 None => {
135 return Err(Error::UnterminatedString {
136 offset: start,
137 token_text: self.input[start..].to_owned(),
138 })
139 }
140 }
141 }
142 }
143
144 fn string(&mut self, start: usize) -> Result<Token> {
145 self.character_string(start, true)
146 }
147
148 fn string_no_escape(&mut self, start: usize) -> Result<Token> {
149 self.bump();
150 self.character_string(start, false)
151 }
152
153 fn character_string(&mut self, start: usize, mut escaping_enabled: bool) -> Result<Token> {
154 let mut value = String::new();
155 loop {
156 self.expect_string_chunk(start, &mut value, escaping_enabled)?;
157 if self.eat_literal_separator() {
158 if self.peek_char() == Some('@') && self.peek_next_char() == Some('\'') {
159 self.bump();
160 escaping_enabled = false;
161 continue;
162 }
163 if self.peek_char() == Some('\'') {
164 escaping_enabled = true;
165 continue;
166 }
167 }
168 break;
169 }
170 Ok(Token::new(TokenKind::String, value, start))
171 }
172
173 fn expect_string_chunk(
174 &mut self,
175 start: usize,
176 value: &mut String,
177 escaping_enabled: bool,
178 ) -> Result<()> {
179 if self.bump() != Some('\'') {
180 return Err(Error::UnrecognizedToken {
181 offset: start,
182 token_text: self.input[start..self.offset].to_owned(),
183 });
184 }
185
186 loop {
187 match self.bump() {
188 Some('\'') => {
189 if self.peek_char() == Some('\'') {
190 self.bump();
191 value.push('\'');
192 } else {
193 return Ok(());
194 }
195 }
196 Some('\\') if escaping_enabled => value.push(self.escaped_character(start)?),
197 Some(ch) => value.push(ch),
198 None => {
199 return Err(Error::UnterminatedString {
200 offset: start,
201 token_text: self.input[start..].to_owned(),
202 })
203 }
204 }
205 }
206 }
207
208 fn escaped_character(&mut self, string_start: usize) -> Result<char> {
209 let escape_start = self.offset.saturating_sub('\\'.len_utf8());
210 match self.bump() {
211 Some('\\') => Ok('\\'),
212 Some('\'') => Ok('\''),
213 Some('"') => Ok('"'),
214 Some('`') => Ok('`'),
215 Some('t') => Ok('\t'),
216 Some('b') => Ok('\u{0008}'),
217 Some('n') => Ok('\n'),
218 Some('r') => Ok('\r'),
219 Some('f') => Ok('\u{000c}'),
220 Some('u') => self.unicode_escape(escape_start, 4),
221 Some('U') => self.unicode_escape(escape_start, 6),
222 Some(ch) => Err(Error::Message {
223 offset: escape_start,
224 message: format!("invalid string escape '\\{ch}'"),
225 }),
226 None => Err(Error::UnterminatedString {
227 offset: string_start,
228 token_text: self.input[string_start..].to_owned(),
229 }),
230 }
231 }
232
233 fn unicode_escape(&mut self, escape_start: usize, digits: usize) -> Result<char> {
234 let mut value = 0_u32;
235 for _ in 0..digits {
236 match self.bump().and_then(|ch| ch.to_digit(16)) {
237 Some(digit) => value = value * 16 + digit,
238 None => {
239 return Err(Error::Message {
240 offset: escape_start,
241 message: "invalid string escape".to_owned(),
242 })
243 }
244 }
245 }
246 char::from_u32(value).ok_or_else(|| Error::Message {
247 offset: escape_start,
248 message: "invalid string escape".to_owned(),
249 })
250 }
251
252 fn byte_string(&mut self, start: usize) -> Result<Token> {
253 self.bump();
254 let mut value = String::new();
255 loop {
256 self.expect_byte_string_chunk(start, &mut value)?;
257
258 if self.eat_literal_separator() && self.peek_char() == Some('\'') {
259 continue;
260 }
261 break;
262 }
263
264 if !value.len().is_multiple_of(2) {
265 return Err(Error::Message {
266 offset: start,
267 message: format!(
268 "invalid byte string literal '{}'",
269 &self.input[start..self.offset]
270 ),
271 });
272 }
273 Ok(Token::new(TokenKind::ByteString, value, start))
274 }
275
276 fn expect_byte_string_chunk(&mut self, start: usize, value: &mut String) -> Result<()> {
277 if self.bump() != Some('\'') {
278 return Err(Error::UnrecognizedToken {
279 offset: start,
280 token_text: self.input[start..self.offset].to_owned(),
281 });
282 }
283
284 loop {
285 let offset = self.offset;
286 match self.bump() {
287 Some('\'') => return Ok(()),
288 Some(ch) if ch.is_ascii_hexdigit() => value.push(ch),
289 Some(ch) if ch.is_whitespace() => {}
290 Some(ch) => {
291 return Err(Error::Message {
292 offset,
293 message: format!("invalid byte string literal character '{ch}'"),
294 })
295 }
296 None => {
297 return Err(Error::UnterminatedString {
298 offset: start,
299 token_text: self.input[start..].to_owned(),
300 })
301 }
302 }
303 }
304 }
305
306 fn eat_literal_separator(&mut self) -> bool {
307 let mut saw_newline = false;
308 while matches!(self.peek_char(), Some(ch) if ch.is_whitespace()) {
309 saw_newline |= self.peek_char().is_some_and(|ch| ch == '\n' || ch == '\r');
310 self.bump();
311 }
312 saw_newline
313 }
314
315 fn number(&mut self, start: usize) -> Result<Token> {
316 if self.peek_char() == Some('.') {
317 let mut text = String::from(".");
318 self.bump();
319 self.digits_with_underscores(start, 10, false, &mut text)?;
320 self.consume_exponent(start, &mut text)?;
321 let kind = self.consume_numeric_suffix(TokenKind::Decimal, start)?;
322 return Ok(Token::new(kind, text, start));
323 }
324
325 if self.peek_char() == Some('0') {
326 if matches!(self.peek_next_char(), Some('x' | 'X')) {
327 return self.radix_integer(start, 16);
328 }
329 if matches!(self.peek_next_char(), Some('o' | 'O')) {
330 return self.radix_integer(start, 8);
331 }
332 if matches!(self.peek_next_char(), Some('b' | 'B')) {
333 return self.radix_integer(start, 2);
334 }
335 }
336
337 let mut text = String::new();
338 self.digits_with_underscores(start, 10, false, &mut text)?;
339 let mut kind = TokenKind::Integer;
340 if self.peek_char() == Some('.') {
341 if self.fraction_continues_number() {
342 kind = TokenKind::Decimal;
343 self.bump();
344 text.push('.');
345 if matches!(self.peek_char(), Some(ch) if ch.is_ascii_digit()) {
346 self.digits_with_underscores(start, 10, false, &mut text)?;
347 }
348 } else if self.peek_next_char().is_some_and(is_identifier_start) {
349 self.bump();
351 self.eat_while(is_identifier_continue);
352 return Err(Error::BadNumber {
353 offset: start,
354 token_text: self.input[start..self.offset].to_owned(),
355 });
356 }
357 }
358 if matches!(self.peek_char(), Some('e' | 'E')) {
359 kind = TokenKind::Decimal;
360 self.consume_exponent(start, &mut text)?;
361 }
362 kind = self.consume_numeric_suffix(kind, start)?;
363 if matches!(self.peek_char(), Some('_')) {
364 self.bump();
365 return Err(Error::BadNumber {
366 offset: start,
367 token_text: self.input[start..self.offset].to_owned(),
368 });
369 }
370 Ok(Token::new(kind, text, start))
371 }
372
373 fn fraction_continues_number(&self) -> bool {
374 match self.peek_next_char() {
375 Some(ch) if ch.is_ascii_digit() => true,
376 Some('.') => false,
377 Some('e' | 'E') => self.exponent_digits_follow(2),
379 Some(ch) if is_identifier_start(ch) => false,
380 _ => true,
381 }
382 }
383
384 fn exponent_digits_follow(&self, from: usize) -> bool {
385 match self.input[self.offset..].chars().nth(from) {
386 Some('+' | '-') => self.input[self.offset..]
387 .chars()
388 .nth(from + 1)
389 .is_some_and(|ch| ch.is_ascii_digit()),
390 Some(ch) if ch.is_ascii_digit() => true,
391 _ => false,
392 }
393 }
394
395 fn consume_exponent(&mut self, start: usize, text: &mut String) -> Result<()> {
396 if !matches!(self.peek_char(), Some('e' | 'E')) {
397 return Ok(());
398 }
399 self.bump();
400 text.push('e');
401 if matches!(self.peek_char(), Some('+' | '-')) {
402 text.push(self.bump().expect("peeked sign"));
403 }
404 let exp_start = self.offset;
405 self.digits_with_underscores(start, 10, false, text)?;
406 if exp_start == self.offset {
407 return Err(Error::BadNumber {
408 offset: start,
409 token_text: self.input[start..self.offset].to_owned(),
410 });
411 }
412 Ok(())
413 }
414
415 fn consume_numeric_suffix(&mut self, kind: TokenKind, start: usize) -> Result<TokenKind> {
416 let kind = if matches!(self.peek_char(), Some('m' | 'M' | 'f' | 'F' | 'd' | 'D')) {
417 self.bump();
418 TokenKind::Decimal
419 } else {
420 kind
421 };
422 if matches!(self.peek_char(), Some(ch) if is_identifier_continue(ch)) {
423 self.bump();
424 return Err(Error::BadNumber {
425 offset: start,
426 token_text: self.input[start..self.offset].to_owned(),
427 });
428 }
429 Ok(kind)
430 }
431
432 fn radix_integer(&mut self, start: usize, radix: u32) -> Result<Token> {
433 self.bump();
434 self.bump();
435 let mut digits = String::new();
436 self.digits_with_underscores(start, radix, true, &mut digits)?;
437 if digits.is_empty() || matches!(self.peek_char(), Some(ch) if is_identifier_continue(ch)) {
438 return Err(Error::BadNumber {
439 offset: start,
440 token_text: self.input[start..self.offset].to_owned(),
441 });
442 }
443 let value = match i64::from_str_radix(&digits, radix) {
444 Ok(value) => value.to_string(),
445 Err(_) => {
446 let unsigned =
447 u64::from_str_radix(&digits, radix).map_err(|_| Error::BadNumber {
448 offset: start,
449 token_text: self.input[start..self.offset].to_owned(),
450 })?;
451 if unsigned != 1_u64 << 63 {
453 return Err(Error::BadNumber {
454 offset: start,
455 token_text: self.input[start..self.offset].to_owned(),
456 });
457 }
458 "9223372036854775808".to_owned()
459 }
460 };
461 Ok(Token::new(TokenKind::Integer, value, start))
462 }
463
464 fn digits_with_underscores(
465 &mut self,
466 start: usize,
467 radix: u32,
468 allow_leading_underscore: bool,
469 out: &mut String,
470 ) -> Result<usize> {
471 let mut digits = 0;
472 let mut pending_underscore = false;
473 loop {
474 match self.peek_char() {
475 Some('_') if allow_leading_underscore || digits > 0 => {
476 if pending_underscore {
477 self.bump();
478 return Err(Error::BadNumber {
479 offset: start,
480 token_text: self.input[start..self.offset].to_owned(),
481 });
482 }
483 pending_underscore = true;
484 self.bump();
485 }
486 Some(ch) if ch.is_digit(radix) => {
487 out.push(ch);
488 digits += 1;
489 pending_underscore = false;
490 self.bump();
491 }
492 _ => break,
493 }
494 }
495 if pending_underscore {
496 return Err(Error::BadNumber {
497 offset: start,
498 token_text: self.input[start..self.offset].to_owned(),
499 });
500 }
501 Ok(digits)
502 }
503
504 fn parameter(&mut self, start: usize) -> Result<Token> {
505 self.bump();
506
507 if self.peek_char() == Some('@') && matches!(self.peek_next_char(), Some('"' | '`')) {
508 self.bump();
509 let quote = self.peek_char().expect("peeked parameter quote");
510 let value = self.quoted_sequence(start, quote, false)?;
511 return Ok(Token::new(TokenKind::Parameter, format!("${value}"), start));
512 }
513
514 if matches!(self.peek_char(), Some('"' | '`')) {
515 let quote = self.peek_char().expect("peeked parameter quote");
516 let value = self.quoted_sequence(start, quote, true)?;
517 return Ok(Token::new(TokenKind::Parameter, format!("${value}"), start));
518 }
519
520 if matches!(self.peek_char(), Some(ch) if is_identifier_continue(ch)) {
521 self.eat_while(is_identifier_continue);
522 return Ok(self.token_from(TokenKind::Parameter, start));
523 }
524
525 Err(Error::UnrecognizedToken {
526 offset: start,
527 token_text: "$".to_owned(),
528 })
529 }
530}
531
532impl Iterator for Lexer<'_> {
533 type Item = Result<Token>;
534
535 fn next(&mut self) -> Option<Self::Item> {
536 if let Err(err) = self.skip_ws_and_comments() {
537 return Some(Err(err));
538 }
539 let start = self.offset;
540 let ch = self.peek_char()?;
541 let token = match ch {
542 '(' => {
543 self.bump();
544 Ok(self.token_from(TokenKind::LParen, start))
545 }
546 ')' => {
547 self.bump();
548 Ok(self.token_from(TokenKind::RParen, start))
549 }
550 '[' => {
551 self.bump();
552 Ok(self.token_from(TokenKind::LBracket, start))
553 }
554 ']' => {
555 self.bump();
556 Ok(self.token_from(TokenKind::RBracket, start))
557 }
558 '{' => {
559 self.bump();
560 Ok(self.token_from(TokenKind::LBrace, start))
561 }
562 '}' => {
563 self.bump();
564 Ok(self.token_from(TokenKind::RBrace, start))
565 }
566 ',' => {
567 self.bump();
568 Ok(self.token_from(TokenKind::Comma, start))
569 }
570 ':' => {
571 self.bump();
572 if self.peek_char() == Some(':') {
573 self.bump();
574 Ok(self.token_from(TokenKind::DoubleColon, start))
575 } else {
576 Ok(self.token_from(TokenKind::Colon, start))
577 }
578 }
579 ';' => {
580 self.bump();
581 Ok(self.token_from(TokenKind::Semicolon, start))
582 }
583 '.' if matches!(self.peek_next_char(), Some(ch) if ch.is_ascii_digit()) => {
584 self.number(start)
585 }
586 '.' => {
587 self.bump();
588 Ok(self.token_from(TokenKind::Dot, start))
589 }
590 '+' => {
591 self.bump();
592 Ok(self.token_from(TokenKind::Plus, start))
593 }
594 '?' => {
595 self.bump();
596 Ok(self.token_from(TokenKind::Question, start))
597 }
598 '*' => {
599 self.bump();
600 Ok(self.token_from(TokenKind::Star, start))
601 }
602 '/' => {
603 self.bump();
604 Ok(self.token_from(TokenKind::Slash, start))
605 }
606 '%' => {
607 self.bump();
608 Ok(self.token_from(TokenKind::Percent, start))
609 }
610 '|' => {
611 self.bump();
612 if self.peek_char() == Some('|') {
613 self.bump();
614 Ok(self.token_from(TokenKind::DoublePipe, start))
615 } else {
616 Ok(self.token_from(TokenKind::Pipe, start))
617 }
618 }
619 '&' => {
620 self.bump();
621 Ok(self.token_from(TokenKind::Ampersand, start))
622 }
623 '~' => {
624 self.bump();
625 Ok(self.token_from(TokenKind::Tilde, start))
626 }
627 '=' => {
628 self.bump();
629 Ok(self.token_from(TokenKind::Eq, start))
630 }
631 '!' if self.peek_next_char() == Some('=') => {
632 self.bump();
633 self.bump();
634 Ok(self.token_from(TokenKind::Neq, start))
635 }
636 '!' => {
637 self.bump();
638 Ok(self.token_from(TokenKind::Bang, start))
639 }
640 '<' if self.peek_next_char() == Some('=') => {
641 self.bump();
642 self.bump();
643 Ok(self.token_from(TokenKind::Le, start))
644 }
645 '<' if self.peek_next_char() == Some('>') => {
646 self.bump();
647 self.bump();
648 Ok(self.token_from(TokenKind::Neq, start))
649 }
650 '<' if self.peek_next_char() == Some('-') => {
651 self.bump();
652 self.bump();
653 Ok(self.token_from(TokenKind::ArrowLeft, start))
654 }
655 '<' => {
656 self.bump();
657 Ok(self.token_from(TokenKind::Lt, start))
658 }
659 '>' if self.peek_next_char() == Some('=') => {
660 self.bump();
661 self.bump();
662 Ok(self.token_from(TokenKind::Ge, start))
663 }
664 '>' => {
665 self.bump();
666 Ok(self.token_from(TokenKind::Gt, start))
667 }
668 '-' if self.peek_next_char() == Some('>') => {
669 self.bump();
670 self.bump();
671 Ok(self.token_from(TokenKind::ArrowRight, start))
672 }
673 '-' => {
674 self.bump();
675 Ok(self.token_from(TokenKind::Minus, start))
676 }
677 '\'' => self.string(start),
678 '"' | '`' => self.quoted_identifier(start, ch),
679 '@' if self.peek_next_char() == Some('\'') => self.string_no_escape(start),
680 '@' if matches!(self.peek_next_char(), Some('"' | '`')) => {
681 self.quoted_identifier_no_escape(start)
682 }
683 '$' => self.parameter(start),
684 ch if ch.is_ascii_digit() => self.number(start),
685 'X' | 'x' if self.peek_next_char() == Some('\'') => self.byte_string(start),
686 ch if is_identifier_start(ch) => Ok(self.identifier(start)),
687 _ => {
688 self.bump();
689 Err(Error::UnrecognizedToken {
690 offset: start,
691 token_text: ch.to_string(),
692 })
693 }
694 };
695 Some(token)
696 }
697}
698
699pub fn is_identifier_start(ch: char) -> bool {
700 ch == '_' || ch.is_alphabetic()
701}
702
703pub fn is_identifier_continue(ch: char) -> bool {
704 ch == '_' || ch.is_alphanumeric()
705}