Skip to main content

gql_parser/
lexer.rs

1use crate::error::{Error, Result};
2use crate::token::{keyword_or_identifier, Token, TokenKind};
3
4#[derive(Clone, Debug)]
5pub struct Lexer<'a> {
6    input: &'a str,
7    offset: usize,
8}
9
10impl<'a> Lexer<'a> {
11    pub fn new(input: &'a str) -> Self {
12        Self { input, offset: 0 }
13    }
14
15    fn peek_char(&self) -> Option<char> {
16        self.input[self.offset..].chars().next()
17    }
18
19    fn peek_next_char(&self) -> Option<char> {
20        let mut chars = self.input[self.offset..].chars();
21        chars.next()?;
22        chars.next()
23    }
24
25    fn bump(&mut self) -> Option<char> {
26        let ch = self.peek_char()?;
27        self.offset += ch.len_utf8();
28        Some(ch)
29    }
30
31    fn token_from(&self, kind: TokenKind, start: usize) -> Token {
32        Token::new(kind, &self.input[start..self.offset], start)
33    }
34
35    fn eat_while(&mut self, pred: impl Fn(char) -> bool) {
36        while matches!(self.peek_char(), Some(ch) if pred(ch)) {
37            self.bump();
38        }
39    }
40
41    fn skip_ws_and_comments(&mut self) -> Result<()> {
42        loop {
43            self.eat_while(char::is_whitespace);
44
45            if self.input[self.offset..].starts_with("//")
46                || self.input[self.offset..].starts_with("--")
47            {
48                self.eat_while(|ch| ch != '\n');
49                continue;
50            }
51
52            if self.input[self.offset..].starts_with("/*") {
53                let start = self.offset;
54                self.offset += 2;
55                while self.offset < self.input.len() && !self.input[self.offset..].starts_with("*/")
56                {
57                    self.bump();
58                }
59                if self.input[self.offset..].starts_with("*/") {
60                    self.offset += 2;
61                } else {
62                    return Err(Error::UnterminatedBlockComment {
63                        offset: start,
64                        token_text: self.input[start..].to_owned(),
65                    });
66                }
67                continue;
68            }
69
70            break;
71        }
72        Ok(())
73    }
74
75    fn identifier(&mut self, start: usize) -> Token {
76        self.eat_while(is_identifier_continue);
77        let text = &self.input[start..self.offset];
78        Token::new(keyword_or_identifier(text), text, start)
79    }
80
81    fn quoted_identifier(&mut self, start: usize, quote: char) -> Result<Token> {
82        self.quoted_identifier_with_escape(start, quote, true)
83    }
84
85    fn quoted_identifier_no_escape(&mut self, start: usize) -> Result<Token> {
86        self.bump();
87        let quote = self.peek_char().ok_or_else(|| Error::UnrecognizedToken {
88            offset: start,
89            token_text: self.input[start..self.offset].to_owned(),
90        })?;
91        self.quoted_identifier_with_escape(start, quote, false)
92    }
93
94    fn quoted_identifier_with_escape(
95        &mut self,
96        start: usize,
97        quote: char,
98        escaping_enabled: bool,
99    ) -> Result<Token> {
100        let value = self.quoted_sequence(start, quote, escaping_enabled)?;
101        let kind = if quote == '"' {
102            TokenKind::DoubleQuotedString
103        } else {
104            TokenKind::QuotedString
105        };
106        Ok(Token::new(kind, value, start))
107    }
108
109    fn quoted_sequence(
110        &mut self,
111        start: usize,
112        quote: char,
113        escaping_enabled: bool,
114    ) -> Result<String> {
115        if self.bump() != Some(quote) {
116            return Err(Error::UnrecognizedToken {
117                offset: start,
118                token_text: self.input[start..self.offset].to_owned(),
119            });
120        }
121        let mut value = String::new();
122        loop {
123            match self.bump() {
124                Some(ch) if ch == quote => {
125                    if self.peek_char() == Some(quote) {
126                        self.bump();
127                        value.push(quote);
128                    } else {
129                        return Ok(value);
130                    }
131                }
132                Some('\\') if escaping_enabled => value.push(self.escaped_character(start)?),
133                Some(ch) => value.push(ch),
134                None => {
135                    return Err(Error::UnterminatedString {
136                        offset: start,
137                        token_text: self.input[start..].to_owned(),
138                    })
139                }
140            }
141        }
142    }
143
144    fn string(&mut self, start: usize) -> Result<Token> {
145        self.character_string(start, true)
146    }
147
148    fn string_no_escape(&mut self, start: usize) -> Result<Token> {
149        self.bump();
150        self.character_string(start, false)
151    }
152
153    fn character_string(&mut self, start: usize, mut escaping_enabled: bool) -> Result<Token> {
154        let mut value = String::new();
155        loop {
156            self.expect_string_chunk(start, &mut value, escaping_enabled)?;
157            if self.eat_literal_separator() {
158                if self.peek_char() == Some('@') && self.peek_next_char() == Some('\'') {
159                    self.bump();
160                    escaping_enabled = false;
161                    continue;
162                }
163                if self.peek_char() == Some('\'') {
164                    escaping_enabled = true;
165                    continue;
166                }
167            }
168            break;
169        }
170        Ok(Token::new(TokenKind::String, value, start))
171    }
172
173    fn expect_string_chunk(
174        &mut self,
175        start: usize,
176        value: &mut String,
177        escaping_enabled: bool,
178    ) -> Result<()> {
179        if self.bump() != Some('\'') {
180            return Err(Error::UnrecognizedToken {
181                offset: start,
182                token_text: self.input[start..self.offset].to_owned(),
183            });
184        }
185
186        loop {
187            match self.bump() {
188                Some('\'') => {
189                    if self.peek_char() == Some('\'') {
190                        self.bump();
191                        value.push('\'');
192                    } else {
193                        return Ok(());
194                    }
195                }
196                Some('\\') if escaping_enabled => value.push(self.escaped_character(start)?),
197                Some(ch) => value.push(ch),
198                None => {
199                    return Err(Error::UnterminatedString {
200                        offset: start,
201                        token_text: self.input[start..].to_owned(),
202                    })
203                }
204            }
205        }
206    }
207
208    fn escaped_character(&mut self, string_start: usize) -> Result<char> {
209        let escape_start = self.offset.saturating_sub('\\'.len_utf8());
210        match self.bump() {
211            Some('\\') => Ok('\\'),
212            Some('\'') => Ok('\''),
213            Some('"') => Ok('"'),
214            Some('`') => Ok('`'),
215            Some('t') => Ok('\t'),
216            Some('b') => Ok('\u{0008}'),
217            Some('n') => Ok('\n'),
218            Some('r') => Ok('\r'),
219            Some('f') => Ok('\u{000c}'),
220            Some('u') => self.unicode_escape(escape_start, 4),
221            Some('U') => self.unicode_escape(escape_start, 6),
222            Some(ch) => Err(Error::Message {
223                offset: escape_start,
224                message: format!("invalid string escape '\\{ch}'"),
225            }),
226            None => Err(Error::UnterminatedString {
227                offset: string_start,
228                token_text: self.input[string_start..].to_owned(),
229            }),
230        }
231    }
232
233    fn unicode_escape(&mut self, escape_start: usize, digits: usize) -> Result<char> {
234        let mut value = 0_u32;
235        for _ in 0..digits {
236            match self.bump().and_then(|ch| ch.to_digit(16)) {
237                Some(digit) => value = value * 16 + digit,
238                None => {
239                    return Err(Error::Message {
240                        offset: escape_start,
241                        message: "invalid string escape".to_owned(),
242                    })
243                }
244            }
245        }
246        char::from_u32(value).ok_or_else(|| Error::Message {
247            offset: escape_start,
248            message: "invalid string escape".to_owned(),
249        })
250    }
251
252    fn byte_string(&mut self, start: usize) -> Result<Token> {
253        self.bump();
254        let mut value = String::new();
255        loop {
256            self.expect_byte_string_chunk(start, &mut value)?;
257
258            if self.eat_literal_separator() && self.peek_char() == Some('\'') {
259                continue;
260            }
261            break;
262        }
263
264        if !value.len().is_multiple_of(2) {
265            return Err(Error::Message {
266                offset: start,
267                message: format!(
268                    "invalid byte string literal '{}'",
269                    &self.input[start..self.offset]
270                ),
271            });
272        }
273        Ok(Token::new(TokenKind::ByteString, value, start))
274    }
275
276    fn expect_byte_string_chunk(&mut self, start: usize, value: &mut String) -> Result<()> {
277        if self.bump() != Some('\'') {
278            return Err(Error::UnrecognizedToken {
279                offset: start,
280                token_text: self.input[start..self.offset].to_owned(),
281            });
282        }
283
284        loop {
285            let offset = self.offset;
286            match self.bump() {
287                Some('\'') => return Ok(()),
288                Some(ch) if ch.is_ascii_hexdigit() => value.push(ch),
289                Some(ch) if ch.is_whitespace() => {}
290                Some(ch) => {
291                    return Err(Error::Message {
292                        offset,
293                        message: format!("invalid byte string literal character '{ch}'"),
294                    })
295                }
296                None => {
297                    return Err(Error::UnterminatedString {
298                        offset: start,
299                        token_text: self.input[start..].to_owned(),
300                    })
301                }
302            }
303        }
304    }
305
306    fn eat_literal_separator(&mut self) -> bool {
307        let mut saw_newline = false;
308        while matches!(self.peek_char(), Some(ch) if ch.is_whitespace()) {
309            saw_newline |= self.peek_char().is_some_and(|ch| ch == '\n' || ch == '\r');
310            self.bump();
311        }
312        saw_newline
313    }
314
315    fn number(&mut self, start: usize) -> Result<Token> {
316        if self.peek_char() == Some('.') {
317            let mut text = String::from(".");
318            self.bump();
319            self.digits_with_underscores(start, 10, false, &mut text)?;
320            self.consume_exponent(start, &mut text)?;
321            let kind = self.consume_numeric_suffix(TokenKind::Decimal, start)?;
322            return Ok(Token::new(kind, text, start));
323        }
324
325        if self.peek_char() == Some('0') {
326            if matches!(self.peek_next_char(), Some('x' | 'X')) {
327                return self.radix_integer(start, 16);
328            }
329            if matches!(self.peek_next_char(), Some('o' | 'O')) {
330                return self.radix_integer(start, 8);
331            }
332            if matches!(self.peek_next_char(), Some('b' | 'B')) {
333                return self.radix_integer(start, 2);
334            }
335        }
336
337        let mut text = String::new();
338        self.digits_with_underscores(start, 10, false, &mut text)?;
339        let mut kind = TokenKind::Integer;
340        if self.peek_char() == Some('.') {
341            if self.fraction_continues_number() {
342                kind = TokenKind::Decimal;
343                self.bump();
344                text.push('.');
345                if matches!(self.peek_char(), Some(ch) if ch.is_ascii_digit()) {
346                    self.digits_with_underscores(start, 10, false, &mut text)?;
347                }
348            } else if self.peek_next_char().is_some_and(is_identifier_start) {
349                // `1.e5` is an exponent. A name glued on (`1.exp`) is a bad number.
350                self.bump();
351                self.eat_while(is_identifier_continue);
352                return Err(Error::BadNumber {
353                    offset: start,
354                    token_text: self.input[start..self.offset].to_owned(),
355                });
356            }
357        }
358        if matches!(self.peek_char(), Some('e' | 'E')) {
359            kind = TokenKind::Decimal;
360            self.consume_exponent(start, &mut text)?;
361        }
362        kind = self.consume_numeric_suffix(kind, start)?;
363        if matches!(self.peek_char(), Some('_')) {
364            self.bump();
365            return Err(Error::BadNumber {
366                offset: start,
367                token_text: self.input[start..self.offset].to_owned(),
368            });
369        }
370        Ok(Token::new(kind, text, start))
371    }
372
373    fn fraction_continues_number(&self) -> bool {
374        match self.peek_next_char() {
375            Some(ch) if ch.is_ascii_digit() => true,
376            Some('.') => false,
377            // `1.e5` is an exponent. `1.exp` is rejected by the caller.
378            Some('e' | 'E') => self.exponent_digits_follow(2),
379            Some(ch) if is_identifier_start(ch) => false,
380            _ => true,
381        }
382    }
383
384    fn exponent_digits_follow(&self, from: usize) -> bool {
385        match self.input[self.offset..].chars().nth(from) {
386            Some('+' | '-') => self.input[self.offset..]
387                .chars()
388                .nth(from + 1)
389                .is_some_and(|ch| ch.is_ascii_digit()),
390            Some(ch) if ch.is_ascii_digit() => true,
391            _ => false,
392        }
393    }
394
395    fn consume_exponent(&mut self, start: usize, text: &mut String) -> Result<()> {
396        if !matches!(self.peek_char(), Some('e' | 'E')) {
397            return Ok(());
398        }
399        self.bump();
400        text.push('e');
401        if matches!(self.peek_char(), Some('+' | '-')) {
402            text.push(self.bump().expect("peeked sign"));
403        }
404        let exp_start = self.offset;
405        self.digits_with_underscores(start, 10, false, text)?;
406        if exp_start == self.offset {
407            return Err(Error::BadNumber {
408                offset: start,
409                token_text: self.input[start..self.offset].to_owned(),
410            });
411        }
412        Ok(())
413    }
414
415    fn consume_numeric_suffix(&mut self, kind: TokenKind, start: usize) -> Result<TokenKind> {
416        let kind = if matches!(self.peek_char(), Some('m' | 'M' | 'f' | 'F' | 'd' | 'D')) {
417            self.bump();
418            TokenKind::Decimal
419        } else {
420            kind
421        };
422        if matches!(self.peek_char(), Some(ch) if is_identifier_continue(ch)) {
423            self.bump();
424            return Err(Error::BadNumber {
425                offset: start,
426                token_text: self.input[start..self.offset].to_owned(),
427            });
428        }
429        Ok(kind)
430    }
431
432    fn radix_integer(&mut self, start: usize, radix: u32) -> Result<Token> {
433        self.bump();
434        self.bump();
435        let mut digits = String::new();
436        self.digits_with_underscores(start, radix, true, &mut digits)?;
437        if digits.is_empty() || matches!(self.peek_char(), Some(ch) if is_identifier_continue(ch)) {
438            return Err(Error::BadNumber {
439                offset: start,
440                token_text: self.input[start..self.offset].to_owned(),
441            });
442        }
443        let value = match i64::from_str_radix(&digits, radix) {
444            Ok(value) => value.to_string(),
445            Err(_) => {
446                let unsigned =
447                    u64::from_str_radix(&digits, radix).map_err(|_| Error::BadNumber {
448                        offset: start,
449                        token_text: self.input[start..self.offset].to_owned(),
450                    })?;
451                // Only the negation of this magnitude fits in i64.
452                if unsigned != 1_u64 << 63 {
453                    return Err(Error::BadNumber {
454                        offset: start,
455                        token_text: self.input[start..self.offset].to_owned(),
456                    });
457                }
458                "9223372036854775808".to_owned()
459            }
460        };
461        Ok(Token::new(TokenKind::Integer, value, start))
462    }
463
464    fn digits_with_underscores(
465        &mut self,
466        start: usize,
467        radix: u32,
468        allow_leading_underscore: bool,
469        out: &mut String,
470    ) -> Result<usize> {
471        let mut digits = 0;
472        let mut pending_underscore = false;
473        loop {
474            match self.peek_char() {
475                Some('_') if allow_leading_underscore || digits > 0 => {
476                    if pending_underscore {
477                        self.bump();
478                        return Err(Error::BadNumber {
479                            offset: start,
480                            token_text: self.input[start..self.offset].to_owned(),
481                        });
482                    }
483                    pending_underscore = true;
484                    self.bump();
485                }
486                Some(ch) if ch.is_digit(radix) => {
487                    out.push(ch);
488                    digits += 1;
489                    pending_underscore = false;
490                    self.bump();
491                }
492                _ => break,
493            }
494        }
495        if pending_underscore {
496            return Err(Error::BadNumber {
497                offset: start,
498                token_text: self.input[start..self.offset].to_owned(),
499            });
500        }
501        Ok(digits)
502    }
503
504    fn parameter(&mut self, start: usize) -> Result<Token> {
505        self.bump();
506
507        if self.peek_char() == Some('@') && matches!(self.peek_next_char(), Some('"' | '`')) {
508            self.bump();
509            let quote = self.peek_char().expect("peeked parameter quote");
510            let value = self.quoted_sequence(start, quote, false)?;
511            return Ok(Token::new(TokenKind::Parameter, format!("${value}"), start));
512        }
513
514        if matches!(self.peek_char(), Some('"' | '`')) {
515            let quote = self.peek_char().expect("peeked parameter quote");
516            let value = self.quoted_sequence(start, quote, true)?;
517            return Ok(Token::new(TokenKind::Parameter, format!("${value}"), start));
518        }
519
520        if matches!(self.peek_char(), Some(ch) if is_identifier_continue(ch)) {
521            self.eat_while(is_identifier_continue);
522            return Ok(self.token_from(TokenKind::Parameter, start));
523        }
524
525        Err(Error::UnrecognizedToken {
526            offset: start,
527            token_text: "$".to_owned(),
528        })
529    }
530}
531
532impl Iterator for Lexer<'_> {
533    type Item = Result<Token>;
534
535    fn next(&mut self) -> Option<Self::Item> {
536        if let Err(err) = self.skip_ws_and_comments() {
537            return Some(Err(err));
538        }
539        let start = self.offset;
540        let ch = self.peek_char()?;
541        let token = match ch {
542            '(' => {
543                self.bump();
544                Ok(self.token_from(TokenKind::LParen, start))
545            }
546            ')' => {
547                self.bump();
548                Ok(self.token_from(TokenKind::RParen, start))
549            }
550            '[' => {
551                self.bump();
552                Ok(self.token_from(TokenKind::LBracket, start))
553            }
554            ']' => {
555                self.bump();
556                Ok(self.token_from(TokenKind::RBracket, start))
557            }
558            '{' => {
559                self.bump();
560                Ok(self.token_from(TokenKind::LBrace, start))
561            }
562            '}' => {
563                self.bump();
564                Ok(self.token_from(TokenKind::RBrace, start))
565            }
566            ',' => {
567                self.bump();
568                Ok(self.token_from(TokenKind::Comma, start))
569            }
570            ':' => {
571                self.bump();
572                if self.peek_char() == Some(':') {
573                    self.bump();
574                    Ok(self.token_from(TokenKind::DoubleColon, start))
575                } else {
576                    Ok(self.token_from(TokenKind::Colon, start))
577                }
578            }
579            ';' => {
580                self.bump();
581                Ok(self.token_from(TokenKind::Semicolon, start))
582            }
583            '.' if matches!(self.peek_next_char(), Some(ch) if ch.is_ascii_digit()) => {
584                self.number(start)
585            }
586            '.' => {
587                self.bump();
588                Ok(self.token_from(TokenKind::Dot, start))
589            }
590            '+' => {
591                self.bump();
592                Ok(self.token_from(TokenKind::Plus, start))
593            }
594            '?' => {
595                self.bump();
596                Ok(self.token_from(TokenKind::Question, start))
597            }
598            '*' => {
599                self.bump();
600                Ok(self.token_from(TokenKind::Star, start))
601            }
602            '/' => {
603                self.bump();
604                Ok(self.token_from(TokenKind::Slash, start))
605            }
606            '%' => {
607                self.bump();
608                Ok(self.token_from(TokenKind::Percent, start))
609            }
610            '|' => {
611                self.bump();
612                if self.peek_char() == Some('|') {
613                    self.bump();
614                    Ok(self.token_from(TokenKind::DoublePipe, start))
615                } else {
616                    Ok(self.token_from(TokenKind::Pipe, start))
617                }
618            }
619            '&' => {
620                self.bump();
621                Ok(self.token_from(TokenKind::Ampersand, start))
622            }
623            '~' => {
624                self.bump();
625                Ok(self.token_from(TokenKind::Tilde, start))
626            }
627            '=' => {
628                self.bump();
629                Ok(self.token_from(TokenKind::Eq, start))
630            }
631            '!' if self.peek_next_char() == Some('=') => {
632                self.bump();
633                self.bump();
634                Ok(self.token_from(TokenKind::Neq, start))
635            }
636            '!' => {
637                self.bump();
638                Ok(self.token_from(TokenKind::Bang, start))
639            }
640            '<' if self.peek_next_char() == Some('=') => {
641                self.bump();
642                self.bump();
643                Ok(self.token_from(TokenKind::Le, start))
644            }
645            '<' if self.peek_next_char() == Some('>') => {
646                self.bump();
647                self.bump();
648                Ok(self.token_from(TokenKind::Neq, start))
649            }
650            '<' if self.peek_next_char() == Some('-') => {
651                self.bump();
652                self.bump();
653                Ok(self.token_from(TokenKind::ArrowLeft, start))
654            }
655            '<' => {
656                self.bump();
657                Ok(self.token_from(TokenKind::Lt, start))
658            }
659            '>' if self.peek_next_char() == Some('=') => {
660                self.bump();
661                self.bump();
662                Ok(self.token_from(TokenKind::Ge, start))
663            }
664            '>' => {
665                self.bump();
666                Ok(self.token_from(TokenKind::Gt, start))
667            }
668            '-' if self.peek_next_char() == Some('>') => {
669                self.bump();
670                self.bump();
671                Ok(self.token_from(TokenKind::ArrowRight, start))
672            }
673            '-' => {
674                self.bump();
675                Ok(self.token_from(TokenKind::Minus, start))
676            }
677            '\'' => self.string(start),
678            '"' | '`' => self.quoted_identifier(start, ch),
679            '@' if self.peek_next_char() == Some('\'') => self.string_no_escape(start),
680            '@' if matches!(self.peek_next_char(), Some('"' | '`')) => {
681                self.quoted_identifier_no_escape(start)
682            }
683            '$' => self.parameter(start),
684            ch if ch.is_ascii_digit() => self.number(start),
685            'X' | 'x' if self.peek_next_char() == Some('\'') => self.byte_string(start),
686            ch if is_identifier_start(ch) => Ok(self.identifier(start)),
687            _ => {
688                self.bump();
689                Err(Error::UnrecognizedToken {
690                    offset: start,
691                    token_text: ch.to_string(),
692                })
693            }
694        };
695        Some(token)
696    }
697}
698
699pub fn is_identifier_start(ch: char) -> bool {
700    ch == '_' || ch.is_alphabetic()
701}
702
703pub fn is_identifier_continue(ch: char) -> bool {
704    ch == '_' || ch.is_alphanumeric()
705}