oxc_graphql_parser/lexer/
mod.rs

1mod cursor;
2mod lookup;
3mod token;
4mod token_kind;
5
6use crate::Error;
7use crate::LimitTracker;
8use crate::lexer::cursor::Cursor;
9use crate::lexer::lookup::ByteClass;
10use std::hint::cold_path;
11pub use token::Token;
12pub use token_kind::TokenKind;
13
14/// Parses GraphQL source text into tokens.
15/// ```rust
16/// use oxc_graphql_parser::Lexer;
17///
18/// let query = "
19/// {
20///     animal
21///     ...snackSelection
22///     ... on Pet {
23///       playmates {
24///         count
25///       }
26///     }
27/// }
28/// ";
29/// let (tokens, errors) = Lexer::new(query).lex();
30/// assert_eq!(errors.len(), 0);
31/// ```
32#[derive(Debug)]
33pub struct Lexer<'a> {
34    finished: bool,
35    cursor: Cursor<'a>,
36    pub(crate) limit_tracker: LimitTracker,
37}
38
39/// States of the number token state machine.
40enum NumberState {
41    MinusSign,
42    LeadingZero,
43    IntegerPart,
44    DecimalPoint,
45    FractionalPart,
46    ExponentIndicator,
47    ExponentSign,
48    ExponentDigit,
49}
50
51impl<'a> Lexer<'a> {
52    /// Create a lexer for a GraphQL source text.
53    ///
54    /// The Lexer is an iterator over tokens and errors:
55    /// ```rust
56    /// use oxc_graphql_parser::Lexer;
57    ///
58    /// let query = "# --- GraphQL here ---";
59    ///
60    /// let mut lexer = Lexer::new(query);
61    /// let mut tokens = vec![];
62    /// for token in lexer {
63    ///     match token {
64    ///         Ok(token) => tokens.push(token),
65    ///         Err(error) => panic!("{:?}", error),
66    ///     }
67    /// }
68    /// ```
69    pub fn new(input: &'a str) -> Self {
70        Self {
71            cursor: Cursor::new(input),
72            finished: false,
73            limit_tracker: LimitTracker::new(usize::MAX),
74        }
75    }
76
77    pub fn with_limit(mut self, limit: usize) -> Self {
78        self.limit_tracker = LimitTracker::new(limit);
79        self
80    }
81
82    /// Lex the full source text, consuming the lexer.
83    pub fn lex(self) -> (Vec<Token<'a>>, Vec<Error>) {
84        let mut tokens = vec![];
85        let mut errors = vec![];
86
87        for item in self {
88            match item {
89                Ok(token) => tokens.push(token),
90                Err(error) => errors.push(error),
91            }
92        }
93
94        (tokens, errors)
95    }
96
97    /// Returns the next token, skipping whitespace and comma trivia without
98    /// materializing tokens for them. Comments are returned so the caller can
99    /// record their spans.
100    ///
101    /// Each skipped trivia token still counts toward the token limit, exactly
102    /// as if it had been yielded by the iterator.
103    pub(crate) fn next_significant(&mut self) -> Option<Result<Token<'a>, Error>> {
104        if self.finished {
105            return None;
106        }
107
108        loop {
109            if self.limit_tracker.check_and_increment() {
110                self.finished = true;
111                return Some(Err(Error::limit(
112                    "token limit reached, aborting lexing",
113                    self.cursor.index(),
114                )));
115            }
116
117            if self.cursor.skip_trivia() {
118                continue;
119            }
120
121            return match self.cursor.advance() {
122                Ok(token) => {
123                    if matches!(token.kind(), TokenKind::Eof) {
124                        self.finished = true;
125                    }
126
127                    Some(Ok(token))
128                }
129                Err(err) => Some(Err(err)),
130            };
131        }
132    }
133}
134
135impl<'a> Iterator for Lexer<'a> {
136    type Item = Result<Token<'a>, Error>;
137
138    #[inline]
139    fn next(&mut self) -> Option<Self::Item> {
140        if self.finished {
141            return None;
142        }
143
144        if self.limit_tracker.check_and_increment() {
145            self.finished = true;
146            return Some(Err(Error::limit(
147                "token limit reached, aborting lexing",
148                self.cursor.index(),
149            )));
150        }
151
152        match self.cursor.advance() {
153            Ok(token) => {
154                if matches!(token.kind(), TokenKind::Eof) {
155                    self.finished = true;
156                }
157
158                Some(Ok(token))
159            }
160            Err(err) => Some(Err(err)),
161        }
162    }
163}
164
165impl<'a> Cursor<'a> {
166    fn advance(&mut self) -> Result<Token<'a>, Error> {
167        // A pending error is only ever set and consumed within `lex_string`;
168        // every other token starts with a clean slate.
169        debug_assert!(self.err.is_none());
170
171        let mut token = Token { kind: TokenKind::Eof, data: "", index: self.index() };
172
173        let Some(c) = self.bump() else {
174            cold_path();
175            // Report EOF at the end of the input rather than one byte past it.
176            let end = self.source.len();
177            self.offset = end;
178            token.index = end;
179            return Ok(token);
180        };
181
182        match lookup::byte_class(c) {
183            ByteClass::Bang => self.punctuation(token, TokenKind::Bang),
184            ByteClass::Dollar => self.punctuation(token, TokenKind::Dollar),
185            ByteClass::Amp => self.punctuation(token, TokenKind::Amp),
186            ByteClass::LParen => self.punctuation(token, TokenKind::LParen),
187            ByteClass::RParen => self.punctuation(token, TokenKind::RParen),
188            ByteClass::Comma => self.punctuation(token, TokenKind::Comma),
189            ByteClass::Colon => self.punctuation(token, TokenKind::Colon),
190            ByteClass::Eq => self.punctuation(token, TokenKind::Eq),
191            ByteClass::At => self.punctuation(token, TokenKind::At),
192            ByteClass::LBracket => self.punctuation(token, TokenKind::LBracket),
193            ByteClass::RBracket => self.punctuation(token, TokenKind::RBracket),
194            ByteClass::LCurly => self.punctuation(token, TokenKind::LCurly),
195            ByteClass::RCurly => self.punctuation(token, TokenKind::RCurly),
196            ByteClass::Pipe => self.punctuation(token, TokenKind::Pipe),
197            ByteClass::Name => {
198                token.kind = TokenKind::Name;
199                token.data = self.consume_name();
200                Ok(token)
201            }
202            ByteClass::Whitespace => {
203                token.kind = TokenKind::Whitespace;
204                token.data = self.consume_whitespace();
205                Ok(token)
206            }
207            ByteClass::Bom => {
208                if self.eat_bom() {
209                    token.kind = TokenKind::Whitespace;
210                    token.data = self.consume_whitespace();
211                    Ok(token)
212                } else {
213                    self.unexpected_character(c, &token)
214                }
215            }
216            ByteClass::Quote => self.lex_string_start(token),
217            ByteClass::Hash => self.lex_comment(token),
218            ByteClass::Dot => self.lex_spread(token),
219            ByteClass::Zero => self.lex_number(NumberState::LeadingZero, token),
220            ByteClass::Digit => self.lex_number(NumberState::IntegerPart, token),
221            ByteClass::Minus => self.lex_number(NumberState::MinusSign, token),
222            ByteClass::Other => {
223                cold_path();
224                self.unexpected_character(c, &token)
225            }
226        }
227    }
228
229    /// Skips one trivia token (whitespace run or comma) without materializing
230    /// it. Returns `false` when the next token is significant. Comments are
231    /// not skipped: callers record their spans, so they lex as normal tokens.
232    fn skip_trivia(&mut self) -> bool {
233        let Some(&c) = self.bytes.get(self.next) else {
234            return false;
235        };
236        match lookup::byte_class(c) {
237            ByteClass::Whitespace => {
238                self.bump();
239                self.consume_whitespace();
240                true
241            }
242            ByteClass::Comma => {
243                self.bump();
244                // Update the cursor position exactly like lexing the token would.
245                let _ = self.current_str();
246                true
247            }
248            ByteClass::Bom if self.at_bom() => {
249                self.bump();
250                self.eat_bom();
251                self.consume_whitespace();
252                true
253            }
254            _ => false,
255        }
256    }
257
258    #[inline]
259    fn punctuation(&mut self, mut token: Token<'a>, kind: TokenKind) -> Result<Token<'a>, Error> {
260        token.kind = kind;
261        token.data = self.current_str();
262        Ok(token)
263    }
264
265    fn lex_comment(&mut self, mut token: Token<'a>) -> Result<Token<'a>, Error> {
266        token.kind = TokenKind::Comment;
267        let start = self.index;
268        let end = self.seek_line_end();
269        token.data = &self.source[start..end];
270        Ok(token)
271    }
272
273    fn lex_spread(&mut self, mut token: Token<'a>) -> Result<Token<'a>, Error> {
274        token.kind = TokenKind::Spread;
275        if let Some(c) = self.bump() {
276            if c == b'.' {
277                if self.eatc(b'.') {
278                    token.data = self.current_str();
279                    return Ok(token);
280                }
281            } else if !c.is_ascii() {
282                // Consume the whole character so the error data slices at a
283                // character boundary.
284                self.consume_current_char();
285            }
286        }
287        let data = self.current_str();
288        Err(Error::with_loc("Unterminated spread operator", data.to_string(), token.index))
289    }
290
291    fn lex_string_start(&mut self, mut token: Token<'a>) -> Result<Token<'a>, Error> {
292        token.kind = TokenKind::StringValue;
293
294        if self.eatc(b'"') {
295            if self.eatc(b'"') {
296                return self.lex_block_string(token);
297            }
298
299            // Empty string: `""`.
300            token.data = self.current_str();
301            return Ok(token);
302        }
303
304        if self.next == self.bytes.len() {
305            // A lone `"` at the end of the input.
306            return Err(Error::with_loc(
307                "unexpected end of data while lexing string value",
308                self.current_str().to_string(),
309                token.index,
310            ));
311        }
312
313        self.lex_string(token)
314    }
315
316    fn lex_string(&mut self, mut token: Token<'a>) -> Result<Token<'a>, Error> {
317        loop {
318            let Some(found) = memchr::memchr2(b'"', b'\\', &self.bytes[self.next..]) else {
319                cold_path();
320                return self.unterminated_string(&token);
321            };
322            let stop = self.next + found;
323
324            if memchr::memchr2(b'\n', b'\r', &self.bytes[self.next..stop]).is_some() {
325                cold_path();
326                self.add_err(Error::with_loc("unexpected line terminator", String::new(), 0));
327            }
328
329            // Consume through the stop byte.
330            self.offset = stop;
331            self.next = stop + 1;
332
333            if self.bytes[stop] == b'"' {
334                token.data = self.current_str();
335                return self.done(token);
336            }
337
338            // Backslash escape sequence.
339            let Some(c) = self.bump() else {
340                cold_path();
341                return self.unterminated_string(&token);
342            };
343            if c == b'u' {
344                // `\uXXXX`: four hex digits. A non-hex byte is consumed as
345                // plain string content after recording an error.
346                for remaining in (1..=4usize).rev() {
347                    let Some(c) = self.bump() else {
348                        cold_path();
349                        return self.unterminated_string(&token);
350                    };
351                    if c == b'"' {
352                        self.add_err(Error::with_loc(
353                            "incomplete unicode escape sequence",
354                            char::from(c).to_string(),
355                            token.index,
356                        ));
357                        token.data = self.current_str();
358                        return self.done(token);
359                    }
360                    if !c.is_ascii_hexdigit() {
361                        self.add_err(Error::with_loc(
362                            "invalid unicode escape sequence",
363                            c.to_string(),
364                            0,
365                        ));
366                        break;
367                    }
368                    if remaining == 1 {
369                        let hex_end = self.offset + 1;
370                        let hex_start = hex_end - 4;
371                        let hex = &self.source[hex_start..hex_end];
372                        // `is_ascii_hexdigit()` checks in previous iterations ensures
373                        // this `unwrap()` does not panic:
374                        let code_point = u32::from_str_radix(hex, 16).unwrap();
375                        if char::from_u32(code_point).is_none() {
376                            // TODO: https://github.com/oxc-project/oxc-graphql-parser/issues/657 needs
377                            // changes both here and in `ast/node_ext.rs`
378                            let escape_sequence_start = hex_start - 2; // include "\u"
379                            let escape_sequence = &self.source[escape_sequence_start..hex_end];
380                            self.add_err(Error::with_loc(
381                                "surrogate code point is invalid in unicode escape sequence \
382                                 (paired surrogate not supported yet: \
383                                 https://github.com/oxc-project/oxc-graphql-parser/issues/657)",
384                                escape_sequence.to_owned(),
385                                0,
386                            ));
387                        }
388                    }
389                }
390            } else if !is_escaped_char(c) {
391                cold_path();
392                let c = self.char_for_error(c);
393                self.add_err(Error::with_loc("unexpected escaped character", c.to_string(), 0));
394            }
395        }
396    }
397
398    fn lex_block_string(&mut self, mut token: Token<'a>) -> Result<Token<'a>, Error> {
399        loop {
400            let Some(found) = memchr::memchr2(b'"', b'\\', &self.bytes[self.next..]) else {
401                cold_path();
402                return self.unterminated_string(&token);
403            };
404            let stop = self.next + found;
405
406            // Consume through the stop byte.
407            self.offset = stop;
408            self.next = stop + 1;
409
410            if self.bytes[stop] == b'"' {
411                // Require two additional quotes to complete the triple quote;
412                // a lone second quote is consumed as content.
413                if self.eatc(b'"') && self.eatc(b'"') {
414                    token.data = self.current_str();
415                    return self.done(token);
416                }
417                continue;
418            }
419
420            // Backslash. If this is \""", we need to eat 3 in total, and then
421            // continue. The lexer does not un-escape escape sequences so it's
422            // OK if we take this path for \"", even if that is technically not
423            // an escape sequence. It's also legal to write \\\""" with two
424            // literal backslashes and then the escape sequence.
425            loop {
426                let Some(c) = self.bump() else {
427                    cold_path();
428                    return self.unterminated_string(&token);
429                };
430                match c {
431                    b'\\' => {}
432                    b'"' => {
433                        if self.eatc(b'"') {
434                            self.eatc(b'"');
435                        }
436                        break;
437                    }
438                    _ => break,
439                }
440            }
441        }
442    }
443
444    fn lex_number(
445        &mut self,
446        mut state: NumberState,
447        mut token: Token<'a>,
448    ) -> Result<Token<'a>, Error> {
449        token.kind = TokenKind::Int;
450
451        loop {
452            let Some(c) = self.bump() else {
453                return match state {
454                    NumberState::MinusSign => Err(Error::with_loc(
455                        "Unexpected character \"-\"",
456                        self.current_str().to_string(),
457                        token.index,
458                    )),
459                    NumberState::DecimalPoint
460                    | NumberState::ExponentIndicator
461                    | NumberState::ExponentSign => Err(Error::with_loc(
462                        "Unexpected EOF in float value",
463                        self.current_str().to_string(),
464                        token.index,
465                    )),
466                    NumberState::LeadingZero
467                    | NumberState::IntegerPart
468                    | NumberState::FractionalPart
469                    | NumberState::ExponentDigit => {
470                        token.data = self.current_str();
471                        Ok(token)
472                    }
473                };
474            };
475
476            match state {
477                NumberState::MinusSign => match c {
478                    b'0' => {
479                        state = NumberState::LeadingZero;
480                    }
481                    curr if curr.is_ascii_digit() => {
482                        state = NumberState::IntegerPart;
483                    }
484                    _ => {
485                        let c = self.char_for_error(c);
486                        return Err(Error::with_loc(
487                            format!("Unexpected character `{c}`"),
488                            self.current_str().to_string(),
489                            token.index,
490                        ));
491                    }
492                },
493                NumberState::LeadingZero => match c {
494                    b'.' => {
495                        token.kind = TokenKind::Float;
496                        state = NumberState::DecimalPoint;
497                    }
498                    b'e' | b'E' => {
499                        token.kind = TokenKind::Float;
500                        state = NumberState::ExponentIndicator;
501                    }
502                    _ if c.is_ascii_digit() => {
503                        return Err(Error::with_loc(
504                            "Numbers must not have non-significant leading zeroes",
505                            self.current_str().to_string(),
506                            token.index,
507                        ));
508                    }
509                    _ if lookup::is_namestart(c) => {
510                        let c = char::from(c);
511                        return Err(Error::with_loc(
512                            format!("Unexpected character `{c}` as integer suffix"),
513                            self.current_str().to_string(),
514                            token.index,
515                        ));
516                    }
517                    _ => {
518                        token.data = self.prev_str();
519                        return Ok(token);
520                    }
521                },
522                NumberState::IntegerPart => match c {
523                    curr if curr.is_ascii_digit() => {}
524                    b'.' => {
525                        token.kind = TokenKind::Float;
526                        state = NumberState::DecimalPoint;
527                    }
528                    b'e' | b'E' => {
529                        token.kind = TokenKind::Float;
530                        state = NumberState::ExponentIndicator;
531                    }
532                    _ if lookup::is_namestart(c) => {
533                        let c = char::from(c);
534                        return Err(Error::with_loc(
535                            format!("Unexpected character `{c}` as integer suffix"),
536                            self.current_str().to_string(),
537                            token.index,
538                        ));
539                    }
540                    _ => {
541                        token.data = self.prev_str();
542                        return Ok(token);
543                    }
544                },
545                NumberState::DecimalPoint => match c {
546                    curr if curr.is_ascii_digit() => {
547                        state = NumberState::FractionalPart;
548                    }
549                    _ => {
550                        let c = self.char_for_error(c);
551                        return Err(Error::with_loc(
552                            format!("Unexpected character `{c}`, expected fractional digit"),
553                            self.current_str().to_string(),
554                            token.index,
555                        ));
556                    }
557                },
558                NumberState::FractionalPart => match c {
559                    curr if curr.is_ascii_digit() => {}
560                    b'e' | b'E' => {
561                        state = NumberState::ExponentIndicator;
562                    }
563                    _ if c == b'.' || lookup::is_namestart(c) => {
564                        let c = char::from(c);
565                        return Err(Error::with_loc(
566                            format!("Unexpected character `{c}` as float suffix"),
567                            self.current_str().to_string(),
568                            token.index,
569                        ));
570                    }
571                    _ => {
572                        token.data = self.prev_str();
573                        return Ok(token);
574                    }
575                },
576                NumberState::ExponentIndicator => match c {
577                    _ if c.is_ascii_digit() => {
578                        state = NumberState::ExponentDigit;
579                    }
580                    b'+' | b'-' => {
581                        state = NumberState::ExponentSign;
582                    }
583                    _ => {
584                        let c = self.char_for_error(c);
585                        return Err(Error::with_loc(
586                            format!("Unexpected character `{c}`, expected exponent digit or sign"),
587                            self.current_str().to_string(),
588                            token.index,
589                        ));
590                    }
591                },
592                NumberState::ExponentSign => match c {
593                    _ if c.is_ascii_digit() => {
594                        state = NumberState::ExponentDigit;
595                    }
596                    _ => {
597                        let c = self.char_for_error(c);
598                        return Err(Error::with_loc(
599                            format!("Unexpected character `{c}`, expected exponent digit"),
600                            self.current_str().to_string(),
601                            token.index,
602                        ));
603                    }
604                },
605                NumberState::ExponentDigit => match c {
606                    _ if c.is_ascii_digit() => {}
607                    _ if c == b'.' || lookup::is_namestart(c) => {
608                        let c = char::from(c);
609                        return Err(Error::with_loc(
610                            format!("Unexpected character `{c}` as float suffix"),
611                            self.current_str().to_string(),
612                            token.index,
613                        ));
614                    }
615                    _ => {
616                        token.data = self.prev_str();
617                        return Ok(token);
618                    }
619                },
620            }
621        }
622    }
623
624    fn unexpected_character(&mut self, c: u8, token: &Token<'a>) -> Result<Token<'a>, Error> {
625        let c = self.char_for_error(c);
626        Err(Error::with_loc(
627            format!(r#"Unexpected character "{c}""#),
628            self.current_str().to_string(),
629            token.index,
630        ))
631    }
632
633    fn unterminated_string(&mut self, token: &Token<'a>) -> Result<Token<'a>, Error> {
634        // Any pending in-string error is superseded by the unterminated error
635        // (it was never observable: only the EOF token can follow a drain).
636        self.err = None;
637        Err(Error::with_loc("unterminated string value", self.drain().to_string(), token.index))
638    }
639
640    fn char_for_error(&mut self, c: u8) -> char {
641        if c.is_ascii() { char::from(c) } else { self.consume_current_char() }
642    }
643
644    #[inline]
645    fn done(&mut self, token: Token<'a>) -> Result<Token<'a>, Error> {
646        if let Some(mut err) = self.err.take() {
647            cold_path();
648            err.set_data(token.data.to_string());
649            err.index = token.index;
650            return Err(err);
651        }
652        Ok(token)
653    }
654}
655
656/// Ignored tokens other than comments and commas are assimilated to whitespace
657/// <https://spec.graphql.org/October2021/#Ignored>
658fn is_whitespace_assimilated(c: u8) -> bool {
659    matches!(
660        c,
661        // https://spec.graphql.org/October2021/#WhiteSpace
662        b'\t'
663        | b' '
664        // https://spec.graphql.org/October2021/#LineTerminator
665        | b'\n'
666        | b'\r'
667    )
668}
669
670/// <https://spec.graphql.org/October2021/#NameContinue>
671fn is_name_continue(c: u8) -> bool {
672    matches!(c, b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9' | b'_')
673}
674
675// EscapedCharacter
676//     "  \  /  b  f  n  r  t
677fn is_escaped_char(c: u8) -> bool {
678    matches!(c, b'"' | b'\\' | b'/' | b'b' | b'f' | b'n' | b'r' | b't')
679}
oxc_graphql_parser/lexer/mod.rs

oxc_graphql_parser/lexer/
mod.rs
