libgraphql_parser/
graphql_token_stream.rs

1//! Streaming lexer that produces [`GraphQLToken`]s given some
2//! [`GraphQLTokenSource`] with a bounded lookahead buffer.
3
4use std::collections::VecDeque;
5
6use crate::SourceMap;
7use crate::token::GraphQLToken;
8use crate::token::GraphQLTokenKind;
9use crate::token::GraphQLTokenSource;
10
11/// Streaming lexer that produces [`GraphQLToken`]s given some
12/// [`GraphQLTokenSource`] with a bounded lookahead buffer.
13///
14/// This structure accepts any [`GraphQLTokenSource`] and provides
15/// lookahead capabilities while maintaining efficient streaming
16/// behavior. It centralizes buffering, peeking, and lookahead logic.
17///
18/// Since trivia is already attached to tokens by the lexer, the
19/// parser can simply call `peek()` and `consume()` without worrying
20/// about trivia.
21///
22/// # Internal Buffer Management
23///
24/// Tokens are stored in a [`VecDeque`] ring buffer. Unconsumed
25/// tokens are buffered at the back; `consume()` pops from the front
26/// and returns the owned token via O(1) `pop_front()`.
27///
28/// # Type Parameters
29///
30/// * `'src` - The lifetime of the source text that tokens are lexed
31///   from.
32/// * `TTokenSource` - The underlying token source, which must
33///   implement [`GraphQLTokenSource`] (i.e.,
34///   `Iterator<Item = GraphQLToken>`).
35///
36/// # Future TODOs
37///
38/// - Consider adding a `GraphQLTokenStreamOptions` struct to
39///   configure behavior:
40///   - `include_trivia: bool` - Whether to include
41///     preceding_trivia in tokens (can be disabled for performance
42///     when trivia is not needed)
43///   - `max_tokens: Option<usize>` - Limit total tokens returned
44///     (DoS protection)
45pub struct GraphQLTokenStream<
46    'src,
47    TTokenSource: GraphQLTokenSource<'src>,
48> {
49    token_source: TTokenSource,
50    /// Ring buffer of unconsumed tokens. Grows at the back via
51    /// `ensure_buffer_has()`; consumed from the front via
52    /// `pop_front()`.
53    buffer: VecDeque<GraphQLToken<'src>>,
54}
55
56impl<'src, TTokenSource: GraphQLTokenSource<'src>>
57    GraphQLTokenStream<'src, TTokenSource>
58{
59    /// Advance to the next token and return it as an owned value.
60    ///
61    /// Returns `None` if the stream is exhausted.
62    pub fn consume(&mut self) -> Option<GraphQLToken<'src>> {
63        self.ensure_buffer_has(1);
64        self.buffer.pop_front()
65    }
66
67    /// Returns the number of [`GraphQLToken`]s currently buffered
68    /// (unconsumed).
69    pub fn current_buffer_len(&self) -> usize {
70        self.buffer.len()
71    }
72
73    /// Fill the buffer to ensure it has at least `count`
74    /// unconsumed elements.
75    fn ensure_buffer_has(&mut self, count: usize) {
76        while self.buffer.len() < count {
77            if let Some(token) = self.token_source.next() {
78                self.buffer.push_back(token);
79            } else {
80                break;
81            }
82        }
83    }
84
85    /// Check if we've reached the end of the stream.
86    ///
87    /// Returns `true` if there are no more tokens to consume, or
88    /// if the next token is `Eof`.
89    pub fn is_at_end(&mut self) -> bool {
90        match self.peek() {
91            None => true,
92            Some(token) => {
93                matches!(token.kind, GraphQLTokenKind::Eof)
94            },
95        }
96    }
97
98    /// Creates a new token stream from a token source.
99    pub fn new(token_source: TTokenSource) -> Self {
100        Self {
101            token_source,
102            buffer: VecDeque::new(),
103        }
104    }
105
106    /// Peek at the next token without consuming it.
107    ///
108    /// Returns the front of the buffer (filling it first if
109    /// empty). Returns `None` if the stream is exhausted.
110    #[inline]
111    pub fn peek(&mut self) -> Option<&GraphQLToken<'src>> {
112        self.peek_nth(0)
113    }
114
115    /// Peek at the nth token ahead (0-indexed from next unconsumed
116    /// token).
117    ///
118    /// `peek_nth(0)` is equivalent to `peek()`.
119    ///
120    /// Fills the buffer up to `n+1` elements if needed. Returns
121    /// `None` if the stream ends before reaching position n.
122    pub fn peek_nth(
123        &mut self,
124        n: usize,
125    ) -> Option<&GraphQLToken<'src>> {
126        self.ensure_buffer_has(n + 1);
127        self.buffer.get(n)
128    }
129
130    /// Borrows the underlying token source's [`SourceMap`] for
131    /// resolving byte offsets to line/column positions mid-stream.
132    pub fn source_map(&self) -> &SourceMap<'src> {
133        self.token_source.source_map()
134    }
135
136    /// Consumes this token stream and returns the underlying
137    /// token source's [`SourceMap`].
138    ///
139    /// Called by the parser after consuming all tokens (EOF) to
140    /// bundle the SourceMap into the [`ParseResult`](crate::ParseResult).
141    pub fn into_source_map(self) -> SourceMap<'src> {
142        self.token_source.into_source_map()
143    }
144}
libgraphql_parser/graphql_token_stream.rs

libgraphql_parser/
graphql_token_stream.rs
