libgraphql_parser/
graphql_token_stream.rs

1//! Streaming lexer that produces [`GraphQLToken`]s given some
2//! [`GraphQLTokenSource`] with a bounded lookahead buffer.
3
4use std::collections::VecDeque;
5
6use crate::token::GraphQLToken;
7use crate::token::GraphQLTokenKind;
8use crate::token_source::GraphQLTokenSource;
9
10/// Streaming lexer that produces [`GraphQLToken`]s given some
11/// [`GraphQLTokenSource`] with a bounded lookahead buffer.
12///
13/// This structure accepts any [`GraphQLTokenSource`] and provides
14/// lookahead capabilities while maintaining efficient streaming
15/// behavior. It centralizes buffering, peeking, and lookahead logic.
16///
17/// Since trivia is already attached to tokens by the lexer, the
18/// parser can simply call `peek()` and `consume()` without worrying
19/// about trivia.
20///
21/// # Internal Buffer Management
22///
23/// Tokens are stored in a [`VecDeque`] ring buffer. Unconsumed
24/// tokens are buffered at the back; `consume()` pops from the front
25/// and returns the owned token via O(1) `pop_front()`.
26///
27/// # Type Parameters
28///
29/// * `'src` - The lifetime of the source text that tokens are lexed
30///   from.
31/// * `TTokenSource` - The underlying token source, which must
32///   implement [`GraphQLTokenSource`] (i.e.,
33///   `Iterator<Item = GraphQLToken>`).
34///
35/// # Future TODOs
36///
37/// - Consider adding a `GraphQLTokenStreamOptions` struct to
38///   configure behavior:
39///   - `include_trivia: bool` - Whether to include
40///     preceding_trivia in tokens (can be disabled for performance
41///     when trivia is not needed)
42///   - `max_tokens: Option<usize>` - Limit total tokens returned
43///     (DoS protection)
44pub struct GraphQLTokenStream<
45    'src,
46    TTokenSource: GraphQLTokenSource<'src>,
47> {
48    token_source: TTokenSource,
49    /// Ring buffer of unconsumed tokens. Grows at the back via
50    /// `ensure_buffer_has()`; consumed from the front via
51    /// `pop_front()`.
52    buffer: VecDeque<GraphQLToken<'src>>,
53}
54
55impl<'src, TTokenSource: GraphQLTokenSource<'src>>
56    GraphQLTokenStream<'src, TTokenSource>
57{
58    /// Advance to the next token and return it as an owned value.
59    ///
60    /// Returns `None` if the stream is exhausted.
61    pub fn consume(&mut self) -> Option<GraphQLToken<'src>> {
62        self.ensure_buffer_has(1);
63        self.buffer.pop_front()
64    }
65
66    /// Returns the number of [`GraphQLToken`]s currently buffered
67    /// (unconsumed).
68    pub fn current_buffer_len(&self) -> usize {
69        self.buffer.len()
70    }
71
72    /// Fill the buffer to ensure it has at least `count`
73    /// unconsumed elements.
74    fn ensure_buffer_has(&mut self, count: usize) {
75        while self.buffer.len() < count {
76            if let Some(token) = self.token_source.next() {
77                self.buffer.push_back(token);
78            } else {
79                break;
80            }
81        }
82    }
83
84    /// Check if we've reached the end of the stream.
85    ///
86    /// Returns `true` if there are no more tokens to consume, or
87    /// if the next token is `Eof`.
88    pub fn is_at_end(&mut self) -> bool {
89        match self.peek() {
90            None => true,
91            Some(token) => {
92                matches!(token.kind, GraphQLTokenKind::Eof)
93            },
94        }
95    }
96
97    /// Creates a new token stream from a token source.
98    pub fn new(token_source: TTokenSource) -> Self {
99        Self {
100            token_source,
101            buffer: VecDeque::new(),
102        }
103    }
104
105    /// Peek at the next token without consuming it.
106    ///
107    /// Returns the front of the buffer (filling it first if
108    /// empty). Returns `None` if the stream is exhausted.
109    #[inline]
110    pub fn peek(&mut self) -> Option<&GraphQLToken<'src>> {
111        self.peek_nth(0)
112    }
113
114    /// Peek at the nth token ahead (0-indexed from next unconsumed
115    /// token).
116    ///
117    /// `peek_nth(0)` is equivalent to `peek()`.
118    ///
119    /// Fills the buffer up to `n+1` elements if needed. Returns
120    /// `None` if the stream ends before reaching position n.
121    pub fn peek_nth(
122        &mut self,
123        n: usize,
124    ) -> Option<&GraphQLToken<'src>> {
125        self.ensure_buffer_has(n + 1);
126        self.buffer.get(n)
127    }
128}
libgraphql_parser/graphql_token_stream.rs

libgraphql_parser/
graphql_token_stream.rs
