libgraphql_parser/token/
graphql_token_source.rs

1use crate::SourceMap;
2use crate::token::GraphQLToken;
3
4/// Trait for [`GraphQLToken`] lexers (iterators that generate
5/// [`GraphQLToken`]).
6///
7/// This trait enables extensibility over different sources of GraphQL text to
8/// be parsed. For example:
9/// [`StrGraphQLTokenSource`](crate::token::StrGraphQLTokenSource) is a
10/// lexer over `&str` types,
11/// [`libgraphql_macros::RustMacroGraphQLTokenSource`](https://github.com/jeffmo/libgraphql/blob/59aa00fe928249c9d7abcd2576e2e37e45345955/crates/libgraphql-macros/src/rust_macro_graphql_token_source.rs#L101)
12/// is a lexer over
13/// [`proc_macro2::Span`](https://docs.rs/proc-macro2/latest/proc_macro2/struct.Span.html)
14/// (for lexing Rust procedural macro input), etc.
15///
16/// Implementors define an [`Iterator`] that produces tokens one at a time.
17/// All lookahead, buffering, and peeking is handled by `GraphQLTokenStream`.
18///
19/// Lexers are responsible for:
20/// - Skipping whitespace (an "ignored token" per the GraphQL spec)
21/// - Accumulating trivia (comments, commas) and attaching to the next token
22/// - Emitting [`GraphQLTokenKind::Error`](crate::token::GraphQLTokenKind::Error)
23///   for lexer errors (enables error recovery)
24/// - Emitting a final token with
25///   [`GraphQLTokenKind::Eof`](crate::token::GraphQLTokenKind::Eof) carrying
26///   any trailing trivia
27///
28/// # Lifetime Parameter
29///
30/// The `'src` lifetime represents the source text that tokens are lexed from.
31/// For string-based lexers, this enables zero-copy lexing where token values
32/// can borrow directly from the input. For proc-macro lexers that must
33/// allocate strings, use `'static` as the lifetime.
34///
35/// # SourceMap
36///
37/// Each token source carries a [`SourceMap`] that maps byte offsets (stored
38/// compactly in [`ByteSpan`](crate::ByteSpan)) to resolved line/column
39/// positions. The `source_map()` method borrows it for mid-stream lookups
40/// (e.g. IDE hover), and `into_source_map()` transfers ownership to
41/// `ParseResult` after parsing completes.
42pub trait GraphQLTokenSource<'src>: Iterator<Item = GraphQLToken<'src>> {
43    /// Borrows the [`SourceMap`] for resolving byte offsets to
44    /// line/column positions while the token source is still active.
45    fn source_map(&self) -> &SourceMap<'src>;
46
47    /// Consumes the token source and returns the owned [`SourceMap`].
48    ///
49    /// Called by the parser after consuming all tokens (EOF) so the
50    /// `SourceMap` can be bundled into `ParseResult`.
51    fn into_source_map(self) -> SourceMap<'src>;
52
53    /// Collects all tokens and returns them alongside the [`SourceMap`].
54    ///
55    /// This is the preferred way to consume a token source when you
56    /// need both the tokens and position-resolution capability — e.g.
57    /// in tests or IDE tooling that operates on a full token stream.
58    ///
59    /// Using `Iterator::collect()` alone would consume `self`, making
60    /// `into_source_map()` unreachable. This method avoids that by
61    /// collecting via `by_ref()` and then extracting the source map.
62    fn collect_with_source_map(mut self) -> (Vec<GraphQLToken<'src>>, SourceMap<'src>)
63    where Self: Sized {
64        let tokens: Vec<_> = self.by_ref().collect();
65        let source_map = self.into_source_map();
66        (tokens, source_map)
67    }
68}
libgraphql_parser/token/graphql_token_source.rs

libgraphql_parser/token/
graphql_token_source.rs
