sim_codec_pratt/
token.rs

1use sim_codec::DecodeBudget;
2use sim_kernel::{CodecId, PrattToken, Result, Trivia};
3
4/// A Pratt token with its source span, produced by a language-specific lexer.
5#[derive(Clone, Debug, PartialEq, Eq)]
6pub struct SpannedPrattToken {
7    /// The scanned Pratt token.
8    pub token: PrattToken,
9    /// Byte offset where the token starts in the source.
10    pub start: usize,
11    /// Byte offset just past the end of the token.
12    pub end: usize,
13    /// Whitespace and comment trivia immediately preceding the token.
14    pub leading_trivia: Vec<Trivia>,
15}
16
17impl SpannedPrattToken {
18    /// Builds a token span with no attached trivia.
19    pub fn new(token: PrattToken, start: usize, end: usize) -> Self {
20        Self {
21            token,
22            start,
23            end,
24            leading_trivia: Vec::new(),
25        }
26    }
27
28    /// Builds a token span with explicit leading trivia.
29    pub fn with_leading_trivia(
30        token: PrattToken,
31        start: usize,
32        end: usize,
33        leading_trivia: Vec<Trivia>,
34    ) -> Self {
35        Self {
36            token,
37            start,
38            end,
39            leading_trivia,
40        }
41    }
42}
43
44/// A lexer that turns source text into Pratt tokens for the shared driver.
45pub trait PrattTokenSource: Send + Sync {
46    /// Tokenizes `source` under `budget`, using `codec` for budget diagnostics.
47    fn tokenize_pratt(
48        &self,
49        codec: CodecId,
50        source: &str,
51        budget: &mut DecodeBudget,
52    ) -> Result<Vec<SpannedPrattToken>>;
53}
sim_codec_pratt/token.rs

sim_codec_pratt/
token.rs
