Skip to main content

muskitty_css_parser/
token_stream.rs

1//! Token stream (§5.3 L1722-1814).
2//!
3//! A token stream is a struct representing a stream of tokens and/or
4//! component values. It has three fields: `tokens` (a list), `index`
5//! (current position), and `marked_indexes` (a stack of saved positions
6//! for backtracking).
7
8use muskitty_css_tokenizer::Token;
9
10/// §5.3 L1725-1754: A token stream.
11#[derive(Debug, Clone)]
12pub struct TokenStream {
13    /// §5.3 L1730-1738: A list of tokens and/or component values. We
14    /// model component values as `Token`s here for simplicity; the
15    /// §5.5 algorithms that consume "component values" wrap tokens
16    /// into [`crate::types::ComponentValue`] at the boundary.
17    pub tokens: Vec<Token>,
18    /// §5.3 L1740-1748: An index into `tokens`, representing parsing
19    /// progress. Starts at 0. Never decreases except via
20    /// [`Self::restore_mark`].
21    pub index: usize,
22    /// §5.3 L1750-1753: A stack of index values for backtracking.
23    /// Starts empty.
24    marked_indexes: Vec<usize>,
25}
26
27impl TokenStream {
28    /// §5.3: Construct a new token stream over `tokens`. The stream
29    /// implicitly appends an EOF token (§5.3 L1811-1813); we model
30    /// it by returning `Token::Eof` from [`Self::next_token`] when
31    /// index is out of bounds, rather than storing a sentinel.
32    pub fn new(mut tokens: Vec<Token>) -> Self {
33        // Ensure an EOF token is present at the end (§5.3 L1811-1813).
34        // The tokenizer already emits one, but be defensive.
35        if tokens.last().is_none_or(|t| !matches!(t, Token::Eof)) {
36            tokens.push(Token::Eof);
37        }
38        Self {
39            tokens,
40            index: 0,
41            marked_indexes: Vec::new(),
42        }
43    }
44
45    /// §5.3 L1769-1773: The item of `tokens` at `index`. If
46    /// out-of-bounds, return `Token::Eof`.
47    pub fn next_token(&self) -> Token {
48        self.tokens.get(self.index).cloned().unwrap_or(Token::Eof)
49    }
50
51    /// §5.3 L1775-1777: A token stream is empty if the next token is
52    /// `<EOF-token>`.
53    pub fn is_empty(&self) -> bool {
54        matches!(self.next_token(), Token::Eof)
55    }
56
57    /// §5.3 L1779-1782: Let `token` be the next token. Increment
58    /// `index`, then return `token`.
59    pub fn consume_token(&mut self) -> Token {
60        let token = self.next_token();
61        if !matches!(token, Token::Eof) {
62            self.index += 1;
63        }
64        token
65    }
66
67    /// §5.3 L1784-1786: If not empty, increment `index`.
68    pub fn discard_token(&mut self) {
69        if !self.is_empty() {
70            self.index += 1;
71        }
72    }
73
74    /// §5.3 L1788-1789: Append `index` to `marked_indexes`.
75    pub fn mark(&mut self) {
76        self.marked_indexes.push(self.index);
77    }
78
79    /// §5.3 L1791-1793: Pop from `marked_indexes` and set `index` to
80    /// the popped value. No-op if stack is empty (defensive).
81    pub fn restore_mark(&mut self) {
82        if let Some(idx) = self.marked_indexes.pop() {
83            self.index = idx;
84        }
85    }
86
87    /// §5.3 L1795-1797: Pop from `marked_indexes` and discard.
88    pub fn discard_mark(&mut self) {
89        let _ = self.marked_indexes.pop();
90    }
91
92    /// §5.3 L1799-1801: While the next token is a
93    /// `<whitespace-token>`, discard a token.
94    pub fn discard_whitespace(&mut self) {
95        while matches!(self.next_token(), Token::Whitespace) {
96            self.discard_token();
97        }
98    }
99}