Skip to main content

muskitty_css_parser/
token_stream.rs

1//! Token stream (§5.3 L1722-1814).
2//!
3//! A token stream is a struct representing a stream of tokens and/or
4//! component values. It has three fields: `tokens` (a list), `index`
5//! (current position), and `marked_indexes` (a stack of saved positions
6//! for backtracking).
7
8use muskitty_css_tokenizer::{CssTokenizer, Token, Tokenizer};
9
10/// §5.3 L1725-1754: A token stream.
11#[derive(Debug, Clone)]
12pub struct TokenStream {
13    /// §5.3 L1730-1738: A list of tokens and/or component values. We
14    /// model component values as `Token`s here for simplicity; the
15    /// §5.5 algorithms that consume "component values" wrap tokens
16    /// into [`crate::types::ComponentValue`] at the boundary.
17    pub tokens: Vec<Token>,
18    /// §5.3 L1740-1748: An index into `tokens`, representing parsing
19    /// progress. Starts at 0. Never decreases except via
20    /// [`Self::restore_mark`].
21    pub index: usize,
22    /// §5.3 L1750-1753: A stack of index values for backtracking.
23    /// Starts empty.
24    marked_indexes: Vec<usize>,
25    /// 与 `tokens` 平行的 byte ranges。空 Vec 表示无 source tracking
26    /// (`new()` 构造的 stream 无 source)。由 [`Self::with_source`] 填充。
27    token_spans: Vec<std::ops::Range<usize>>,
28    /// 原始 source text。`None` 表示无 source tracking。
29    source: Option<String>,
30}
31
32impl TokenStream {
33    /// §5.3: Construct a new token stream over `tokens`. The stream
34    /// implicitly appends an EOF token (§5.3 L1811-1813); we model
35    /// it by returning `Token::Eof` from [`Self::next_token`] when
36    /// index is out of bounds, rather than storing a sentinel.
37    ///
38    /// 构造的 stream **不带** source-text tracking。如需追踪原始 source
39    /// (用于 §5.5.6 `original_text`),请用 [`Self::with_source`]。
40    pub fn new(mut tokens: Vec<Token>) -> Self {
41        // Ensure an EOF token is present at the end (§5.3 L1811-1813).
42        // The tokenizer already emits one, but be defensive.
43        if tokens.last().is_none_or(|t| !matches!(t, Token::Eof)) {
44            tokens.push(Token::Eof);
45        }
46        Self {
47            tokens,
48            index: 0,
49            marked_indexes: Vec::new(),
50            token_spans: Vec::new(),
51            source: None,
52        }
53    }
54
55    /// 构造一个带 source-text tracking 的 TokenStream。
56    ///
57    /// tokenize `source` 并记录每个 token 的 byte range,使
58    /// [`Self::source_slice`] 能返回原始 source 片段。供 §5.5.6
59    /// `original_text`(custom property)使用。
60    pub fn with_source(source: &str) -> Self {
61        let mut tz = CssTokenizer::new(source);
62        // 建立 char-index → byte-offset 映射(tokenizer 内部用 char 索引)
63        let char_to_byte: Vec<usize> = source.char_indices().map(|(i, _)| i).collect();
64        let source_len = source.len();
65
66        let mut tokens = Vec::new();
67        let mut spans = Vec::new();
68        while let Some((token, char_range)) = tz.next_token_with_span() {
69            let start_byte = char_to_byte
70                .get(char_range.start)
71                .copied()
72                .unwrap_or(source_len);
73            let end_byte = char_to_byte
74                .get(char_range.end)
75                .copied()
76                .unwrap_or(source_len);
77            let is_eof = matches!(token, Token::Eof);
78            tokens.push(token);
79            spans.push(start_byte..end_byte);
80            if is_eof {
81                break;
82            }
83        }
84        // 防御:确保 EOF 存在(与 new() 一致)
85        if tokens.last().is_none_or(|t| !matches!(t, Token::Eof)) {
86            tokens.push(Token::Eof);
87            spans.push(source_len..source_len);
88        }
89        Self {
90            tokens,
91            index: 0,
92            marked_indexes: Vec::new(),
93            token_spans: spans,
94            source: Some(source.to_string()),
95        }
96    }
97
98    /// 返回 `tokens[start_index..end_index]` 对应的原始 source text。
99    ///
100    /// 参数是 **token 索引**(不是 char/byte 索引)。
101    /// 返回 `None` 如果无 source tracking 或 index 越界。
102    pub fn source_slice(&self, start_index: usize, end_index: usize) -> Option<&str> {
103        let source = self.source.as_ref()?;
104        if end_index == 0 || start_index >= end_index {
105            return Some("");
106        }
107        let start = self.token_spans.get(start_index)?.start;
108        let end = self.token_spans.get(end_index - 1)?.end;
109        source.get(start..end)
110    }
111
112    /// §5.3 L1769-1773: The item of `tokens` at `index`. If
113    /// out-of-bounds, return `Token::Eof`.
114    pub fn next_token(&self) -> Token {
115        self.tokens.get(self.index).cloned().unwrap_or(Token::Eof)
116    }
117
118    /// §5.3 L1775-1777: A token stream is empty if the next token is
119    /// `<EOF-token>`.
120    pub fn is_empty(&self) -> bool {
121        matches!(self.next_token(), Token::Eof)
122    }
123
124    /// §5.3 L1779-1782: Let `token` be the next token. Increment
125    /// `index`, then return `token`.
126    pub fn consume_token(&mut self) -> Token {
127        let token = self.next_token();
128        if !matches!(token, Token::Eof) {
129            self.index += 1;
130        }
131        token
132    }
133
134    /// §5.3 L1784-1786: If not empty, increment `index`.
135    pub fn discard_token(&mut self) {
136        if !self.is_empty() {
137            self.index += 1;
138        }
139    }
140
141    /// §5.3 L1788-1789: Append `index` to `marked_indexes`.
142    pub fn mark(&mut self) {
143        self.marked_indexes.push(self.index);
144    }
145
146    /// §5.3 L1791-1793: Pop from `marked_indexes` and set `index` to
147    /// the popped value. No-op if stack is empty (defensive).
148    pub fn restore_mark(&mut self) {
149        if let Some(idx) = self.marked_indexes.pop() {
150            self.index = idx;
151        }
152    }
153
154    /// §5.3 L1795-1797: Pop from `marked_indexes` and discard.
155    pub fn discard_mark(&mut self) {
156        let _ = self.marked_indexes.pop();
157    }
158
159    /// §5.3 L1799-1801: While the next token is a
160    /// `<whitespace-token>`, discard a token.
161    pub fn discard_whitespace(&mut self) {
162        while matches!(self.next_token(), Token::Whitespace) {
163            self.discard_token();
164        }
165    }
166}