muskitty_css_tokenizer/trait_def.rs
1//! Tokenizer trait definition.
2//!
3//! CSS Syntax Module Level 3 §4.3 "Tokenizer Algorithms".
4
5use crate::types::{State, Token};
6
7/// The CSS tokenizer.
8///
9/// Implements the tokenization algorithm of CSS Syntax §4.3. The tokenizer
10/// consumes Unicode code points from an input stream (preprocessed per
11/// §5.3 to normalize newlines) and emits [`Token`]s.
12///
13/// # Algorithm structure
14///
15/// Unlike the HTML tokenizer's explicit state machine, the CSS tokenizer
16/// is recursive-descent: a single entry point `consume_a_token` (§4.3.1)
17/// dispatches to sub-algorithms (`consume_an_ident_like_token` §4.3.4,
18/// `consume_a_numeric_token` §4.3.3, `consume_a_string_token` §4.3.5,
19/// `consume_a_url_token` §4.3.6) which in turn call primitives
20/// (`consume_an_escaped_code_point` §4.3.7, `consume_an_ident_sequence`
21/// §4.3.12, `consume_a_number` §4.3.13).
22///
23/// # Reentrancy
24///
25/// The CSS tokenizer is *not* reentrant in the same way the HTML
26/// tokenizer is: CSS has no analogue of HTML's content-model switching
27/// (RCDATA / RAWTEXT / ScriptData). The state is therefore minimal —
28/// just "consuming" vs. "EOF emitted". Future CSSOM incremental-parsing
29/// use cases may require reentrancy; the trait exposes
30/// [`state`](Tokenizer::state) and [`set_state`](Tokenizer::set_state)
31/// to support that future need.
32pub trait Tokenizer {
33 /// Consume and return the next token from the input stream.
34 ///
35 /// Implements the §4.3.1 "Consume a token" algorithm at the top level.
36 /// Returns `Some(token)` for each token, ending with
37 /// `Some(Token::Eof)` for `<EOF-token>` (§5.3). After `<EOF-token>`
38 /// has been emitted, returns `None`.
39 fn next_token(&mut self) -> Option<Token>;
40
41 /// Return the current tokenizer state.
42 fn state(&self) -> State;
43
44 /// Set the current tokenizer state.
45 ///
46 /// Currently only useful for resetting to [`State::Data`] after EOF,
47 /// which re-enables token production (though the input stream is
48 /// already exhausted, so this is a no-op in practice).
49 fn set_state(&mut self, state: State);
50
51 /// Reset the tokenizer to its initial state over the same input.
52 ///
53 /// Clears any partial token state and resets position to 0. Used by
54 /// future CSSOM incremental parsing and by test harnesses.
55 fn reset(&mut self);
56
57 /// §4.3.1 L782-783: Set the `unicode_ranges_allowed` flag.
58 ///
59 /// Defaults to `false`. The `U+`/`u+` branch of §4.3.1 only produces a
60 /// `<unicode-range-token>` (§4.3.14) when this flag is `true`; otherwise
61 /// `U`/`u` is tokenized as an ident-like token. Per §4.3.14 L1500-1506,
62 /// unicode-range tokens are not produced by the top-level tokenizer under
63 /// normal circumstances — the flag is set to `true` only when parsing the
64 /// value of the `@font-face/unicode-range` descriptor.
65 fn set_unicode_ranges_allowed(&mut self, allowed: bool);
66
67 /// 当前 position(tokenizer 内部的字符索引,0-based)。
68 ///
69 /// 用于 [`next_token_with_span`](Self::next_token_with_span) 追踪 token
70 /// 的 source range。默认实现返回 0;具体 tokenizer 应覆盖。
71 ///
72 /// 注意:对 `CssTokenizer` 而言,这是 `Vec<char>` 上的 **char 索引**,
73 /// 不是 byte offset。调用方需要自行将 char 索引映射到 byte offset。
74 fn position(&self) -> usize {
75 0
76 }
77
78 /// 返回下一个 token 及其在输入流中的字符范围 `[start, end)`。
79 ///
80 /// 默认实现基于 [`position`](Self::position) + [`next_token`](Self::next_token):
81 /// 记录消费前的 position 作为 `start`,消费后的 position 作为 `end`。
82 /// 具体 tokenizer 可覆盖以获得更精确的 span。
83 ///
84 /// 返回的 range 是 **char 索引**(不是 byte offset),与
85 /// [`position`](Self::position) 一致。EOF token 的 span 是空 range
86 /// `pos..pos`(`next_token` 在 EOF 后不推进 position)。
87 fn next_token_with_span(&mut self) -> Option<(Token, std::ops::Range<usize>)> {
88 let start = self.position();
89 let token = self.next_token()?;
90 let end = self.position();
91 Some((token, start..end))
92 }
93}