Skip to main content

muskitty_css_tokenizer/
impls.rs

1//! CSS Syntax tokenizer implementation.
2//!
3//! Implements the "Tokenizer Algorithms" of CSS Syntax Module Level 3 §4.3.
4//! The tokenizer is recursive-descent: [`CssTokenizer::next_token`] calls
5//! [`consume_a_token`](CssTokenizer::consume_a_token) (§4.3.1), which
6//! dispatches to sub-algorithms based on the current input code point.
7//!
8//! # Coverage
9//!
10//! All §4.3 sub-algorithms are implemented:
11//! - §4.3.1 Consume a token (full dispatch, incl. `unicode_ranges_allowed`)
12//! - §4.3.2 Consume comments
13//! - §4.3.3 Consume a numeric token
14//! - §4.3.4 Consume an ident-like token (incl. `url(` special case)
15//! - §4.3.5 Consume a string token
16//! - §4.3.6 Consume a url token
17//! - §4.3.7 Consume an escaped code point
18//! - §4.3.8 Check if two code points are a valid escape
19//! - §4.3.9 Check if three code points would start an ident sequence
20//! - §4.3.10 Check if three code points would start a number
21//! - §4.3.11 Check if three code points would start a unicode-range
22//! - §4.3.12 Consume an ident sequence
23//! - §4.3.13 Consume a number
24//! - §4.3.14 Consume a unicode-range token
25//! - §4.3.15 Consume the remnants of a bad url
26
27use crate::trait_def::Tokenizer;
28use crate::types::{HashType, Numeric, State, Token};
29
30/// The CSS tokenizer (§4.3).
31///
32/// Holds the preprocessed input stream (§5.3 normalization applied at
33/// construction), the current position, and EOF tracking. Sub-algorithms
34/// are implemented as methods on this struct so they share the input
35/// stream and position.
36pub struct CssTokenizer {
37    /// Input code points after §5.3 preprocessing (CR/LF/FF normalized to LF).
38    input: Vec<char>,
39    /// Current position in `input` (0-based index of the next code point
40    /// to consume). `pos == input.len()` means the stream is exhausted.
41    pos: usize,
42    /// Top-level state (§4.3.1 dispatch vs. EOF).
43    state: State,
44    /// Whether `<EOF-token>` has been emitted. After this is `true`,
45    /// [`next_token`](Tokenizer::next_token) returns `None`.
46    eof_emitted: bool,
47    /// §4.3.1 L782-783: `unicode_ranges_allowed` flag, default `false`.
48    /// When `true`, the `U+`/`u+` branch of §4.3.1 produces a
49    /// `<unicode-range-token>` (§4.3.14); otherwise `U`/`u` is tokenized
50    /// as an ident-like token.
51    unicode_ranges_allowed: bool,
52}
53
54impl CssTokenizer {
55    /// Construct a new tokenizer over `input`.
56    ///
57    /// Applies §5.3 input preprocessing: per the "input stream"
58    /// definition, U+000D CR followed by U+000A LF is collapsed to a
59    /// single LF, lone CR is replaced with LF, and U+000C FF is replaced
60    /// with LF. The tokenizer operates on this normalized stream.
61    pub fn new(input: &str) -> Self {
62        let input = preprocess_input(input);
63        Self {
64            input,
65            pos: 0,
66            state: State::Data,
67            eof_emitted: false,
68            unicode_ranges_allowed: false,
69        }
70    }
71
72    /// Peek at the code point at `offset` from the current position
73    /// without consuming it. Returns `None` if past end of input.
74    ///
75    /// Used by §4.3.8 / §4.3.9 / §4.3.10 / §4.3.11 "check if N code
76    /// points..." predicates, which must look ahead without advancing.
77    fn peek(&self, offset: usize) -> Option<char> {
78        self.input.get(self.pos + offset).copied()
79    }
80
81    /// Consume and return the next code point, advancing `pos`. Returns
82    /// `None` if at end of input.
83    ///
84    /// Implements the "consume the next input code point" primitive used
85    /// throughout §4.3.
86    fn consume(&mut self) -> Option<char> {
87        let c = self.input.get(self.pos).copied();
88        if c.is_some() {
89            self.pos += 1;
90        }
91        c
92    }
93
94    /// Re-consume the current code point: step `pos` back by one.
95    ///
96    /// Per §4.3.1 "reconsume the current input code point": the next
97    /// call to [`consume`](Self::consume) will return the same code point
98    /// that was just consumed. No-op if `pos == 0`.
99    fn reconsume(&mut self) {
100        if self.pos > 0 {
101            self.pos -= 1;
102        }
103    }
104
105    /// §4.3.1 Consume a token.
106    ///
107    /// This is the main entry point of the tokenizer. Dispatches based on
108    /// the current input code point to one of the sub-algorithms (§4.3.2
109    /// through §4.3.6) or emits a simple token directly. The optional
110    /// `unicode_ranges_allowed` flag (§4.3.1 L782-783, default `false`)
111    /// gates the `U+`/`u+` unicode-range branch.
112    fn consume_a_token(&mut self) -> Token {
113        // §4.3.2: If the next two input code points are U+002F SOLIDUS
114        // (`/`) and U+002A ASTERISK (`*`), consume comments and
115        // recursively consume the next token. We implement this as a
116        // loop: any leading run of `/* ... */` comments is skipped before
117        // producing the next real token.
118        loop {
119            let Some(c) = self.consume() else {
120                // §5.3: end of stream → <EOF-token>
121                self.eof_emitted = true;
122                self.state = State::Eof;
123                return Token::Eof;
124            };
125
126            // §4.3.2 comment dispatch: `/` followed by `*`.
127            if c == '/' && self.peek(0) == Some('*') {
128                self.consume(); // consume `*`
129                self.consume_comments_body();
130                // Loop back to consume the next token (which may itself
131                // be preceded by another comment).
132                continue;
133            }
134
135            return match c {
136                // §4.3.1: whitespace → <whitespace-token>
137                ' ' | '\t' | '\n' | '\r' | '\u{000C}' => {
138                    // Consume all consecutive whitespace.
139                    while let Some(next) = self.peek(0) {
140                        if matches!(next, ' ' | '\t' | '\n' | '\r' | '\u{000C}') {
141                            self.consume();
142                        } else {
143                            break;
144                        }
145                    }
146                    Token::Whitespace
147                }
148                // §4.3.1: `"` or `'` → consume a string token (§4.3.5)
149                '"' | '\'' => self.consume_a_string_token(c),
150                // §4.3.1: `#` → consume an ident-like token (§4.3.4) for hash
151                '#' => self.consume_a_hash_token(),
152                // §4.3.1: `(` → <(-token>
153                '(' => Token::OpenParen,
154                // §4.3.1: `)` → <)-token>
155                ')' => Token::CloseParen,
156                // §4.3.1: `+` or `-` → if would-start-a-number, consume a
157                // numeric token (§4.3.3); else <delim-token>.
158                // §4.3.1: `-` additionally: if next two are `-` and `>`
159                // (i.e. `-->`), consume them and return <CDC-token>.
160                // §4.3.1: `.` → if would-start-a-number, consume a numeric
161                // token; else <delim-token>.
162                '-' => {
163                    if self.starts_with_number(c) {
164                        self.reconsume();
165                        self.consume_a_numeric_token()
166                    } else if self.peek(0) == Some('-') && self.peek(1) == Some('>') {
167                        // §4.3.1: `-->` → <CDC-token>
168                        self.consume(); // consume second `-`
169                        self.consume(); // consume `>`
170                        Token::Cdc
171                    } else if self.would_start_ident_sequence_at(0) {
172                        // §4.3.1: `-` starting an ident sequence → reconsume
173                        // and consume an ident-like token.
174                        self.reconsume();
175                        self.consume_an_ident_like_token()
176                    } else {
177                        Token::Delim(c)
178                    }
179                }
180                '+' | '.' => {
181                    if self.starts_with_number(c) {
182                        self.reconsume();
183                        self.consume_a_numeric_token()
184                    } else {
185                        Token::Delim(c)
186                    }
187                }
188                // §4.3.1: `<` → if next three are `!`, `-`, `-` (i.e.
189                // `<!--`), consume them and return <CDO-token>. Otherwise
190                // emit <delim-token>.
191                '<' => {
192                    if self.peek(0) == Some('!')
193                        && self.peek(1) == Some('-')
194                        && self.peek(2) == Some('-')
195                    {
196                        // Consume `!`, `-`, `-`.
197                        self.consume();
198                        self.consume();
199                        self.consume();
200                        Token::Cdo
201                    } else {
202                        Token::Delim(c)
203                    }
204                }
205                // §4.3.1: `@` → if next would-start-an-ident-sequence,
206                // consume an ident-like token (§4.3.4) for at-keyword.
207                '@' => self.consume_an_at_keyword_token(),
208                // §4.3.1: `[` → <[-token>
209                '[' => Token::OpenBracket,
210                // §4.3.1: `\` → if would-start-an-escape (§4.3.8), reconsume
211                // and consume an ident-like token (§4.3.4); else parse error,
212                // <delim-token>.
213                '\\' => self.consume_ident_or_delim(),
214                // §4.3.1: `]` → <]-token>
215                ']' => Token::CloseBracket,
216                // §4.3.1: `{` → <{-token>
217                '{' => Token::OpenBrace,
218                // §4.3.1: `}` → <}-token>
219                '}' => Token::CloseBrace,
220                // §4.3.1: digit → reconsume and consume a numeric token (§4.3.3)
221                '0'..='9' => {
222                    self.reconsume();
223                    self.consume_a_numeric_token()
224                }
225                // §4.3.1: `U+` or `u+` → if would-start-a-unicode-range (§4.3.11),
226                // consume a unicode-range token (§4.3.14).
227                'u' | 'U' => self.consume_u_or_unicode_range(),
228                // §4.3.1: ident-start code point → reconsume and consume an
229                // ident-like token (§4.3.4)
230                _ if is_ident_start_code_point(c) => {
231                    self.reconsume();
232                    self.consume_an_ident_like_token()
233                }
234                // §4.3.1: `:` → <colon-token>
235                ':' => Token::Colon,
236                // §4.3.1: `;` → <semicolon-token>
237                ';' => Token::Semicolon,
238                // §4.3.1: `,` → <comma-token>
239                ',' => Token::Comma,
240                // §4.3.1: anything else → <delim-token>
241                _ => Token::Delim(c),
242            };
243        }
244    }
245
246    /// §4.3.2 Consume comments — body only.
247    ///
248    /// Precondition: the opening `/*` has already been consumed. Consume
249    /// code points until the closing `*/` or EOF. Per §4.3.2, an
250    /// unterminated comment (EOF before `*/`) is a parse error; we
251    /// silently consume to EOF in that case (no error reporting yet).
252    fn consume_comments_body(&mut self) {
253        loop {
254            match self.consume() {
255                None => return, // EOF in comment: parse error (unreported)
256                Some('*') => {
257                    if self.peek(0) == Some('/') {
258                        self.consume(); // consume `/`
259                        return;
260                    }
261                }
262                Some(_) => {}
263            }
264        }
265    }
266
267    // ── Sub-algorithm stubs (implemented in subsequent commits) ───────
268
269    /// §4.3.5 (L1081-1128) Consume a string token.
270    ///
271    /// `quote` is the ending code point (the opening quote `"` or `'`).
272    /// Returns a `string-token` (normal or EOF-terminated) or a
273    /// `bad-string-token` (unescaped newline).
274    ///
275    /// Per §4.3.5:
276    /// - ending code point (L1097-1099) → return string-token.
277    /// - EOF (L1101-1104) → parse error, return string-token.
278    /// - newline (L1106-1110) → parse error, reconsume, return bad-string.
279    /// - `\` (L1112-1124):
280    ///   - next EOF → do nothing (loop; next consume hits EOF → string).
281    ///   - next newline → consume it (line continuation).
282    ///   - otherwise (valid escape) → consume an escaped code point, append.
283    /// - anything else (L1126-1128) → append.
284    fn consume_a_string_token(&mut self, quote: char) -> Token {
285        let mut value = String::new();
286        loop {
287            match self.consume() {
288                None => {
289                    // §4.3.5 L1101-1104: EOF → parse error, return string-token.
290                    return Token::String(value);
291                }
292                Some(c) if c == quote => {
293                    // §4.3.5 L1097-1099: ending code point → return string-token.
294                    return Token::String(value);
295                }
296                Some('\n') => {
297                    // §4.3.5 L1106-1110: newline → parse error, reconsume, return bad-string.
298                    self.reconsume();
299                    return Token::BadString;
300                }
301                Some('\\') => {
302                    // §4.3.5 L1112-1124: escape handling.
303                    match self.peek(0) {
304                        None => {
305                            // §4.3.5 L1114-1115: next is EOF, do nothing.
306                            // Loop continues; next consume() returns None → String.
307                        }
308                        Some('\n') => {
309                            // §4.3.5 L1117-1119: next is newline, consume it (line continuation).
310                            self.consume();
311                        }
312                        Some(_) => {
313                            // §4.3.5 L1121-1124: valid escape, consume escaped code point.
314                            let escaped = self.consume_an_escaped_code_point();
315                            value.push(escaped);
316                        }
317                    }
318                }
319                Some(c) => {
320                    // §4.3.5 L1126-1128: anything else → append.
321                    value.push(c);
322                }
323            }
324        }
325    }
326
327    /// §4.3.1 (L801-826) `#` branch: Consume a hash token.
328    ///
329    /// Precondition: `#` has been consumed. Per §4.3.1 L801-826:
330    /// - If the next code point is an ident code point OR the next two
331    ///   are a valid escape, then create a `hash-token` (type "id" if
332    ///   would-start-ident-sequence, else "unrestricted"), consume an
333    ///   ident sequence for the value, and return it.
334    /// - Otherwise, return a `delim-token` with value `#`.
335    fn consume_a_hash_token(&mut self) -> Token {
336        // §4.3.1 L803-805: next is ident code point OR next two are valid escape.
337        let next_is_ident = self.peek(0).is_some_and(is_ident_code_point);
338        let next_is_valid_escape = self.is_valid_escape_at(0);
339        if !next_is_ident && !next_is_valid_escape {
340            // §4.3.1 L824-826: return delim-token with value `#`.
341            return Token::Delim('#');
342        }
343        let hash_type = if self.would_start_ident_sequence_at(0) {
344            HashType::Id
345        } else {
346            HashType::Unrestricted
347        };
348        let name = self.consume_an_ident_sequence();
349        Token::Hash(name, hash_type)
350    }
351
352    /// §4.3.3 (L1011-1042) Consume a numeric token.
353    ///
354    /// Returns a `number-token`, `percentage-token`, or `dimension-token`.
355    /// Per §4.3.3:
356    /// 1. Consume a number (§4.3.13).
357    /// 2. If next 3 would start an ident sequence → dimension-token (consume
358    ///    an ident sequence for the unit).
359    /// 3. Else if next is `%` → percentage-token (consume `%`).
360    /// 4. Else → number-token.
361    fn consume_a_numeric_token(&mut self) -> Token {
362        let (value, is_integer) = self.consume_a_number();
363        let numeric = Numeric { value, is_integer };
364        // §4.3.3 L1019-1029: dimension
365        if self.would_start_ident_sequence_at(0) {
366            let unit = self.consume_an_ident_sequence();
367            return Token::Dimension(numeric, unit);
368        }
369        // §4.3.3 L1032-1036: percentage
370        if self.peek(0) == Some('%') {
371            self.consume();
372            return Token::Percentage(numeric);
373        }
374        // §4.3.3 L1040-1042: number
375        Token::Number(numeric)
376    }
377
378    /// §4.3.13 (L1415-1483) Consume a number.
379    ///
380    /// Returns `(value, is_integer)` where `is_integer` is true when the
381    /// source representation had no fractional part and no exponent (type
382    /// "integer"), false when it had either (type "number"). The sign is
383    /// included in `value`.
384    ///
385    /// Per §4.3.13:
386    /// 1. type = "integer"; number_part = ""; exponent_part = "".
387    /// 2. If next is `+`/`-`, consume, append to number_part.
388    /// 3. While next is digit, consume, append to number_part.
389    /// 4. If next 2 are `.` + digit: consume `.`, consume digits; type = "number".
390    /// 5. If next 2/3 are `e`/`E` + optional `+`/`-` + digit: consume `e`/`E`,
391    ///    optional sign, digits into exponent_part; type = "number".
392    /// 6. value = parse number_part; if exponent_part non-empty, value *= 10^exp.
393    /// 7. Return value, type, sign.
394    fn consume_a_number(&mut self) -> (f64, bool) {
395        let mut type_is_number = false;
396        let mut number_part = String::new();
397        let mut exponent_part = String::new();
398
399        // §4.3.13 L1436-1440: sign
400        if self.peek(0) == Some('+') || self.peek(0) == Some('-') {
401            number_part.push(self.consume().unwrap());
402        }
403
404        // §4.3.13 L1442-1444: integer digits
405        while let Some(d) = self.peek(0) {
406            if is_digit(d) {
407                number_part.push(d);
408                self.consume();
409            } else {
410                break;
411            }
412        }
413
414        // §4.3.13 L1446-1454: fraction part
415        if self.peek(0) == Some('.') && matches!(self.peek(1), Some(d) if is_digit(d)) {
416            number_part.push(self.consume().unwrap()); // consume `.`
417            type_is_number = true;
418            while let Some(d) = self.peek(0) {
419                if is_digit(d) {
420                    number_part.push(d);
421                    self.consume();
422                } else {
423                    break;
424                }
425            }
426        }
427
428        // §4.3.13 L1457-1469: exponent part
429        if matches!(self.peek(0), Some('e') | Some('E')) {
430            let p1 = self.peek(1);
431            let p2 = self.peek(2);
432            let valid_exp = match p1 {
433                Some('+') | Some('-') => matches!(p2, Some(d) if is_digit(d)),
434                Some(d) if is_digit(d) => true,
435                _ => false,
436            };
437            if valid_exp {
438                self.consume(); // consume `e`/`E`
439                if self.peek(0) == Some('+') || self.peek(0) == Some('-') {
440                    exponent_part.push(self.consume().unwrap());
441                }
442                while let Some(d) = self.peek(0) {
443                    if is_digit(d) {
444                        exponent_part.push(d);
445                        self.consume();
446                    } else {
447                        break;
448                    }
449                }
450                type_is_number = true;
451            }
452        }
453
454        // §4.3.13 L1472-1480: compute value
455        let mut value: f64 = number_part.parse().unwrap_or(0.0);
456        if !exponent_part.is_empty() {
457            let exp: i32 = exponent_part.parse().unwrap_or(0);
458            value *= 10.0f64.powi(exp);
459        }
460
461        (value, !type_is_number)
462    }
463
464    /// §4.3.4 (L1045-1078) Consume an ident-like token.
465    ///
466    /// Precondition: the current position is at the start of an ident
467    /// sequence (the caller has not consumed anything; or has reconsumed
468    /// the first code point). Returns an `ident-token`, `function-token`,
469    /// `url-token`, or `bad-url-token`.
470    ///
471    /// Per §4.3.4:
472    /// 1. Consume an ident sequence (§4.3.12), yielding `name`.
473    /// 2. If `name` is ASCII-case-insensitive "url" and next is `(`:
474    ///    consume `(`, collapse leading whitespace pairs, then if the
475    ///    next 1-2 code points are a quote (or whitespace+quote) return
476    ///    a `function-token`; otherwise consume a url token (§4.3.6).
477    /// 3. Else if next is `(`: consume it, return `function-token`.
478    /// 4. Else return `ident-token`.
479    fn consume_an_ident_like_token(&mut self) -> Token {
480        let name = self.consume_an_ident_sequence();
481        // §4.3.4 L1053-1066: url( special case
482        if name.eq_ignore_ascii_case("url") && self.peek(0) == Some('(') {
483            self.consume(); // consume `(`
484                            // §4.3.4 L1056-1057: while next two are whitespace, consume one
485            while self.peek(0).is_some_and(is_whitespace) && self.peek(1).is_some_and(is_whitespace)
486            {
487                self.consume();
488            }
489            // §4.3.4 L1058-1063: if next 1-2 are " / ' / ws+" / ws+' → Function
490            let p0 = self.peek(0);
491            let p1 = self.peek(1);
492            let is_quote_case = matches!(p0, Some('"') | Some('\''))
493                || (p0.is_some_and(is_whitespace) && matches!(p1, Some('"') | Some('\'')));
494            if is_quote_case {
495                return Token::Function(name);
496            }
497            // §4.3.4 L1064-1066: otherwise consume a url token
498            return self.consume_a_url_token();
499        }
500        // §4.3.4 L1068-1073: ordinary function
501        if self.peek(0) == Some('(') {
502            self.consume();
503            return Token::Function(name);
504        }
505        // §4.3.4 L1075-1078: ident
506        Token::Ident(name)
507    }
508
509    /// §4.3.4 (at-keyword branch) Consume an at-keyword token.
510    ///
511    /// Precondition: `@` has been consumed. Per §4.3.1, if the next code
512    /// points would start an ident sequence, consume it and return an
513    /// `<at-keyword-token>`; otherwise return a `<delim-token>` with `@`.
514    fn consume_an_at_keyword_token(&mut self) -> Token {
515        if self.would_start_ident_sequence_at(0) {
516            let name = self.consume_an_ident_sequence();
517            Token::AtKeyword(name)
518        } else {
519            Token::Delim('@')
520        }
521    }
522
523    /// §4.3.1 (backslash branch): if would-start-an-escape, consume an
524    /// ident-like token; else emit `<delim-token>`.
525    fn consume_ident_or_delim(&mut self) -> Token {
526        if self.is_valid_escape_next() {
527            self.reconsume();
528            self.consume_an_ident_like_token()
529        } else {
530            Token::Delim('\\')
531        }
532    }
533
534    /// §4.3.6 (L1132-1203) Consume a url token.
535    ///
536    /// Precondition: `url(` has been consumed by the caller (§4.3.4
537    /// L1053-1066). This algorithm consumes the "unquoted" url body and
538    /// returns a `url-token` or `bad-url-token`. A quoted value like
539    /// `url("foo")` is handled by §4.3.4 as a `function-token` and never
540    /// reaches here.
541    ///
542    /// Per §4.3.6:
543    /// - `)` → return url-token.
544    /// - EOF → parse error, return url-token.
545    /// - whitespace → consume trailing whitespace; if next is `)` or EOF
546    ///   return url-token, else bad-url.
547    /// - `"` / `'` / `(` / non-printable → parse error, bad-url.
548    /// - `\` → if valid escape, consume escaped and append; else bad-url.
549    /// - anything else → append.
550    fn consume_a_url_token(&mut self) -> Token {
551        let mut value = String::new();
552        // §4.3.6 L1151: consume as much whitespace as possible
553        while self.peek(0).is_some_and(is_whitespace) {
554            self.consume();
555        }
556        loop {
557            match self.consume() {
558                // §4.3.6 L1157-1159: `)` → return url-token
559                Some(')') => return Token::Url(value),
560                // §4.3.6 L1161-1164: EOF → parse error, return url-token
561                None => return Token::Url(value),
562                // §4.3.6 L1166-1175: whitespace → consume trailing ws, then check
563                Some(c) if is_whitespace(c) => {
564                    while self.peek(0).is_some_and(is_whitespace) {
565                        self.consume();
566                    }
567                    match self.peek(0) {
568                        Some(')') => {
569                            self.consume();
570                            return Token::Url(value);
571                        }
572                        None => return Token::Url(value), // parse error
573                        _ => {
574                            self.consume_the_remnants_of_a_bad_url();
575                            return Token::BadUrl;
576                        }
577                    }
578                }
579                // §4.3.6 L1177-1185: " / ' / ( / non-printable → bad url
580                Some(c) if c == '"' || c == '\'' || c == '(' || is_non_printable(c) => {
581                    self.consume_the_remnants_of_a_bad_url();
582                    return Token::BadUrl;
583                }
584                // §4.3.6 L1187-1197: `\`
585                Some('\\') => {
586                    if self.is_valid_escape_next() {
587                        let escaped = self.consume_an_escaped_code_point();
588                        value.push(escaped);
589                    } else {
590                        self.consume_the_remnants_of_a_bad_url();
591                        return Token::BadUrl;
592                    }
593                }
594                // §4.3.6 L1199-1202: anything else → append
595                Some(c) => value.push(c),
596            }
597        }
598    }
599
600    /// §4.3.15 (L1551-1577) Consume the remnants of a bad url.
601    ///
602    /// Consumes code points until `)` or EOF, allowing an escaped `)`
603    /// (`\)`) to be consumed without ending the bad-url. Per §4.3.15:
604    /// - `)` or EOF → return.
605    /// - valid escape (stream starts with `\` + non-newline non-EOF) →
606    ///   consume an escaped code point.
607    /// - anything else → do nothing.
608    fn consume_the_remnants_of_a_bad_url(&mut self) {
609        loop {
610            match self.consume() {
611                // §4.3.15 L1563-1566: `)` or EOF → return
612                Some(')') | None => return,
613                // §4.3.15 L1568-1572: valid escape → consume escaped code point
614                Some('\\') if self.is_valid_escape_next() => {
615                    let _ = self.consume_an_escaped_code_point();
616                }
617                // §4.3.15 L1574-1576: anything else → do nothing
618                Some(_) => {}
619            }
620        }
621    }
622
623    /// §4.3.1 (L960-972) U/u branch: if `unicode_ranges_allowed` is true
624    /// and the stream would start a unicode-range (§4.3.11), consume a
625    /// unicode-range token (§4.3.14); otherwise consume an ident-like token.
626    ///
627    /// Precondition: `u`/`U` has been consumed, `pos` points just past it.
628    /// This method reconsumes back to `u` before dispatching.
629    fn consume_u_or_unicode_range(&mut self) -> Token {
630        self.reconsume(); // pos back to `u`/`U`
631                          // §4.3.1 L963-967: unicode_ranges_allowed && would-start-unicode-range
632        if self.unicode_ranges_allowed && self.would_start_unicode_range_at(0) {
633            return self.consume_a_unicode_range_token();
634        }
635        // §4.3.1 L969-972: otherwise ident-like
636        self.consume_an_ident_like_token()
637    }
638
639    /// §4.3.11 (L1356-1378) Check if three code points would start a
640    /// unicode-range, examining code points starting at `offset` from the
641    /// current position.
642    ///
643    /// Per §4.3.11: true iff first is `U`/`u`, second is `+`, third is `?`
644    /// or a hex digit.
645    fn would_start_unicode_range_at(&self, offset: usize) -> bool {
646        let first = match self.peek(offset) {
647            Some(c) => c,
648            None => return false,
649        };
650        if !matches!(first, 'U' | 'u') {
651            return false;
652        }
653        if self.peek(offset + 1) != Some('+') {
654            return false;
655        }
656        match self.peek(offset + 2) {
657            Some('?') => true,
658            Some(c) if is_hex_digit(c) => true,
659            _ => false,
660        }
661    }
662
663    /// §4.3.14 (L1487-1548) Consume a unicode-range token.
664    ///
665    /// Precondition: the stream starts with `u`/`U` + `+` + (`?` or hex
666    /// digit) (i.e. [`would_start_unicode_range_at`](Self::would_start_unicode_range_at)
667    /// returns true at the current position).
668    ///
669    /// Returns `UnicodeRange(Some(start), Some(end))`.
670    ///
671    /// Per §4.3.14:
672    /// 1. Consume and discard `u`/`U` and `+`.
673    /// 2. Consume up to 6 hex digits; if fewer than 6, consume `?` up to a
674    ///    total of 6 → `first_segment`.
675    /// 3. If `first_segment` contains `?`: start = `?`→`0`, end = `?`→`F`.
676    /// 4. Else start = hex value of `first_segment`.
677    /// 5. If next two are `-` + hex digit: consume `-`, consume up to 6 hex
678    ///    → end.
679    /// 6. Else end = start.
680    fn consume_a_unicode_range_token(&mut self) -> Token {
681        // §4.3.14 step 1: consume and discard `u`/`U` + `+`
682        self.consume(); // `u`/`U`
683        self.consume(); // `+`
684
685        // §4.3.14 step 2: consume up to 6 hex digits
686        let mut first_segment = String::new();
687        while first_segment.len() < 6 {
688            match self.peek(0) {
689                Some(c) if is_hex_digit(c) => {
690                    first_segment.push(c);
691                    self.consume();
692                }
693                _ => break,
694            }
695        }
696        // then consume `?` up to a total of 6
697        while first_segment.len() < 6 {
698            match self.peek(0) {
699                Some('?') => {
700                    first_segment.push('?');
701                    self.consume();
702                }
703                _ => break,
704            }
705        }
706
707        // §4.3.14 step 3: if first_segment contains `?`
708        if first_segment.contains('?') {
709            let start_str: String = first_segment
710                .chars()
711                .map(|c| if c == '?' { '0' } else { c })
712                .collect();
713            let start = u32::from_str_radix(&start_str, 16).unwrap_or(0);
714            let end_str: String = first_segment
715                .chars()
716                .map(|c| if c == '?' { 'F' } else { c })
717                .collect();
718            let end = u32::from_str_radix(&end_str, 16).unwrap_or(0);
719            return Token::UnicodeRange(Some(start), Some(end));
720        }
721
722        // §4.3.14 step 4: first_segment as hex → start
723        let start = u32::from_str_radix(&first_segment, 16).unwrap_or(0);
724
725        // §4.3.14 step 5: if next two are `-` + hex digit
726        if self.peek(0) == Some('-') && self.peek(1).is_some_and(is_hex_digit) {
727            self.consume(); // consume `-`
728            let mut end_segment = String::new();
729            while end_segment.len() < 6 {
730                match self.peek(0) {
731                    Some(c) if is_hex_digit(c) => {
732                        end_segment.push(c);
733                        self.consume();
734                    }
735                    _ => break,
736                }
737            }
738            let end = u32::from_str_radix(&end_segment, 16).unwrap_or(0);
739            return Token::UnicodeRange(Some(start), Some(end));
740        }
741
742        // §4.3.14 step 6: otherwise start == end
743        Token::UnicodeRange(Some(start), Some(start))
744    }
745
746    // ── §4.3.12 Consume an ident sequence ────────────────────────────
747
748    /// §4.3.12 Consume an ident sequence.
749    ///
750    /// Repeatedly consume code points as long as they form an ident
751    /// sequence: ident code points, valid escapes (consumed via
752    /// §4.3.7), or `\`-`-` (for `--`-prefixed names). Returns the
753    /// decoded name with escapes resolved.
754    ///
755    /// Per §4.3.12, the loop terminates when:
756    /// - EOF is reached, or
757    /// - the next code point is not an ident code point, not a valid
758    ///   escape, and not part of a `\`-`-` continuation.
759    fn consume_an_ident_sequence(&mut self) -> String {
760        let mut result = String::new();
761        loop {
762            let Some(c) = self.consume() else {
763                return result;
764            };
765            if is_ident_code_point(c) {
766                result.push(c);
767            } else if c == '\\' && self.is_valid_escape_next() {
768                // §4.3.12: valid escape → consume the escaped code point
769                // and append it.
770                let escaped = self.consume_an_escaped_code_point();
771                result.push(escaped);
772            } else {
773                // Not part of the ident sequence: reconsume and stop.
774                self.reconsume();
775                return result;
776            }
777        }
778    }
779
780    // ── §4.3.7 Consume an escaped code point ────────────────────────
781
782    /// §4.3.7 Consume an escaped code point.
783    ///
784    /// Precondition: the `\` has been consumed. Returns the decoded code
785    /// point.
786    ///
787    /// Per §4.3.7 (L1207-1240):
788    /// - EOF (L1233-1236) → parse error, return U+FFFD REPLACEMENT CHARACTER.
789    /// - newline → parse error, reconsume (invalid escape; the caller
790    ///   should have checked §4.3.8 first, but we handle defensively).
791    /// - hex digit(s) (L1219-1231) → consume 1-6 hex digits, optionally
792    ///   followed by whitespace, and return the code point with that
793    ///   value. U+0000 NULL or out-of-range → replacement character U+FFFD.
794    /// - anything else (L1238-1240) → return the consumed code point.
795    fn consume_an_escaped_code_point(&mut self) -> char {
796        let Some(c) = self.consume() else {
797            // §4.3.7 L1233-1236: EOF → parse error, return U+FFFD.
798            return '\u{FFFD}';
799        };
800        if is_hex_digit(c) {
801            // §4.3.7 step 4: consume up to 5 more hex digits (total ≤ 6).
802            let mut hex = String::new();
803            hex.push(c);
804            while hex.len() < 6 {
805                match self.peek(0) {
806                    Some(next) if is_hex_digit(next) => {
807                        hex.push(next);
808                        self.consume();
809                    }
810                    _ => break,
811                }
812            }
813            // §4.3.7 step 4: consume one trailing whitespace.
814            if self.peek(0) == Some(' ') || self.peek(0) == Some('\t') || self.peek(0) == Some('\n')
815            {
816                self.consume();
817            }
818            // Parse the hex value.
819            let value = u32::from_str_radix(&hex, 16).unwrap_or(0);
820            // §4.3.7 step 4: 0 or > 0x10FFFF → U+FFFD. Also surrogates
821            // (0xD800–0xDFFF) → U+FFFD.
822            if value == 0 || value > 0x10FFFF || (0xD800..=0xDFFF).contains(&value) {
823                '\u{FFFD}'
824            } else {
825                char::from_u32(value).unwrap_or('\u{FFFD}')
826            }
827        } else if c == '\n' {
828            // §4.3.7 step 3: newline after `\` is a parse error. This
829            // should not happen if the caller checked §4.3.8, but handle
830            // defensively by returning `\` and reconsuming the newline.
831            self.reconsume();
832            '\\'
833        } else {
834            // §4.3.7 step 5: any other code point → return it.
835            c
836        }
837    }
838
839    // ── §4.3.8 / §4.3.9 Predicates ──────────────────────────────────
840
841    /// §4.3.8 Check if two code points are a valid escape.
842    ///
843    /// Precondition: the `\` has been consumed. Returns true if the next
844    /// code point forms a valid escape with the consumed `\` (i.e. the
845    /// next code point is not a newline and not EOF).
846    fn is_valid_escape_next(&self) -> bool {
847        match self.peek(0) {
848            None => false,
849            Some('\n') => false,
850            Some(_) => true,
851        }
852    }
853
854    /// §4.3.9 Check if three code points would start an ident sequence,
855    /// examining code points starting at `offset` from the current
856    /// position.
857    ///
858    /// Per §4.3.9:
859    /// - U+002D HYPHEN-MINUS (`-`):
860    ///   - If the next is also `-`, true (start of `--...`).
861    ///   - If the next is an ident-start code point (or a valid escape),
862    ///     true.
863    ///   - Else false.
864    /// - U+005C REVERSE SOLIDUS (`\`): true if valid escape.
865    /// - ident-start code point: true.
866    /// - Else false.
867    fn would_start_ident_sequence_at(&self, offset: usize) -> bool {
868        let first = match self.peek(offset) {
869            Some(c) => c,
870            None => return false,
871        };
872        match first {
873            '-' => {
874                // §4.3.9: if next is `-`, true.
875                match self.peek(offset + 1) {
876                    Some('-') => true,
877                    Some(next) if is_ident_start_code_point(next) => true,
878                    Some('\\') => {
879                        // §4.3.9: if `\` and the following forms a valid escape.
880                        self.is_valid_escape_at(offset + 1)
881                    }
882                    _ => false,
883                }
884            }
885            '\\' => self.is_valid_escape_at(offset),
886            _ if is_ident_start_code_point(first) => true,
887            _ => false,
888        }
889    }
890
891    /// §4.3.8 Check if the code point at `offset` starts a valid escape
892    /// (i.e. `offset` points at `\` and `offset+1` is not newline/EOF).
893    fn is_valid_escape_at(&self, offset: usize) -> bool {
894        if self.peek(offset) != Some('\\') {
895            return false;
896        }
897        match self.peek(offset + 1) {
898            None => false,
899            Some('\n') => false,
900            Some(_) => true,
901        }
902    }
903
904    // ── Predicates (stubs returning false; implemented in C-4/C-6) ──
905
906    /// §4.3.10 (L1307-1352) Check if three code points would start a number.
907    ///
908    /// `first` is the code point already consumed (the caller passes it
909    /// so the predicate can examine the next two without reconsume). The
910    /// three code points examined are `first`, `peek(0)`, `peek(1)`.
911    ///
912    /// Per §4.3.10:
913    /// - `+`/`-`: second is digit → true; second is `.` and third is digit
914    ///   → true; else false.
915    /// - `.`: second is digit → true; else false.
916    /// - digit: true.
917    /// - anything else: false.
918    fn starts_with_number(&self, first: char) -> bool {
919        match first {
920            '+' | '-' => match self.peek(0) {
921                Some(d) if is_digit(d) => true,
922                Some('.') => matches!(self.peek(1), Some(d) if is_digit(d)),
923                _ => false,
924            },
925            '.' => matches!(self.peek(0), Some(d) if is_digit(d)),
926            d if is_digit(d) => true,
927            _ => false,
928        }
929    }
930}
931
932// ── Free functions (§4.2 code point classes) ──────────────────────────
933
934/// §4.2 Whether `c` is an ident-start code point.
935///
936/// An ident-start code point is an ASCII letter, a non-ASCII code point
937/// (U+0080 or higher), or U+005F LOW LINE (`_`).
938fn is_ident_start_code_point(c: char) -> bool {
939    matches!(c, 'A'..='Z' | 'a'..='z' | '_' | '\u{0080}'..=char::MAX)
940}
941
942/// §4.2 Whether `c` is an ident code point (allowed after the first).
943///
944/// An ident code point is an ident-start code point, or an ASCII digit,
945/// or U+002D HYPHEN-MINUS (`-`).
946fn is_ident_code_point(c: char) -> bool {
947    matches!(c, 'A'..='Z' | 'a'..='z' | '0'..='9' | '_' | '-' | '\u{0080}'..=char::MAX)
948}
949
950/// §4.2 Whether `c` is a digit (ASCII 0-9).
951fn is_digit(c: char) -> bool {
952    c.is_ascii_digit()
953}
954
955/// §4.2 Whether `c` is a hex digit (ASCII 0-9, A-F, a-f).
956fn is_hex_digit(c: char) -> bool {
957    c.is_ascii_hexdigit()
958}
959
960/// §4.2 Whether `c` is a whitespace code point.
961///
962/// Per §4.2: U+0009 TAB, U+000A LF, U+000C FF, U+000D CR, U+0020 SPACE.
963/// (After §5.3 preprocessing, U+000C FF and lone U+000D CR have been
964/// normalized to U+000A LF, but the helper follows the full §4.2
965/// definition for correctness when called on arbitrary input.)
966fn is_whitespace(c: char) -> bool {
967    matches!(c, '\t' | '\n' | '\u{000C}' | '\r' | ' ')
968}
969
970/// §4.2 Whether `c` is a non-printable code point.
971///
972/// Per §4.2: U+0000-U+0008, U+000B, U+000E-U+001F, U+007F-U+009F.
973fn is_non_printable(c: char) -> bool {
974    matches!(
975        c,
976        '\u{0000}'..='\u{0008}' | '\u{000B}' | '\u{000E}'..='\u{001F}' | '\u{007F}'..='\u{009F}'
977    )
978}
979
980/// §5.3 input preprocessing.
981///
982/// Per the "input stream" definition in §5.3:
983/// - U+000D CR followed by U+000A LF → single U+000A LF
984/// - lone U+000D CR → U+000A LF
985/// - U+000C FF → U+000A LF
986///
987/// The tokenizer operates on this normalized stream.
988fn preprocess_input(s: &str) -> Vec<char> {
989    let mut out = Vec::with_capacity(s.len());
990    let mut chars = s.chars().peekable();
991    while let Some(c) = chars.next() {
992        match c {
993            '\r' => {
994                if chars.peek() == Some(&'\n') {
995                    chars.next();
996                }
997                out.push('\n');
998            }
999            '\u{000C}' => out.push('\n'),
1000            _ => out.push(c),
1001        }
1002    }
1003    out
1004}
1005
1006impl Tokenizer for CssTokenizer {
1007    fn next_token(&mut self) -> Option<Token> {
1008        if self.eof_emitted {
1009            return None;
1010        }
1011        Some(self.consume_a_token())
1012    }
1013
1014    fn state(&self) -> State {
1015        self.state
1016    }
1017
1018    fn set_state(&mut self, state: State) {
1019        self.state = state;
1020        if state == State::Data {
1021            // Resetting to Data after EOF re-enables the iterator; useful
1022            // for `reset()` callers that want to re-tokenize.
1023            self.eof_emitted = false;
1024        }
1025    }
1026
1027    fn reset(&mut self) {
1028        self.pos = 0;
1029        self.state = State::Data;
1030        self.eof_emitted = false;
1031    }
1032
1033    fn set_unicode_ranges_allowed(&mut self, allowed: bool) {
1034        self.unicode_ranges_allowed = allowed;
1035    }
1036}
1037
1038// ── Test helpers exposed for unit tests in this file ──────────────────
1039
1040#[cfg(test)]
1041impl CssTokenizer {
1042    /// Tokenize the entire input and return all tokens except the
1043    /// trailing `<EOF-token>`. Test-only convenience.
1044    fn collect(input: &str) -> Vec<Token> {
1045        let mut tz = Self::new(input);
1046        let mut out = Vec::new();
1047        while let Some(t) = tz.next_token() {
1048            if matches!(t, Token::Eof) {
1049                break;
1050            }
1051            out.push(t);
1052        }
1053        out
1054    }
1055}
1056
1057#[cfg(test)]
1058mod tests {
1059    use super::*;
1060
1061    #[test]
1062    fn empty_input_emits_eof_only() {
1063        let mut tz = CssTokenizer::new("");
1064        assert!(matches!(tz.next_token(), Some(Token::Eof)));
1065        assert!(tz.next_token().is_none());
1066    }
1067
1068    #[test]
1069    fn whitespace_collapses_runs() {
1070        let tokens = CssTokenizer::collect("   \t\n  ");
1071        assert_eq!(tokens.len(), 1, "whitespace run → single token");
1072        assert!(matches!(tokens[0], Token::Whitespace));
1073    }
1074
1075    #[test]
1076    fn simple_punctuation_tokens() {
1077        let tokens = CssTokenizer::collect(":;,.{}[]()");
1078        assert_eq!(tokens.len(), 10);
1079        assert!(matches!(tokens[0], Token::Colon));
1080        assert!(matches!(tokens[1], Token::Semicolon));
1081        assert!(matches!(tokens[2], Token::Comma));
1082        assert!(matches!(tokens[3], Token::Delim('.')));
1083        assert!(matches!(tokens[4], Token::OpenBrace));
1084        assert!(matches!(tokens[5], Token::CloseBrace));
1085        assert!(matches!(tokens[6], Token::OpenBracket));
1086        assert!(matches!(tokens[7], Token::CloseBracket));
1087        assert!(matches!(tokens[8], Token::OpenParen));
1088        assert!(matches!(tokens[9], Token::CloseParen));
1089    }
1090
1091    #[test]
1092    fn close_paren_is_emitted() {
1093        let tokens = CssTokenizer::collect(")");
1094        assert_eq!(tokens.len(), 1);
1095        assert!(matches!(tokens[0], Token::CloseParen));
1096    }
1097
1098    #[test]
1099    fn unknown_delim_chars() {
1100        let tokens = CssTokenizer::collect("^~`>");
1101        assert_eq!(tokens.len(), 4);
1102        assert!(matches!(tokens[0], Token::Delim('^')));
1103        assert!(matches!(tokens[1], Token::Delim('~')));
1104        assert!(matches!(tokens[2], Token::Delim('`')));
1105        assert!(matches!(tokens[3], Token::Delim('>')));
1106    }
1107
1108    #[test]
1109    fn preprocessing_normalizes_cr_lf_ff() {
1110        // CR LF → single LF
1111        let tz = CssTokenizer::new("a\r\nb");
1112        assert_eq!(tz.input, vec!['a', '\n', 'b']);
1113        // Lone CR → LF
1114        let tz = CssTokenizer::new("a\rb");
1115        assert_eq!(tz.input, vec!['a', '\n', 'b']);
1116        // FF → LF
1117        let tz = CssTokenizer::new("a\u{000C}b");
1118        assert_eq!(tz.input, vec!['a', '\n', 'b']);
1119    }
1120
1121    #[test]
1122    fn eof_emitted_once() {
1123        // C-1: use a non-ident-start, non-digit char to avoid the
1124        // ident-like / numeric branches (which are stubbed with todo!()
1125        // until C-2 / C-4).
1126        let mut tz = CssTokenizer::new(",");
1127        let t1 = tz.next_token();
1128        let t2 = tz.next_token();
1129        let t3 = tz.next_token();
1130        assert!(matches!(t1, Some(Token::Comma)));
1131        assert!(matches!(t2, Some(Token::Eof)));
1132        assert!(t3.is_none());
1133    }
1134
1135    #[test]
1136    fn block_comment_is_skipped() {
1137        // §4.3.2: `/* ... */` is consumed and no token is emitted.
1138        let tokens = CssTokenizer::collect("/* hello */:/*x*/;");
1139        assert_eq!(tokens.len(), 2);
1140        assert!(matches!(tokens[0], Token::Colon));
1141        assert!(matches!(tokens[1], Token::Semicolon));
1142    }
1143
1144    #[test]
1145    fn unterminated_comment_consumes_to_eof() {
1146        // §4.3.2: EOF before `*/` is a parse error; we consume to EOF.
1147        let tokens = CssTokenizer::collect("/* unfinished");
1148        assert_eq!(tokens.len(), 0, "unterminated comment → no tokens");
1149    }
1150
1151    #[test]
1152    fn comment_between_tokens() {
1153        // C-2 not yet: `a`/`b` would be ident tokens (todo!()). Verify
1154        // comment is consumed without affecting surrounding simple tokens
1155        // by using only punctuation.
1156        let tokens = CssTokenizer::collect(":/*c*/;");
1157        assert_eq!(tokens.len(), 2);
1158        assert!(matches!(tokens[0], Token::Colon));
1159        assert!(matches!(tokens[1], Token::Semicolon));
1160    }
1161
1162    #[test]
1163    fn cdo_token_emitted() {
1164        // §4.3.1: `<!--` → <CDO-token>
1165        let tokens = CssTokenizer::collect("<!--");
1166        assert_eq!(tokens.len(), 1);
1167        assert!(matches!(tokens[0], Token::Cdo));
1168    }
1169
1170    #[test]
1171    fn cdc_token_emitted() {
1172        // §4.3.1: `-->` → <CDC-token>
1173        let tokens = CssTokenizer::collect("-->");
1174        assert_eq!(tokens.len(), 1);
1175        assert!(matches!(tokens[0], Token::Cdc));
1176    }
1177
1178    #[test]
1179    fn cdo_cdc_wrapping_stylesheet() {
1180        // Legacy CSS 1/2.1 stylesheet wrapping.
1181        // `<!-- : ; -->`
1182        // tokens: Cdo, WS, Colon, WS, Semicolon, WS, Cdc = 7
1183        let tokens = CssTokenizer::collect("<!-- : ; -->");
1184        assert_eq!(tokens.len(), 7);
1185        assert!(matches!(tokens[0], Token::Cdo));
1186        assert!(matches!(tokens[1], Token::Whitespace));
1187        assert!(matches!(tokens[2], Token::Colon));
1188        assert!(matches!(tokens[3], Token::Whitespace));
1189        assert!(matches!(tokens[4], Token::Semicolon));
1190        assert!(matches!(tokens[5], Token::Whitespace));
1191        assert!(matches!(tokens[6], Token::Cdc));
1192    }
1193
1194    #[test]
1195    fn less_than_alone_is_delim() {
1196        // §4.3.1: `<` not followed by `!-` → <delim-token>
1197        let tokens = CssTokenizer::collect("< ");
1198        assert_eq!(tokens.len(), 2);
1199        assert!(matches!(tokens[0], Token::Delim('<')));
1200        assert!(matches!(tokens[1], Token::Whitespace));
1201    }
1202
1203    #[test]
1204    fn hyphen_alone_is_delim() {
1205        // §4.3.1: `-` not followed by number/ident/`->` → <delim-token>
1206        let tokens = CssTokenizer::collect("- ");
1207        assert_eq!(tokens.len(), 2);
1208        assert!(matches!(tokens[0], Token::Delim('-')));
1209        assert!(matches!(tokens[1], Token::Whitespace));
1210    }
1211
1212    // ── C-2 tests: ident / function / at-keyword / hash ──────────────
1213
1214    #[test]
1215    fn simple_ident() {
1216        let tokens = CssTokenizer::collect("color");
1217        assert_eq!(tokens.len(), 1);
1218        assert!(matches!(&tokens[0], Token::Ident(s) if s == "color"));
1219    }
1220
1221    #[test]
1222    fn ident_with_hyphen_prefix() {
1223        let tokens = CssTokenizer::collect("-webkit-flex");
1224        assert_eq!(tokens.len(), 1);
1225        assert!(matches!(&tokens[0], Token::Ident(s) if s == "-webkit-flex"));
1226    }
1227
1228    #[test]
1229    fn ident_with_double_hyphen() {
1230        // §4.3.9: `--` starts an ident sequence (custom properties).
1231        let tokens = CssTokenizer::collect("--my-var");
1232        assert_eq!(tokens.len(), 1);
1233        assert!(matches!(&tokens[0], Token::Ident(s) if s == "--my-var"));
1234    }
1235
1236    #[test]
1237    fn ident_with_underscore() {
1238        let tokens = CssTokenizer::collect("_private");
1239        assert_eq!(tokens.len(), 1);
1240        assert!(matches!(&tokens[0], Token::Ident(s) if s == "_private"));
1241    }
1242
1243    #[test]
1244    fn ident_with_escape() {
1245        // §4.3.7: `\26` → '&' (U+0026)
1246        let tokens = CssTokenizer::collect("color\\26 B");
1247        assert_eq!(tokens.len(), 1);
1248        assert!(matches!(&tokens[0], Token::Ident(s) if s == "color&B"));
1249    }
1250
1251    #[test]
1252    fn function_token() {
1253        let tokens = CssTokenizer::collect("translate(");
1254        assert_eq!(tokens.len(), 1);
1255        assert!(matches!(&tokens[0], Token::Function(s) if s == "translate"));
1256    }
1257
1258    #[test]
1259    fn at_keyword_token() {
1260        let tokens = CssTokenizer::collect("@media");
1261        assert_eq!(tokens.len(), 1);
1262        assert!(matches!(&tokens[0], Token::AtKeyword(s) if s == "media"));
1263    }
1264
1265    #[test]
1266    fn at_keyword_with_hyphen() {
1267        let tokens = CssTokenizer::collect("@-webkit-keyframes");
1268        assert_eq!(tokens.len(), 1);
1269        assert!(matches!(&tokens[0], Token::AtKeyword(s) if s == "-webkit-keyframes"));
1270    }
1271
1272    #[test]
1273    fn at_sign_alone_is_delim() {
1274        // §4.3.1: `@` not followed by ident-start → <delim-token>
1275        let tokens = CssTokenizer::collect("@ ");
1276        assert_eq!(tokens.len(), 2);
1277        assert!(matches!(tokens[0], Token::Delim('@')));
1278        assert!(matches!(tokens[1], Token::Whitespace));
1279    }
1280
1281    #[test]
1282    fn hash_id_type() {
1283        // §4.3.1 / §4.3.4: `#main` → <hash-token> with type "id"
1284        let tokens = CssTokenizer::collect("#main");
1285        assert_eq!(tokens.len(), 1);
1286        match &tokens[0] {
1287            Token::Hash(s, t) => {
1288                assert_eq!(s, "main");
1289                assert_eq!(*t, crate::types::HashType::Id);
1290            }
1291            _ => panic!("expected Hash, got {:?}", tokens[0]),
1292        }
1293    }
1294
1295    #[test]
1296    fn hash_unrestricted_type() {
1297        // §4.3.1 L801-826 + §4.3.4: `#123` → next `1` is an ident code
1298        // point (digits are ident code points per §4.2), so a hash-token
1299        // is created. `123` does not would-start-an-ident-sequence (digit
1300        // is not ident-start), so type is "unrestricted". The value is
1301        // "123" (consume_an_ident_sequence consumes all ident code points).
1302        let tokens = CssTokenizer::collect("#123");
1303        assert_eq!(tokens.len(), 1);
1304        match &tokens[0] {
1305            Token::Hash(s, t) => {
1306                assert_eq!(s, "123");
1307                assert_eq!(*t, crate::types::HashType::Unrestricted);
1308            }
1309            _ => panic!("expected Hash, got {:?}", tokens[0]),
1310        }
1311    }
1312
1313    #[test]
1314    fn hash_hex_color_is_unrestricted() {
1315        // §4.3.4: `#ff00aa` → starts with `f` which IS ident-start, so
1316        // this is actually type "id" (the hex digits form a valid ident).
1317        let tokens = CssTokenizer::collect("#ff00aa");
1318        assert_eq!(tokens.len(), 1);
1319        match &tokens[0] {
1320            Token::Hash(s, t) => {
1321                assert_eq!(s, "ff00aa");
1322                assert_eq!(*t, crate::types::HashType::Id);
1323            }
1324            _ => panic!("expected Hash, got {:?}", tokens[0]),
1325        }
1326    }
1327
1328    #[test]
1329    fn backslash_escape_in_ident() {
1330        // §4.3.7: `\` followed by non-hex → literal char. Use `z` (not
1331        // `b`, which is a hex digit and would decode to U+000B).
1332        let tokens = CssTokenizer::collect("a\\z");
1333        assert_eq!(tokens.len(), 1);
1334        assert!(matches!(&tokens[0], Token::Ident(s) if s == "az"));
1335    }
1336
1337    #[test]
1338    fn backslash_alone_is_delim() {
1339        // §4.3.1: `\` not followed by valid escape (EOF or newline) →
1340        // <delim-token>
1341        let tokens = CssTokenizer::collect("\\");
1342        assert_eq!(tokens.len(), 1);
1343        assert!(matches!(tokens[0], Token::Delim('\\')));
1344    }
1345
1346    #[test]
1347    fn backslash_newline_is_delim() {
1348        // §4.3.8: `\` followed by newline is not a valid escape.
1349        let tokens = CssTokenizer::collect("\\\n");
1350        assert_eq!(tokens.len(), 2);
1351        assert!(matches!(tokens[0], Token::Delim('\\')));
1352        assert!(matches!(tokens[1], Token::Whitespace));
1353    }
1354
1355    #[test]
1356    fn declaration_with_ident_and_value() {
1357        let tokens = CssTokenizer::collect("color: red");
1358        assert_eq!(tokens.len(), 4);
1359        assert!(matches!(&tokens[0], Token::Ident(s) if s == "color"));
1360        assert!(matches!(tokens[1], Token::Colon));
1361        assert!(matches!(tokens[2], Token::Whitespace));
1362        assert!(matches!(&tokens[3], Token::Ident(s) if s == "red"));
1363    }
1364
1365    #[test]
1366    fn unicode_escape_in_ident() {
1367        // §4.3.7 L1223-1225: `\000026` → '&' (U+0026); after 6 hex digits,
1368        // if next is whitespace, consume it. So `\000026 B` → the space is
1369        // consumed as trailing whitespace, `B` joins the ident sequence →
1370        // single token Ident("&B").
1371        let tokens = CssTokenizer::collect("\\000026 B");
1372        assert_eq!(tokens.len(), 1);
1373        assert!(matches!(&tokens[0], Token::Ident(s) if s == "&B"));
1374    }
1375
1376    #[test]
1377    fn null_escape_becomes_replacement_char() {
1378        // §4.3.7: `\0` → U+FFFD (NULL is not allowed)
1379        let tokens = CssTokenizer::collect("\\0");
1380        assert_eq!(tokens.len(), 1);
1381        assert!(matches!(&tokens[0], Token::Ident(s) if s == "\u{FFFD}"));
1382    }
1383
1384    #[test]
1385    fn surrogate_escape_becomes_replacement_char() {
1386        // §4.3.7: `\D800` → U+FFFD (surrogates not allowed)
1387        let tokens = CssTokenizer::collect("\\D800");
1388        assert_eq!(tokens.len(), 1);
1389        assert!(matches!(&tokens[0], Token::Ident(s) if s == "\u{FFFD}"));
1390    }
1391
1392    // ── C-2 Bug 1 回归测试: # Delim 回退 (§4.3.1 L824-826) ────────────
1393
1394    #[test]
1395    fn hash_alone_is_delim() {
1396        // §4.3.1 L824-826: `#` followed by EOF → next is not ident code
1397        // point, not valid escape → return delim-token with value `#`.
1398        let tokens = CssTokenizer::collect("#");
1399        assert_eq!(tokens.len(), 1);
1400        assert!(matches!(tokens[0], Token::Delim('#')));
1401    }
1402
1403    #[test]
1404    fn hash_followed_by_space_is_delim() {
1405        // §4.3.1 L824-826: `#` followed by space (not ident code point,
1406        // not valid escape) → delim-token with value `#`.
1407        let tokens = CssTokenizer::collect("# ");
1408        assert_eq!(tokens.len(), 2);
1409        assert!(matches!(tokens[0], Token::Delim('#')));
1410        assert!(matches!(tokens[1], Token::Whitespace));
1411    }
1412
1413    #[test]
1414    fn hash_followed_by_at_is_delim() {
1415        // §4.3.1 L824-826: `#` followed by `@` (not ident code point,
1416        // not valid escape) → delim-token with value `#`.
1417        let tokens = CssTokenizer::collect("#@");
1418        assert_eq!(tokens.len(), 2);
1419        assert!(matches!(tokens[0], Token::Delim('#')));
1420        assert!(matches!(tokens[1], Token::Delim('@')));
1421    }
1422
1423    // ── C-3 tests: §4.3.5 string + bad-string ────────────────────────
1424
1425    #[test]
1426    fn string_double_quoted() {
1427        // §4.3.5: "hello" → String("hello")
1428        let tokens = CssTokenizer::collect("\"hello\"");
1429        assert_eq!(tokens.len(), 1);
1430        assert!(matches!(&tokens[0], Token::String(s) if s == "hello"));
1431    }
1432
1433    #[test]
1434    fn string_single_quoted() {
1435        // §4.3.5: 'hello' → String("hello")
1436        let tokens = CssTokenizer::collect("'hello'");
1437        assert_eq!(tokens.len(), 1);
1438        assert!(matches!(&tokens[0], Token::String(s) if s == "hello"));
1439    }
1440
1441    #[test]
1442    fn string_with_escape() {
1443        // §4.3.5 + §4.3.7: "a\z" → String("az") (z is non-hex → literal)
1444        let tokens = CssTokenizer::collect("\"a\\z\"");
1445        assert_eq!(tokens.len(), 1);
1446        assert!(matches!(&tokens[0], Token::String(s) if s == "az"));
1447    }
1448
1449    #[test]
1450    fn string_unterminated_eof() {
1451        // §4.3.5 L1101-1104: EOF before closing quote → parse error,
1452        // return string-token with value accumulated so far.
1453        let tokens = CssTokenizer::collect("\"hello");
1454        assert_eq!(tokens.len(), 1);
1455        assert!(matches!(&tokens[0], Token::String(s) if s == "hello"));
1456    }
1457
1458    #[test]
1459    fn string_unescaped_newline_is_bad_string() {
1460        // §4.3.5 L1106-1110: unescaped newline → parse error, reconsume,
1461        // return bad-string. The reconsumed newline is then tokenized as
1462        // whitespace by the next consume_a_token call.
1463        let tokens = CssTokenizer::collect("\"\n");
1464        assert_eq!(tokens.len(), 2);
1465        assert!(matches!(tokens[0], Token::BadString));
1466        assert!(matches!(tokens[1], Token::Whitespace));
1467    }
1468
1469    #[test]
1470    fn string_line_continuation() {
1471        // §4.3.5 L1117-1119: backslash followed by newline → line
1472        // continuation (newline consumed, no char appended).
1473        let tokens = CssTokenizer::collect("\"a\\\nb\"");
1474        assert_eq!(tokens.len(), 1);
1475        assert!(matches!(&tokens[0], Token::String(s) if s == "ab"));
1476    }
1477
1478    #[test]
1479    fn string_hex_escape() {
1480        // §4.3.5 + §4.3.7: "\26" → String("&") (U+0026)
1481        let tokens = CssTokenizer::collect("\"\\26\"");
1482        assert_eq!(tokens.len(), 1);
1483        assert!(matches!(&tokens[0], Token::String(s) if s == "&"));
1484    }
1485
1486    // ── C-4 tests: §4.3.3/§4.3.10/§4.3.13 number/percentage/dimension ─
1487
1488    #[test]
1489    fn number_integer() {
1490        let tokens = CssTokenizer::collect("42");
1491        assert_eq!(tokens.len(), 1);
1492        assert_eq!(
1493            tokens[0],
1494            Token::Number(Numeric {
1495                value: 42.0,
1496                is_integer: true
1497            })
1498        );
1499    }
1500
1501    #[test]
1502    fn number_decimal() {
1503        let tokens = CssTokenizer::collect("3.5");
1504        assert_eq!(tokens.len(), 1);
1505        assert_eq!(
1506            tokens[0],
1507            Token::Number(Numeric {
1508                value: 3.5,
1509                is_integer: false
1510            })
1511        );
1512    }
1513
1514    #[test]
1515    fn number_signed() {
1516        let tokens = CssTokenizer::collect("-5");
1517        assert_eq!(tokens.len(), 1);
1518        assert_eq!(
1519            tokens[0],
1520            Token::Number(Numeric {
1521                value: -5.0,
1522                is_integer: true
1523            })
1524        );
1525    }
1526
1527    #[test]
1528    fn number_exponent() {
1529        // §4.3.13: 1e3 → 1.0 * 10^3 = 1000.0, type "number"
1530        let tokens = CssTokenizer::collect("1e3");
1531        assert_eq!(tokens.len(), 1);
1532        assert_eq!(
1533            tokens[0],
1534            Token::Number(Numeric {
1535                value: 1000.0,
1536                is_integer: false
1537            })
1538        );
1539    }
1540
1541    #[test]
1542    fn number_decimal_exponent() {
1543        // §4.3.13: 1.5e2 → 1.5 * 10^2 = 150.0, type "number"
1544        let tokens = CssTokenizer::collect("1.5e2");
1545        assert_eq!(tokens.len(), 1);
1546        assert_eq!(
1547            tokens[0],
1548            Token::Number(Numeric {
1549                value: 150.0,
1550                is_integer: false
1551            })
1552        );
1553    }
1554
1555    #[test]
1556    fn percentage_token() {
1557        let tokens = CssTokenizer::collect("50%");
1558        assert_eq!(tokens.len(), 1);
1559        assert_eq!(
1560            tokens[0],
1561            Token::Percentage(Numeric {
1562                value: 50.0,
1563                is_integer: true
1564            })
1565        );
1566    }
1567
1568    #[test]
1569    fn dimension_px() {
1570        let tokens = CssTokenizer::collect("10px");
1571        assert_eq!(tokens.len(), 1);
1572        assert_eq!(
1573            tokens[0],
1574            Token::Dimension(
1575                Numeric {
1576                    value: 10.0,
1577                    is_integer: true
1578                },
1579                "px".to_string(),
1580            )
1581        );
1582    }
1583
1584    #[test]
1585    fn dimension_em() {
1586        let tokens = CssTokenizer::collect("1.5em");
1587        assert_eq!(tokens.len(), 1);
1588        assert_eq!(
1589            tokens[0],
1590            Token::Dimension(
1591                Numeric {
1592                    value: 1.5,
1593                    is_integer: false
1594                },
1595                "em".to_string(),
1596            )
1597        );
1598    }
1599
1600    #[test]
1601    fn dimension_signed() {
1602        let tokens = CssTokenizer::collect("-30deg");
1603        assert_eq!(tokens.len(), 1);
1604        assert_eq!(
1605            tokens[0],
1606            Token::Dimension(
1607                Numeric {
1608                    value: -30.0,
1609                    is_integer: true
1610                },
1611                "deg".to_string(),
1612            )
1613        );
1614    }
1615
1616    #[test]
1617    fn plus_sign_number() {
1618        let tokens = CssTokenizer::collect("+5");
1619        assert_eq!(tokens.len(), 1);
1620        assert_eq!(
1621            tokens[0],
1622            Token::Number(Numeric {
1623                value: 5.0,
1624                is_integer: true
1625            })
1626        );
1627    }
1628
1629    #[test]
1630    fn dot_starts_number() {
1631        let tokens = CssTokenizer::collect(".5");
1632        assert_eq!(tokens.len(), 1);
1633        assert_eq!(
1634            tokens[0],
1635            Token::Number(Numeric {
1636                value: 0.5,
1637                is_integer: false
1638            })
1639        );
1640    }
1641
1642    // ── C-5 tests: §4.3.4 url( special case + §4.3.6 Url + §4.3.15 BadUrl ─
1643
1644    #[test]
1645    fn url_unquoted_simple() {
1646        // §4.3.6: url(foo) → Url("foo")
1647        let tokens = CssTokenizer::collect("url(foo)");
1648        assert_eq!(tokens.len(), 1);
1649        assert_eq!(tokens[0], Token::Url("foo".to_string()));
1650    }
1651
1652    #[test]
1653    fn url_unquoted_with_spaces() {
1654        // §4.3.6: leading/trailing whitespace consumed; url( foo ) → Url("foo")
1655        let tokens = CssTokenizer::collect("url( foo )");
1656        assert_eq!(tokens.len(), 1);
1657        assert_eq!(tokens[0], Token::Url("foo".to_string()));
1658    }
1659
1660    #[test]
1661    fn url_empty() {
1662        // §4.3.6: url() → Url("")
1663        let tokens = CssTokenizer::collect("url()");
1664        assert_eq!(tokens.len(), 1);
1665        assert_eq!(tokens[0], Token::Url(String::new()));
1666    }
1667
1668    #[test]
1669    fn url_eof_unterminated() {
1670        // §4.3.6 L1161-1164: EOF → parse error, return url-token with value so far
1671        let tokens = CssTokenizer::collect("url(foo");
1672        assert_eq!(tokens.len(), 1);
1673        assert_eq!(tokens[0], Token::Url("foo".to_string()));
1674    }
1675
1676    #[test]
1677    fn url_quoted_is_function() {
1678        // §4.3.4 L1058-1063: url("foo") → Function("url"); the body
1679        // `"foo")` remains in the stream → String("foo") + CloseParen.
1680        let tokens = CssTokenizer::collect("url(\"foo\")");
1681        assert_eq!(tokens.len(), 3);
1682        assert_eq!(tokens[0], Token::Function("url".to_string()));
1683        assert_eq!(tokens[1], Token::String("foo".to_string()));
1684        assert_eq!(tokens[2], Token::CloseParen);
1685    }
1686
1687    #[test]
1688    fn url_single_quoted_is_function() {
1689        // §4.3.4 L1058-1063: url('foo') → Function("url") + String + CloseParen
1690        let tokens = CssTokenizer::collect("url('foo')");
1691        assert_eq!(tokens.len(), 3);
1692        assert_eq!(tokens[0], Token::Function("url".to_string()));
1693        assert_eq!(tokens[1], Token::String("foo".to_string()));
1694        assert_eq!(tokens[2], Token::CloseParen);
1695    }
1696
1697    #[test]
1698    fn url_ws_then_quote_is_function() {
1699        // §4.3.4 L1058-1063: url( "foo") → whitespace+quote → Function.
1700        // L1056-1057 only collapses when *two* consecutive whitespace
1701        // follow `(`; here only one space precedes `"`, so it is NOT
1702        // consumed and becomes a Whitespace token.
1703        let tokens = CssTokenizer::collect("url( \"foo\")");
1704        assert_eq!(tokens.len(), 4);
1705        assert_eq!(tokens[0], Token::Function("url".to_string()));
1706        assert_eq!(tokens[1], Token::Whitespace);
1707        assert_eq!(tokens[2], Token::String("foo".to_string()));
1708        assert_eq!(tokens[3], Token::CloseParen);
1709    }
1710
1711    #[test]
1712    fn url_with_escape() {
1713        // §4.3.6 + §4.3.7: url(foo\29 bar) → \29 consumes trailing space,
1714        // decodes to `)`, then `bar` appended → Url("foo)bar")
1715        let tokens = CssTokenizer::collect("url(foo\\29 bar)");
1716        assert_eq!(tokens.len(), 1);
1717        assert_eq!(tokens[0], Token::Url("foo)bar".to_string()));
1718    }
1719
1720    #[test]
1721    fn url_bad_paren_in_unquoted() {
1722        // §4.3.6 L1177-1185: `(` in unquoted url → BadUrl
1723        let tokens = CssTokenizer::collect("url(foo(bar)");
1724        assert_eq!(tokens.len(), 1);
1725        assert_eq!(tokens[0], Token::BadUrl);
1726    }
1727
1728    #[test]
1729    fn url_bad_quote_in_unquoted() {
1730        // §4.3.6 L1177-1185: `"` in unquoted url → BadUrl
1731        let tokens = CssTokenizer::collect("url(foo\"bar)");
1732        assert_eq!(tokens.len(), 1);
1733        assert_eq!(tokens[0], Token::BadUrl);
1734    }
1735
1736    // ── C-6 tests: §4.3.1 unicode_ranges_allowed + §4.3.11 + §4.3.14 ──
1737
1738    #[test]
1739    fn unicode_range_disabled_by_default() {
1740        // §4.3.1 L782-783: default unicode_ranges_allowed=false.
1741        // `U+1234` → `U` ident-start → Ident("U") (`+` stops ident seq);
1742        // then `+1234` → starts_with_number (`+`+digit) → Number(1234).
1743        let tokens = CssTokenizer::collect("U+1234");
1744        assert_eq!(tokens.len(), 2);
1745        assert_eq!(tokens[0], Token::Ident("U".to_string()));
1746        assert_eq!(
1747            tokens[1],
1748            Token::Number(Numeric {
1749                value: 1234.0,
1750                is_integer: true
1751            })
1752        );
1753    }
1754
1755    #[test]
1756    fn unicode_range_simple() {
1757        // §4.3.14: U+1234 → UnicodeRange(0x1234, 0x1234)
1758        let mut tz = CssTokenizer::new("U+1234");
1759        tz.set_unicode_ranges_allowed(true);
1760        let t = tz.next_token().unwrap();
1761        assert_eq!(t, Token::UnicodeRange(Some(0x1234), Some(0x1234)));
1762    }
1763
1764    #[test]
1765    fn unicode_range_question_marks() {
1766        // §4.3.14 step 3: U+12?? → start=0x1200 (?→0), end=0x12FF (?→F)
1767        let mut tz = CssTokenizer::new("U+12??");
1768        tz.set_unicode_ranges_allowed(true);
1769        let t = tz.next_token().unwrap();
1770        assert_eq!(t, Token::UnicodeRange(Some(0x1200), Some(0x12FF)));
1771    }
1772
1773    #[test]
1774    fn unicode_range_range() {
1775        // §4.3.14 step 5: U+12-34FF → start=0x12, end=0x34FF
1776        let mut tz = CssTokenizer::new("U+12-34FF");
1777        tz.set_unicode_ranges_allowed(true);
1778        let t = tz.next_token().unwrap();
1779        assert_eq!(t, Token::UnicodeRange(Some(0x12), Some(0x34FF)));
1780    }
1781
1782    #[test]
1783    fn unicode_range_max_hex() {
1784        // §4.3.14: U+10FFFF → UnicodeRange(0x10FFFF, 0x10FFFF)
1785        let mut tz = CssTokenizer::new("U+10FFFF");
1786        tz.set_unicode_ranges_allowed(true);
1787        let t = tz.next_token().unwrap();
1788        assert_eq!(t, Token::UnicodeRange(Some(0x10FFFF), Some(0x10FFFF)));
1789    }
1790
1791    #[test]
1792    fn unicode_range_lowercase_u() {
1793        // §4.3.11: lowercase u also triggers; u+abc → UnicodeRange(0xABC, 0xABC)
1794        let mut tz = CssTokenizer::new("u+abc");
1795        tz.set_unicode_ranges_allowed(true);
1796        let t = tz.next_token().unwrap();
1797        assert_eq!(t, Token::UnicodeRange(Some(0xABC), Some(0xABC)));
1798    }
1799}