Skip to main content

muskitty_css_tokenizer/
impls.rs

1//! CSS Syntax tokenizer implementation.
2//!
3//! Implements the "Tokenizer Algorithms" of CSS Syntax Module Level 3 §4.3.
4//! The tokenizer is recursive-descent: [`CssTokenizer::next_token`] calls
5//! [`consume_a_token`](CssTokenizer::consume_a_token) (§4.3.1), which
6//! dispatches to sub-algorithms based on the current input code point.
7//!
8//! # Coverage
9//!
10//! All §4.3 sub-algorithms are implemented:
11//! - §4.3.1 Consume a token (full dispatch, incl. `unicode_ranges_allowed`)
12//! - §4.3.2 Consume comments
13//! - §4.3.3 Consume a numeric token
14//! - §4.3.4 Consume an ident-like token (incl. `url(` special case)
15//! - §4.3.5 Consume a string token
16//! - §4.3.6 Consume a url token
17//! - §4.3.7 Consume an escaped code point
18//! - §4.3.8 Check if two code points are a valid escape
19//! - §4.3.9 Check if three code points would start an ident sequence
20//! - §4.3.10 Check if three code points would start a number
21//! - §4.3.11 Check if three code points would start a unicode-range
22//! - §4.3.12 Consume an ident sequence
23//! - §4.3.13 Consume a number
24//! - §4.3.14 Consume a unicode-range token
25//! - §4.3.15 Consume the remnants of a bad url
26
27use crate::trait_def::Tokenizer;
28use crate::types::{HashType, Numeric, State, Token};
29
30/// The CSS tokenizer (§4.3).
31///
32/// Holds the preprocessed input stream (§5.3 normalization applied at
33/// construction), the current position, and EOF tracking. Sub-algorithms
34/// are implemented as methods on this struct so they share the input
35/// stream and position.
36pub struct CssTokenizer {
37    /// Input code points after §5.3 preprocessing (CR/LF/FF normalized to LF).
38    input: Vec<char>,
39    /// Current position in `input` (0-based index of the next code point
40    /// to consume). `pos == input.len()` means the stream is exhausted.
41    pos: usize,
42    /// Top-level state (§4.3.1 dispatch vs. EOF).
43    state: State,
44    /// Whether `<EOF-token>` has been emitted. After this is `true`,
45    /// [`next_token`](Tokenizer::next_token) returns `None`.
46    eof_emitted: bool,
47    /// §4.3.1 L782-783: `unicode_ranges_allowed` flag, default `false`.
48    /// When `true`, the `U+`/`u+` branch of §4.3.1 produces a
49    /// `<unicode-range-token>` (§4.3.14); otherwise `U`/`u` is tokenized
50    /// as an ident-like token.
51    unicode_ranges_allowed: bool,
52}
53
54impl CssTokenizer {
55    /// Construct a new tokenizer over `input`.
56    ///
57    /// Applies §5.3 input preprocessing: per the "input stream"
58    /// definition, U+000D CR followed by U+000A LF is collapsed to a
59    /// single LF, lone CR is replaced with LF, and U+000C FF is replaced
60    /// with LF. The tokenizer operates on this normalized stream.
61    pub fn new(input: &str) -> Self {
62        let input = preprocess_input(input);
63        Self {
64            input,
65            pos: 0,
66            state: State::Data,
67            eof_emitted: false,
68            unicode_ranges_allowed: false,
69        }
70    }
71
72    /// Peek at the code point at `offset` from the current position
73    /// without consuming it. Returns `None` if past end of input.
74    ///
75    /// Used by §4.3.8 / §4.3.9 / §4.3.10 / §4.3.11 "check if N code
76    /// points..." predicates, which must look ahead without advancing.
77    fn peek(&self, offset: usize) -> Option<char> {
78        self.input.get(self.pos + offset).copied()
79    }
80
81    /// Consume and return the next code point, advancing `pos`. Returns
82    /// `None` if at end of input.
83    ///
84    /// Implements the "consume the next input code point" primitive used
85    /// throughout §4.3.
86    fn consume(&mut self) -> Option<char> {
87        let c = self.input.get(self.pos).copied();
88        if c.is_some() {
89            self.pos += 1;
90        }
91        c
92    }
93
94    /// Re-consume the current code point: step `pos` back by one.
95    ///
96    /// Per §4.3.1 "reconsume the current input code point": the next
97    /// call to [`consume`](Self::consume) will return the same code point
98    /// that was just consumed. No-op if `pos == 0`.
99    fn reconsume(&mut self) {
100        if self.pos > 0 {
101            self.pos -= 1;
102        }
103    }
104
105    /// §4.3.1 Consume a token.
106    ///
107    /// This is the main entry point of the tokenizer. Dispatches based on
108    /// the current input code point to one of the sub-algorithms (§4.3.2
109    /// through §4.3.6) or emits a simple token directly. The optional
110    /// `unicode_ranges_allowed` flag (§4.3.1 L782-783, default `false`)
111    /// gates the `U+`/`u+` unicode-range branch.
112    fn consume_a_token(&mut self) -> Token {
113        // §4.3.2: If the next two input code points are U+002F SOLIDUS
114        // (`/`) and U+002A ASTERISK (`*`), consume comments and
115        // recursively consume the next token. We implement this as a
116        // loop: any leading run of `/* ... */` comments is skipped before
117        // producing the next real token.
118        loop {
119            let Some(c) = self.consume() else {
120                // §5.3: end of stream → <EOF-token>
121                self.eof_emitted = true;
122                self.state = State::Eof;
123                return Token::Eof;
124            };
125
126            // §4.3.2 comment dispatch: `/` followed by `*`.
127            if c == '/' && self.peek(0) == Some('*') {
128                self.consume(); // consume `*`
129                self.consume_comments_body();
130                // Loop back to consume the next token (which may itself
131                // be preceded by another comment).
132                continue;
133            }
134
135            return match c {
136                // §4.3.1: whitespace → <whitespace-token>
137                ' ' | '\t' | '\n' | '\r' | '\u{000C}' => {
138                    // Consume all consecutive whitespace.
139                    while let Some(next) = self.peek(0) {
140                        if matches!(next, ' ' | '\t' | '\n' | '\r' | '\u{000C}') {
141                            self.consume();
142                        } else {
143                            break;
144                        }
145                    }
146                    Token::Whitespace
147                }
148                // §4.3.1: `"` or `'` → consume a string token (§4.3.5)
149                '"' | '\'' => self.consume_a_string_token(c),
150                // §4.3.1: `#` → consume an ident-like token (§4.3.4) for hash
151                '#' => self.consume_a_hash_token(),
152                // §4.3.1: `(` → <(-token>
153                '(' => Token::OpenParen,
154                // §4.3.1: `)` → <)-token>
155                ')' => Token::CloseParen,
156                // §4.3.1: `+` or `-` → if would-start-a-number, consume a
157                // numeric token (§4.3.3); else <delim-token>.
158                // §4.3.1: `-` additionally: if next two are `-` and `>`
159                // (i.e. `-->`), consume them and return <CDC-token>.
160                // §4.3.1: `.` → if would-start-a-number, consume a numeric
161                // token; else <delim-token>.
162                '-' => {
163                    if self.starts_with_number(c) {
164                        self.reconsume();
165                        self.consume_a_numeric_token()
166                    } else if self.peek(0) == Some('-') && self.peek(1) == Some('>') {
167                        // §4.3.1: `-->` → <CDC-token>
168                        self.consume(); // consume second `-`
169                        self.consume(); // consume `>`
170                        Token::Cdc
171                    } else if self.would_start_ident_sequence_at(0) {
172                        // §4.3.1: `-` starting an ident sequence → reconsume
173                        // and consume an ident-like token.
174                        self.reconsume();
175                        self.consume_an_ident_like_token()
176                    } else {
177                        Token::Delim(c)
178                    }
179                }
180                '+' | '.' => {
181                    if self.starts_with_number(c) {
182                        self.reconsume();
183                        self.consume_a_numeric_token()
184                    } else {
185                        Token::Delim(c)
186                    }
187                }
188                // §4.3.1: `<` → if next three are `!`, `-`, `-` (i.e.
189                // `<!--`), consume them and return <CDO-token>. Otherwise
190                // emit <delim-token>.
191                '<' => {
192                    if self.peek(0) == Some('!')
193                        && self.peek(1) == Some('-')
194                        && self.peek(2) == Some('-')
195                    {
196                        // Consume `!`, `-`, `-`.
197                        self.consume();
198                        self.consume();
199                        self.consume();
200                        Token::Cdo
201                    } else {
202                        Token::Delim(c)
203                    }
204                }
205                // §4.3.1: `@` → if next would-start-an-ident-sequence,
206                // consume an ident-like token (§4.3.4) for at-keyword.
207                '@' => self.consume_an_at_keyword_token(),
208                // §4.3.1: `[` → <[-token>
209                '[' => Token::OpenBracket,
210                // §4.3.1: `\` → if would-start-an-escape (§4.3.8), reconsume
211                // and consume an ident-like token (§4.3.4); else parse error,
212                // <delim-token>.
213                '\\' => self.consume_ident_or_delim(),
214                // §4.3.1: `]` → <]-token>
215                ']' => Token::CloseBracket,
216                // §4.3.1: `{` → <{-token>
217                '{' => Token::OpenBrace,
218                // §4.3.1: `}` → <}-token>
219                '}' => Token::CloseBrace,
220                // §4.3.1: digit → reconsume and consume a numeric token (§4.3.3)
221                '0'..='9' => {
222                    self.reconsume();
223                    self.consume_a_numeric_token()
224                }
225                // §4.3.1: `U+` or `u+` → if would-start-a-unicode-range (§4.3.11),
226                // consume a unicode-range token (§4.3.14).
227                'u' | 'U' => self.consume_u_or_unicode_range(),
228                // §4.3.1: ident-start code point → reconsume and consume an
229                // ident-like token (§4.3.4)
230                _ if is_ident_start_code_point(c) => {
231                    self.reconsume();
232                    self.consume_an_ident_like_token()
233                }
234                // §4.3.1: `:` → <colon-token>
235                ':' => Token::Colon,
236                // §4.3.1: `;` → <semicolon-token>
237                ';' => Token::Semicolon,
238                // §4.3.1: `,` → <comma-token>
239                ',' => Token::Comma,
240                // §4.3.1: anything else → <delim-token>
241                _ => Token::Delim(c),
242            };
243        }
244    }
245
246    /// §4.3.2 Consume comments — body only.
247    ///
248    /// Precondition: the opening `/*` has already been consumed. Consume
249    /// code points until the closing `*/` or EOF. Per §4.3.2, an
250    /// unterminated comment (EOF before `*/`) is a parse error; we
251    /// silently consume to EOF in that case (no error reporting yet).
252    fn consume_comments_body(&mut self) {
253        loop {
254            match self.consume() {
255                None => return, // EOF in comment: parse error (unreported)
256                Some('*') => {
257                    if self.peek(0) == Some('/') {
258                        self.consume(); // consume `/`
259                        return;
260                    }
261                }
262                Some(_) => {}
263            }
264        }
265    }
266
267    // ── Sub-algorithm stubs (implemented in subsequent commits) ───────
268
269    /// §4.3.5 (L1081-1128) Consume a string token.
270    ///
271    /// `quote` is the ending code point (the opening quote `"` or `'`).
272    /// Returns a `string-token` (normal or EOF-terminated) or a
273    /// `bad-string-token` (unescaped newline).
274    ///
275    /// Per §4.3.5:
276    /// - ending code point (L1097-1099) → return string-token.
277    /// - EOF (L1101-1104) → parse error, return string-token.
278    /// - newline (L1106-1110) → parse error, reconsume, return bad-string.
279    /// - `\` (L1112-1124):
280    ///   - next EOF → do nothing (loop; next consume hits EOF → string).
281    ///   - next newline → consume it (line continuation).
282    ///   - otherwise (valid escape) → consume an escaped code point, append.
283    /// - anything else (L1126-1128) → append.
284    fn consume_a_string_token(&mut self, quote: char) -> Token {
285        let mut value = String::new();
286        loop {
287            match self.consume() {
288                None => {
289                    // §4.3.5 L1101-1104: EOF → parse error, return string-token.
290                    return Token::String(value);
291                }
292                Some(c) if c == quote => {
293                    // §4.3.5 L1097-1099: ending code point → return string-token.
294                    return Token::String(value);
295                }
296                Some('\n') => {
297                    // §4.3.5 L1106-1110: newline → parse error, reconsume, return bad-string.
298                    self.reconsume();
299                    return Token::BadString;
300                }
301                Some('\\') => {
302                    // §4.3.5 L1112-1124: escape handling.
303                    match self.peek(0) {
304                        None => {
305                            // §4.3.5 L1114-1115: next is EOF, do nothing.
306                            // Loop continues; next consume() returns None → String.
307                        }
308                        Some('\n') => {
309                            // §4.3.5 L1117-1119: next is newline, consume it (line continuation).
310                            self.consume();
311                        }
312                        Some(_) => {
313                            // §4.3.5 L1121-1124: valid escape, consume escaped code point.
314                            let escaped = self.consume_an_escaped_code_point();
315                            value.push(escaped);
316                        }
317                    }
318                }
319                Some(c) => {
320                    // §4.3.5 L1126-1128: anything else → append.
321                    value.push(c);
322                }
323            }
324        }
325    }
326
327    /// §4.3.1 (L801-826) `#` branch: Consume a hash token.
328    ///
329    /// Precondition: `#` has been consumed. Per §4.3.1 L801-826:
330    /// - If the next code point is an ident code point OR the next two
331    ///   are a valid escape, then create a `hash-token` (type "id" if
332    ///   would-start-ident-sequence, else "unrestricted"), consume an
333    ///   ident sequence for the value, and return it.
334    /// - Otherwise, return a `delim-token` with value `#`.
335    fn consume_a_hash_token(&mut self) -> Token {
336        // §4.3.1 L803-805: next is ident code point OR next two are valid escape.
337        let next_is_ident = self.peek(0).is_some_and(is_ident_code_point);
338        let next_is_valid_escape = self.is_valid_escape_at(0);
339        if !next_is_ident && !next_is_valid_escape {
340            // §4.3.1 L824-826: return delim-token with value `#`.
341            return Token::Delim('#');
342        }
343        let hash_type = if self.would_start_ident_sequence_at(0) {
344            HashType::Id
345        } else {
346            HashType::Unrestricted
347        };
348        let name = self.consume_an_ident_sequence();
349        Token::Hash(name, hash_type)
350    }
351
352    /// §4.3.3 (L1011-1042) Consume a numeric token.
353    ///
354    /// Returns a `number-token`, `percentage-token`, or `dimension-token`.
355    /// Per §4.3.3:
356    /// 1. Consume a number (§4.3.13).
357    /// 2. If next 3 would start an ident sequence → dimension-token (consume
358    ///    an ident sequence for the unit).
359    /// 3. Else if next is `%` → percentage-token (consume `%`).
360    /// 4. Else → number-token.
361    fn consume_a_numeric_token(&mut self) -> Token {
362        let (value, is_integer) = self.consume_a_number();
363        let numeric = Numeric { value, is_integer };
364        // §4.3.3 L1019-1029: dimension
365        if self.would_start_ident_sequence_at(0) {
366            let unit = self.consume_an_ident_sequence();
367            return Token::Dimension(numeric, unit);
368        }
369        // §4.3.3 L1032-1036: percentage
370        if self.peek(0) == Some('%') {
371            self.consume();
372            return Token::Percentage(numeric);
373        }
374        // §4.3.3 L1040-1042: number
375        Token::Number(numeric)
376    }
377
378    /// §4.3.13 (L1415-1483) Consume a number.
379    ///
380    /// Returns `(value, is_integer)` where `is_integer` is true when the
381    /// source representation had no fractional part and no exponent (type
382    /// "integer"), false when it had either (type "number"). The sign is
383    /// included in `value`.
384    ///
385    /// Per §4.3.13:
386    /// 1. type = "integer"; number_part = ""; exponent_part = "".
387    /// 2. If next is `+`/`-`, consume, append to number_part.
388    /// 3. While next is digit, consume, append to number_part.
389    /// 4. If next 2 are `.` + digit: consume `.`, consume digits; type = "number".
390    /// 5. If next 2/3 are `e`/`E` + optional `+`/`-` + digit: consume `e`/`E`,
391    ///    optional sign, digits into exponent_part; type = "number".
392    /// 6. value = parse number_part; if exponent_part non-empty, value *= 10^exp.
393    /// 7. Return value, type, sign.
394    fn consume_a_number(&mut self) -> (f64, bool) {
395        let mut type_is_number = false;
396        let mut number_part = String::new();
397        let mut exponent_part = String::new();
398
399        // §4.3.13 L1436-1440: sign
400        if self.peek(0) == Some('+') || self.peek(0) == Some('-') {
401            number_part.push(self.consume().unwrap());
402        }
403
404        // §4.3.13 L1442-1444: integer digits
405        while let Some(d) = self.peek(0) {
406            if is_digit(d) {
407                number_part.push(d);
408                self.consume();
409            } else {
410                break;
411            }
412        }
413
414        // §4.3.13 L1446-1454: fraction part
415        if self.peek(0) == Some('.') && matches!(self.peek(1), Some(d) if is_digit(d)) {
416            number_part.push(self.consume().unwrap()); // consume `.`
417            type_is_number = true;
418            while let Some(d) = self.peek(0) {
419                if is_digit(d) {
420                    number_part.push(d);
421                    self.consume();
422                } else {
423                    break;
424                }
425            }
426        }
427
428        // §4.3.13 L1457-1469: exponent part
429        if matches!(self.peek(0), Some('e') | Some('E')) {
430            let p1 = self.peek(1);
431            let p2 = self.peek(2);
432            let valid_exp = match p1 {
433                Some('+') | Some('-') => matches!(p2, Some(d) if is_digit(d)),
434                Some(d) if is_digit(d) => true,
435                _ => false,
436            };
437            if valid_exp {
438                self.consume(); // consume `e`/`E`
439                if self.peek(0) == Some('+') || self.peek(0) == Some('-') {
440                    exponent_part.push(self.consume().unwrap());
441                }
442                while let Some(d) = self.peek(0) {
443                    if is_digit(d) {
444                        exponent_part.push(d);
445                        self.consume();
446                    } else {
447                        break;
448                    }
449                }
450                type_is_number = true;
451            }
452        }
453
454        // §4.3.13 L1472-1480: compute value
455        let mut value: f64 = number_part.parse().unwrap_or(0.0);
456        if !exponent_part.is_empty() {
457            let exp: i32 = exponent_part.parse().unwrap_or(0);
458            value *= 10.0f64.powi(exp);
459        }
460
461        (value, !type_is_number)
462    }
463
464    /// §4.3.4 (L1045-1078) Consume an ident-like token.
465    ///
466    /// Precondition: the current position is at the start of an ident
467    /// sequence (the caller has not consumed anything; or has reconsumed
468    /// the first code point). Returns an `ident-token`, `function-token`,
469    /// `url-token`, or `bad-url-token`.
470    ///
471    /// Per §4.3.4:
472    /// 1. Consume an ident sequence (§4.3.12), yielding `name`.
473    /// 2. If `name` is ASCII-case-insensitive "url" and next is `(`:
474    ///    consume `(`, collapse leading whitespace pairs, then if the
475    ///    next 1-2 code points are a quote (or whitespace+quote) return
476    ///    a `function-token`; otherwise consume a url token (§4.3.6).
477    /// 3. Else if next is `(`: consume it, return `function-token`.
478    /// 4. Else return `ident-token`.
479    fn consume_an_ident_like_token(&mut self) -> Token {
480        let name = self.consume_an_ident_sequence();
481        // §4.3.4 L1053-1066: url( special case
482        if name.eq_ignore_ascii_case("url") && self.peek(0) == Some('(') {
483            self.consume(); // consume `(`
484                            // §4.3.4 L1056-1057: while next two are whitespace, consume one
485            while self.peek(0).is_some_and(is_whitespace) && self.peek(1).is_some_and(is_whitespace)
486            {
487                self.consume();
488            }
489            // §4.3.4 L1058-1063: if next 1-2 are " / ' / ws+" / ws+' → Function
490            let p0 = self.peek(0);
491            let p1 = self.peek(1);
492            let is_quote_case = matches!(p0, Some('"') | Some('\''))
493                || (p0.is_some_and(is_whitespace) && matches!(p1, Some('"') | Some('\'')));
494            if is_quote_case {
495                return Token::Function(name);
496            }
497            // §4.3.4 L1064-1066: otherwise consume a url token
498            return self.consume_a_url_token();
499        }
500        // §4.3.4 L1068-1073: ordinary function
501        if self.peek(0) == Some('(') {
502            self.consume();
503            return Token::Function(name);
504        }
505        // §4.3.4 L1075-1078: ident
506        Token::Ident(name)
507    }
508
509    /// §4.3.4 (at-keyword branch) Consume an at-keyword token.
510    ///
511    /// Precondition: `@` has been consumed. Per §4.3.1, if the next code
512    /// points would start an ident sequence, consume it and return an
513    /// `<at-keyword-token>`; otherwise return a `<delim-token>` with `@`.
514    fn consume_an_at_keyword_token(&mut self) -> Token {
515        if self.would_start_ident_sequence_at(0) {
516            let name = self.consume_an_ident_sequence();
517            Token::AtKeyword(name)
518        } else {
519            Token::Delim('@')
520        }
521    }
522
523    /// §4.3.1 (backslash branch): if would-start-an-escape, consume an
524    /// ident-like token; else emit `<delim-token>`.
525    fn consume_ident_or_delim(&mut self) -> Token {
526        if self.is_valid_escape_next() {
527            self.reconsume();
528            self.consume_an_ident_like_token()
529        } else {
530            Token::Delim('\\')
531        }
532    }
533
534    /// §4.3.6 (L1132-1203) Consume a url token.
535    ///
536    /// Precondition: `url(` has been consumed by the caller (§4.3.4
537    /// L1053-1066). This algorithm consumes the "unquoted" url body and
538    /// returns a `url-token` or `bad-url-token`. A quoted value like
539    /// `url("foo")` is handled by §4.3.4 as a `function-token` and never
540    /// reaches here.
541    ///
542    /// Per §4.3.6:
543    /// - `)` → return url-token.
544    /// - EOF → parse error, return url-token.
545    /// - whitespace → consume trailing whitespace; if next is `)` or EOF
546    ///   return url-token, else bad-url.
547    /// - `"` / `'` / `(` / non-printable → parse error, bad-url.
548    /// - `\` → if valid escape, consume escaped and append; else bad-url.
549    /// - anything else → append.
550    fn consume_a_url_token(&mut self) -> Token {
551        let mut value = String::new();
552        // §4.3.6 L1151: consume as much whitespace as possible
553        while self.peek(0).is_some_and(is_whitespace) {
554            self.consume();
555        }
556        loop {
557            match self.consume() {
558                // §4.3.6 L1157-1159: `)` → return url-token
559                Some(')') => return Token::Url(value),
560                // §4.3.6 L1161-1164: EOF → parse error, return url-token
561                None => return Token::Url(value),
562                // §4.3.6 L1166-1175: whitespace → consume trailing ws, then check
563                Some(c) if is_whitespace(c) => {
564                    while self.peek(0).is_some_and(is_whitespace) {
565                        self.consume();
566                    }
567                    match self.peek(0) {
568                        Some(')') => {
569                            self.consume();
570                            return Token::Url(value);
571                        }
572                        None => return Token::Url(value), // parse error
573                        _ => {
574                            self.consume_the_remnants_of_a_bad_url();
575                            return Token::BadUrl;
576                        }
577                    }
578                }
579                // §4.3.6 L1177-1185: " / ' / ( / non-printable → bad url
580                Some(c) if c == '"' || c == '\'' || c == '(' || is_non_printable(c) => {
581                    self.consume_the_remnants_of_a_bad_url();
582                    return Token::BadUrl;
583                }
584                // §4.3.6 L1187-1197: `\`
585                Some('\\') => {
586                    if self.is_valid_escape_next() {
587                        let escaped = self.consume_an_escaped_code_point();
588                        value.push(escaped);
589                    } else {
590                        self.consume_the_remnants_of_a_bad_url();
591                        return Token::BadUrl;
592                    }
593                }
594                // §4.3.6 L1199-1202: anything else → append
595                Some(c) => value.push(c),
596            }
597        }
598    }
599
600    /// §4.3.15 (L1551-1577) Consume the remnants of a bad url.
601    ///
602    /// Consumes code points until `)` or EOF, allowing an escaped `)`
603    /// (`\)`) to be consumed without ending the bad-url. Per §4.3.15:
604    /// - `)` or EOF → return.
605    /// - valid escape (stream starts with `\` + non-newline non-EOF) →
606    ///   consume an escaped code point.
607    /// - anything else → do nothing.
608    fn consume_the_remnants_of_a_bad_url(&mut self) {
609        loop {
610            match self.consume() {
611                // §4.3.15 L1563-1566: `)` or EOF → return
612                Some(')') | None => return,
613                // §4.3.15 L1568-1572: valid escape → consume escaped code point
614                Some('\\') if self.is_valid_escape_next() => {
615                    let _ = self.consume_an_escaped_code_point();
616                }
617                // §4.3.15 L1574-1576: anything else → do nothing
618                Some(_) => {}
619            }
620        }
621    }
622
623    /// §4.3.1 (L960-972) U/u branch: if `unicode_ranges_allowed` is true
624    /// and the stream would start a unicode-range (§4.3.11), consume a
625    /// unicode-range token (§4.3.14); otherwise consume an ident-like token.
626    ///
627    /// Precondition: `u`/`U` has been consumed, `pos` points just past it.
628    /// This method reconsumes back to `u` before dispatching.
629    fn consume_u_or_unicode_range(&mut self) -> Token {
630        self.reconsume(); // pos back to `u`/`U`
631                          // §4.3.1 L963-967: unicode_ranges_allowed && would-start-unicode-range
632        if self.unicode_ranges_allowed && self.would_start_unicode_range_at(0) {
633            return self.consume_a_unicode_range_token();
634        }
635        // §4.3.1 L969-972: otherwise ident-like
636        self.consume_an_ident_like_token()
637    }
638
639    /// §4.3.11 (L1356-1378) Check if three code points would start a
640    /// unicode-range, examining code points starting at `offset` from the
641    /// current position.
642    ///
643    /// Per §4.3.11: true iff first is `U`/`u`, second is `+`, third is `?`
644    /// or a hex digit.
645    fn would_start_unicode_range_at(&self, offset: usize) -> bool {
646        let first = match self.peek(offset) {
647            Some(c) => c,
648            None => return false,
649        };
650        if !matches!(first, 'U' | 'u') {
651            return false;
652        }
653        if self.peek(offset + 1) != Some('+') {
654            return false;
655        }
656        match self.peek(offset + 2) {
657            Some('?') => true,
658            Some(c) if is_hex_digit(c) => true,
659            _ => false,
660        }
661    }
662
663    /// §4.3.14 (L1487-1548) Consume a unicode-range token.
664    ///
665    /// Precondition: the stream starts with `u`/`U` + `+` + (`?` or hex
666    /// digit) (i.e. [`would_start_unicode_range_at`](Self::would_start_unicode_range_at)
667    /// returns true at the current position).
668    ///
669    /// Returns `UnicodeRange(Some(start), Some(end))`.
670    ///
671    /// Per §4.3.14:
672    /// 1. Consume and discard `u`/`U` and `+`.
673    /// 2. Consume up to 6 hex digits; if fewer than 6, consume `?` up to a
674    ///    total of 6 → `first_segment`.
675    /// 3. If `first_segment` contains `?`: start = `?`→`0`, end = `?`→`F`.
676    /// 4. Else start = hex value of `first_segment`.
677    /// 5. If next two are `-` + hex digit: consume `-`, consume up to 6 hex
678    ///    → end.
679    /// 6. Else end = start.
680    fn consume_a_unicode_range_token(&mut self) -> Token {
681        // §4.3.14 step 1: consume and discard `u`/`U` + `+`
682        self.consume(); // `u`/`U`
683        self.consume(); // `+`
684
685        // §4.3.14 step 2: consume up to 6 hex digits
686        let mut first_segment = String::new();
687        while first_segment.len() < 6 {
688            match self.peek(0) {
689                Some(c) if is_hex_digit(c) => {
690                    first_segment.push(c);
691                    self.consume();
692                }
693                _ => break,
694            }
695        }
696        // then consume `?` up to a total of 6
697        while first_segment.len() < 6 {
698            match self.peek(0) {
699                Some('?') => {
700                    first_segment.push('?');
701                    self.consume();
702                }
703                _ => break,
704            }
705        }
706
707        // §4.3.14 step 3: if first_segment contains `?`
708        if first_segment.contains('?') {
709            let start_str: String = first_segment
710                .chars()
711                .map(|c| if c == '?' { '0' } else { c })
712                .collect();
713            let start = u32::from_str_radix(&start_str, 16).unwrap_or(0);
714            let end_str: String = first_segment
715                .chars()
716                .map(|c| if c == '?' { 'F' } else { c })
717                .collect();
718            let end = u32::from_str_radix(&end_str, 16).unwrap_or(0);
719            return Token::UnicodeRange(Some(start), Some(end));
720        }
721
722        // §4.3.14 step 4: first_segment as hex → start
723        let start = u32::from_str_radix(&first_segment, 16).unwrap_or(0);
724
725        // §4.3.14 step 5: if next two are `-` + hex digit
726        if self.peek(0) == Some('-') && self.peek(1).is_some_and(is_hex_digit) {
727            self.consume(); // consume `-`
728            let mut end_segment = String::new();
729            while end_segment.len() < 6 {
730                match self.peek(0) {
731                    Some(c) if is_hex_digit(c) => {
732                        end_segment.push(c);
733                        self.consume();
734                    }
735                    _ => break,
736                }
737            }
738            let end = u32::from_str_radix(&end_segment, 16).unwrap_or(0);
739            return Token::UnicodeRange(Some(start), Some(end));
740        }
741
742        // §4.3.14 step 6: otherwise start == end
743        Token::UnicodeRange(Some(start), Some(start))
744    }
745
746    // ── §4.3.12 Consume an ident sequence ────────────────────────────
747
748    /// §4.3.12 Consume an ident sequence.
749    ///
750    /// Repeatedly consume code points as long as they form an ident
751    /// sequence: ident code points, valid escapes (consumed via
752    /// §4.3.7), or `\`-`-` (for `--`-prefixed names). Returns the
753    /// decoded name with escapes resolved.
754    ///
755    /// Per §4.3.12, the loop terminates when:
756    /// - EOF is reached, or
757    /// - the next code point is not an ident code point, not a valid
758    ///   escape, and not part of a `\`-`-` continuation.
759    fn consume_an_ident_sequence(&mut self) -> String {
760        let mut result = String::new();
761        loop {
762            let Some(c) = self.consume() else {
763                return result;
764            };
765            if is_ident_code_point(c) {
766                result.push(c);
767            } else if c == '\\' && self.is_valid_escape_next() {
768                // §4.3.12: valid escape → consume the escaped code point
769                // and append it.
770                let escaped = self.consume_an_escaped_code_point();
771                result.push(escaped);
772            } else {
773                // Not part of the ident sequence: reconsume and stop.
774                self.reconsume();
775                return result;
776            }
777        }
778    }
779
780    // ── §4.3.7 Consume an escaped code point ────────────────────────
781
782    /// §4.3.7 Consume an escaped code point.
783    ///
784    /// Precondition: the `\` has been consumed. Returns the decoded code
785    /// point.
786    ///
787    /// Per §4.3.7 (L1207-1240):
788    /// - EOF (L1233-1236) → parse error, return U+FFFD REPLACEMENT CHARACTER.
789    /// - newline → parse error, reconsume (invalid escape; the caller
790    ///   should have checked §4.3.8 first, but we handle defensively).
791    /// - hex digit(s) (L1219-1231) → consume 1-6 hex digits, optionally
792    ///   followed by whitespace, and return the code point with that
793    ///   value. U+0000 NULL or out-of-range → replacement character U+FFFD.
794    /// - anything else (L1238-1240) → return the consumed code point.
795    fn consume_an_escaped_code_point(&mut self) -> char {
796        let Some(c) = self.consume() else {
797            // §4.3.7 L1233-1236: EOF → parse error, return U+FFFD.
798            return '\u{FFFD}';
799        };
800        if is_hex_digit(c) {
801            // §4.3.7 step 4: consume up to 5 more hex digits (total ≤ 6).
802            let mut hex = String::new();
803            hex.push(c);
804            while hex.len() < 6 {
805                match self.peek(0) {
806                    Some(next) if is_hex_digit(next) => {
807                        hex.push(next);
808                        self.consume();
809                    }
810                    _ => break,
811                }
812            }
813            // §4.3.7 step 4: consume one trailing whitespace.
814            if self.peek(0) == Some(' ') || self.peek(0) == Some('\t') || self.peek(0) == Some('\n')
815            {
816                self.consume();
817            }
818            // Parse the hex value.
819            let value = u32::from_str_radix(&hex, 16).unwrap_or(0);
820            // §4.3.7 step 4: 0 or > 0x10FFFF → U+FFFD. Also surrogates
821            // (0xD800–0xDFFF) → U+FFFD.
822            if value == 0 || value > 0x10FFFF || (0xD800..=0xDFFF).contains(&value) {
823                '\u{FFFD}'
824            } else {
825                char::from_u32(value).unwrap_or('\u{FFFD}')
826            }
827        } else if c == '\n' {
828            // §4.3.7 step 3: newline after `\` is a parse error. This
829            // should not happen if the caller checked §4.3.8, but handle
830            // defensively by returning `\` and reconsuming the newline.
831            self.reconsume();
832            '\\'
833        } else {
834            // §4.3.7 step 5: any other code point → return it.
835            c
836        }
837    }
838
839    // ── §4.3.8 / §4.3.9 Predicates ──────────────────────────────────
840
841    /// §4.3.8 Check if two code points are a valid escape.
842    ///
843    /// Precondition: the `\` has been consumed. Returns true if the next
844    /// code point forms a valid escape with the consumed `\` (i.e. the
845    /// next code point is not a newline and not EOF).
846    fn is_valid_escape_next(&self) -> bool {
847        match self.peek(0) {
848            None => false,
849            Some('\n') => false,
850            Some(_) => true,
851        }
852    }
853
854    /// §4.3.9 Check if three code points would start an ident sequence,
855    /// examining code points starting at `offset` from the current
856    /// position.
857    ///
858    /// Per §4.3.9:
859    /// - U+002D HYPHEN-MINUS (`-`):
860    ///   - If the next is also `-`, true (start of `--...`).
861    ///   - If the next is an ident-start code point (or a valid escape),
862    ///     true.
863    ///   - Else false.
864    /// - U+005C REVERSE SOLIDUS (`\`): true if valid escape.
865    /// - ident-start code point: true.
866    /// - Else false.
867    fn would_start_ident_sequence_at(&self, offset: usize) -> bool {
868        let first = match self.peek(offset) {
869            Some(c) => c,
870            None => return false,
871        };
872        match first {
873            '-' => {
874                // §4.3.9: if next is `-`, true.
875                match self.peek(offset + 1) {
876                    Some('-') => true,
877                    Some(next) if is_ident_start_code_point(next) => true,
878                    Some('\\') => {
879                        // §4.3.9: if `\` and the following forms a valid escape.
880                        self.is_valid_escape_at(offset + 1)
881                    }
882                    _ => false,
883                }
884            }
885            '\\' => self.is_valid_escape_at(offset),
886            _ if is_ident_start_code_point(first) => true,
887            _ => false,
888        }
889    }
890
891    /// §4.3.8 Check if the code point at `offset` starts a valid escape
892    /// (i.e. `offset` points at `\` and `offset+1` is not newline/EOF).
893    fn is_valid_escape_at(&self, offset: usize) -> bool {
894        if self.peek(offset) != Some('\\') {
895            return false;
896        }
897        match self.peek(offset + 1) {
898            None => false,
899            Some('\n') => false,
900            Some(_) => true,
901        }
902    }
903
904    // ── Predicates (stubs returning false; implemented in C-4/C-6) ──
905
906    /// §4.3.10 (L1307-1352) Check if three code points would start a number.
907    ///
908    /// `first` is the code point already consumed (the caller passes it
909    /// so the predicate can examine the next two without reconsume). The
910    /// three code points examined are `first`, `peek(0)`, `peek(1)`.
911    ///
912    /// Per §4.3.10:
913    /// - `+`/`-`: second is digit → true; second is `.` and third is digit
914    ///   → true; else false.
915    /// - `.`: second is digit → true; else false.
916    /// - digit: true.
917    /// - anything else: false.
918    fn starts_with_number(&self, first: char) -> bool {
919        match first {
920            '+' | '-' => match self.peek(0) {
921                Some(d) if is_digit(d) => true,
922                Some('.') => matches!(self.peek(1), Some(d) if is_digit(d)),
923                _ => false,
924            },
925            '.' => matches!(self.peek(0), Some(d) if is_digit(d)),
926            d if is_digit(d) => true,
927            _ => false,
928        }
929    }
930}
931
932// ── Free functions (§4.2 code point classes) ──────────────────────────
933
934/// §4.2 Whether `c` is an ident-start code point.
935///
936/// An ident-start code point is an ASCII letter, a non-ASCII code point
937/// (U+0080 or higher), or U+005F LOW LINE (`_`).
938fn is_ident_start_code_point(c: char) -> bool {
939    matches!(c, 'A'..='Z' | 'a'..='z' | '_' | '\u{0080}'..=char::MAX)
940}
941
942/// §4.2 Whether `c` is an ident code point (allowed after the first).
943///
944/// An ident code point is an ident-start code point, or an ASCII digit,
945/// or U+002D HYPHEN-MINUS (`-`).
946fn is_ident_code_point(c: char) -> bool {
947    matches!(c, 'A'..='Z' | 'a'..='z' | '0'..='9' | '_' | '-' | '\u{0080}'..=char::MAX)
948}
949
950/// §4.2 Whether `c` is a digit (ASCII 0-9).
951fn is_digit(c: char) -> bool {
952    c.is_ascii_digit()
953}
954
955/// §4.2 Whether `c` is a hex digit (ASCII 0-9, A-F, a-f).
956fn is_hex_digit(c: char) -> bool {
957    c.is_ascii_hexdigit()
958}
959
960/// §4.2 Whether `c` is a whitespace code point.
961///
962/// Per §4.2: U+0009 TAB, U+000A LF, U+000C FF, U+000D CR, U+0020 SPACE.
963/// (After §5.3 preprocessing, U+000C FF and lone U+000D CR have been
964/// normalized to U+000A LF, but the helper follows the full §4.2
965/// definition for correctness when called on arbitrary input.)
966fn is_whitespace(c: char) -> bool {
967    matches!(c, '\t' | '\n' | '\u{000C}' | '\r' | ' ')
968}
969
970/// §4.2 Whether `c` is a non-printable code point.
971///
972/// Per §4.2: U+0000-U+0008, U+000B, U+000E-U+001F, U+007F-U+009F.
973fn is_non_printable(c: char) -> bool {
974    matches!(
975        c,
976        '\u{0000}'..='\u{0008}' | '\u{000B}' | '\u{000E}'..='\u{001F}' | '\u{007F}'..='\u{009F}'
977    )
978}
979
980/// §5.3 input preprocessing.
981///
982/// Per the "input stream" definition in §5.3:
983/// - U+000D CR followed by U+000A LF → single U+000A LF
984/// - lone U+000D CR → U+000A LF
985/// - U+000C FF → U+000A LF
986///
987/// The tokenizer operates on this normalized stream.
988fn preprocess_input(s: &str) -> Vec<char> {
989    let mut out = Vec::with_capacity(s.len());
990    let mut chars = s.chars().peekable();
991    while let Some(c) = chars.next() {
992        match c {
993            '\r' => {
994                if chars.peek() == Some(&'\n') {
995                    chars.next();
996                }
997                out.push('\n');
998            }
999            '\u{000C}' => out.push('\n'),
1000            _ => out.push(c),
1001        }
1002    }
1003    out
1004}
1005
1006impl Tokenizer for CssTokenizer {
1007    fn next_token(&mut self) -> Option<Token> {
1008        if self.eof_emitted {
1009            return None;
1010        }
1011        Some(self.consume_a_token())
1012    }
1013
1014    fn state(&self) -> State {
1015        self.state
1016    }
1017
1018    fn set_state(&mut self, state: State) {
1019        self.state = state;
1020        if state == State::Data {
1021            // Resetting to Data after EOF re-enables the iterator; useful
1022            // for `reset()` callers that want to re-tokenize.
1023            self.eof_emitted = false;
1024        }
1025    }
1026
1027    fn reset(&mut self) {
1028        self.pos = 0;
1029        self.state = State::Data;
1030        self.eof_emitted = false;
1031    }
1032
1033    fn set_unicode_ranges_allowed(&mut self, allowed: bool) {
1034        self.unicode_ranges_allowed = allowed;
1035    }
1036
1037    fn position(&self) -> usize {
1038        self.pos
1039    }
1040}
1041
1042// ── Test helpers exposed for unit tests in this file ──────────────────
1043
1044#[cfg(test)]
1045impl CssTokenizer {
1046    /// Tokenize the entire input and return all tokens except the
1047    /// trailing `<EOF-token>`. Test-only convenience.
1048    fn collect(input: &str) -> Vec<Token> {
1049        let mut tz = Self::new(input);
1050        let mut out = Vec::new();
1051        while let Some(t) = tz.next_token() {
1052            if matches!(t, Token::Eof) {
1053                break;
1054            }
1055            out.push(t);
1056        }
1057        out
1058    }
1059}
1060
1061#[cfg(test)]
1062mod tests {
1063    use super::*;
1064
1065    #[test]
1066    fn empty_input_emits_eof_only() {
1067        let mut tz = CssTokenizer::new("");
1068        assert!(matches!(tz.next_token(), Some(Token::Eof)));
1069        assert!(tz.next_token().is_none());
1070    }
1071
1072    #[test]
1073    fn whitespace_collapses_runs() {
1074        let tokens = CssTokenizer::collect("   \t\n  ");
1075        assert_eq!(tokens.len(), 1, "whitespace run → single token");
1076        assert!(matches!(tokens[0], Token::Whitespace));
1077    }
1078
1079    #[test]
1080    fn simple_punctuation_tokens() {
1081        let tokens = CssTokenizer::collect(":;,.{}[]()");
1082        assert_eq!(tokens.len(), 10);
1083        assert!(matches!(tokens[0], Token::Colon));
1084        assert!(matches!(tokens[1], Token::Semicolon));
1085        assert!(matches!(tokens[2], Token::Comma));
1086        assert!(matches!(tokens[3], Token::Delim('.')));
1087        assert!(matches!(tokens[4], Token::OpenBrace));
1088        assert!(matches!(tokens[5], Token::CloseBrace));
1089        assert!(matches!(tokens[6], Token::OpenBracket));
1090        assert!(matches!(tokens[7], Token::CloseBracket));
1091        assert!(matches!(tokens[8], Token::OpenParen));
1092        assert!(matches!(tokens[9], Token::CloseParen));
1093    }
1094
1095    #[test]
1096    fn close_paren_is_emitted() {
1097        let tokens = CssTokenizer::collect(")");
1098        assert_eq!(tokens.len(), 1);
1099        assert!(matches!(tokens[0], Token::CloseParen));
1100    }
1101
1102    #[test]
1103    fn unknown_delim_chars() {
1104        let tokens = CssTokenizer::collect("^~`>");
1105        assert_eq!(tokens.len(), 4);
1106        assert!(matches!(tokens[0], Token::Delim('^')));
1107        assert!(matches!(tokens[1], Token::Delim('~')));
1108        assert!(matches!(tokens[2], Token::Delim('`')));
1109        assert!(matches!(tokens[3], Token::Delim('>')));
1110    }
1111
1112    #[test]
1113    fn preprocessing_normalizes_cr_lf_ff() {
1114        // CR LF → single LF
1115        let tz = CssTokenizer::new("a\r\nb");
1116        assert_eq!(tz.input, vec!['a', '\n', 'b']);
1117        // Lone CR → LF
1118        let tz = CssTokenizer::new("a\rb");
1119        assert_eq!(tz.input, vec!['a', '\n', 'b']);
1120        // FF → LF
1121        let tz = CssTokenizer::new("a\u{000C}b");
1122        assert_eq!(tz.input, vec!['a', '\n', 'b']);
1123    }
1124
1125    #[test]
1126    fn eof_emitted_once() {
1127        // C-1: use a non-ident-start, non-digit char to avoid the
1128        // ident-like / numeric branches (which are stubbed with todo!()
1129        // until C-2 / C-4).
1130        let mut tz = CssTokenizer::new(",");
1131        let t1 = tz.next_token();
1132        let t2 = tz.next_token();
1133        let t3 = tz.next_token();
1134        assert!(matches!(t1, Some(Token::Comma)));
1135        assert!(matches!(t2, Some(Token::Eof)));
1136        assert!(t3.is_none());
1137    }
1138
1139    #[test]
1140    fn block_comment_is_skipped() {
1141        // §4.3.2: `/* ... */` is consumed and no token is emitted.
1142        let tokens = CssTokenizer::collect("/* hello */:/*x*/;");
1143        assert_eq!(tokens.len(), 2);
1144        assert!(matches!(tokens[0], Token::Colon));
1145        assert!(matches!(tokens[1], Token::Semicolon));
1146    }
1147
1148    #[test]
1149    fn unterminated_comment_consumes_to_eof() {
1150        // §4.3.2: EOF before `*/` is a parse error; we consume to EOF.
1151        let tokens = CssTokenizer::collect("/* unfinished");
1152        assert_eq!(tokens.len(), 0, "unterminated comment → no tokens");
1153    }
1154
1155    #[test]
1156    fn comment_between_tokens() {
1157        // C-2 not yet: `a`/`b` would be ident tokens (todo!()). Verify
1158        // comment is consumed without affecting surrounding simple tokens
1159        // by using only punctuation.
1160        let tokens = CssTokenizer::collect(":/*c*/;");
1161        assert_eq!(tokens.len(), 2);
1162        assert!(matches!(tokens[0], Token::Colon));
1163        assert!(matches!(tokens[1], Token::Semicolon));
1164    }
1165
1166    #[test]
1167    fn cdo_token_emitted() {
1168        // §4.3.1: `<!--` → <CDO-token>
1169        let tokens = CssTokenizer::collect("<!--");
1170        assert_eq!(tokens.len(), 1);
1171        assert!(matches!(tokens[0], Token::Cdo));
1172    }
1173
1174    #[test]
1175    fn cdc_token_emitted() {
1176        // §4.3.1: `-->` → <CDC-token>
1177        let tokens = CssTokenizer::collect("-->");
1178        assert_eq!(tokens.len(), 1);
1179        assert!(matches!(tokens[0], Token::Cdc));
1180    }
1181
1182    #[test]
1183    fn cdo_cdc_wrapping_stylesheet() {
1184        // Legacy CSS 1/2.1 stylesheet wrapping.
1185        // `<!-- : ; -->`
1186        // tokens: Cdo, WS, Colon, WS, Semicolon, WS, Cdc = 7
1187        let tokens = CssTokenizer::collect("<!-- : ; -->");
1188        assert_eq!(tokens.len(), 7);
1189        assert!(matches!(tokens[0], Token::Cdo));
1190        assert!(matches!(tokens[1], Token::Whitespace));
1191        assert!(matches!(tokens[2], Token::Colon));
1192        assert!(matches!(tokens[3], Token::Whitespace));
1193        assert!(matches!(tokens[4], Token::Semicolon));
1194        assert!(matches!(tokens[5], Token::Whitespace));
1195        assert!(matches!(tokens[6], Token::Cdc));
1196    }
1197
1198    #[test]
1199    fn less_than_alone_is_delim() {
1200        // §4.3.1: `<` not followed by `!-` → <delim-token>
1201        let tokens = CssTokenizer::collect("< ");
1202        assert_eq!(tokens.len(), 2);
1203        assert!(matches!(tokens[0], Token::Delim('<')));
1204        assert!(matches!(tokens[1], Token::Whitespace));
1205    }
1206
1207    #[test]
1208    fn hyphen_alone_is_delim() {
1209        // §4.3.1: `-` not followed by number/ident/`->` → <delim-token>
1210        let tokens = CssTokenizer::collect("- ");
1211        assert_eq!(tokens.len(), 2);
1212        assert!(matches!(tokens[0], Token::Delim('-')));
1213        assert!(matches!(tokens[1], Token::Whitespace));
1214    }
1215
1216    // ── C-2 tests: ident / function / at-keyword / hash ──────────────
1217
1218    #[test]
1219    fn simple_ident() {
1220        let tokens = CssTokenizer::collect("color");
1221        assert_eq!(tokens.len(), 1);
1222        assert!(matches!(&tokens[0], Token::Ident(s) if s == "color"));
1223    }
1224
1225    #[test]
1226    fn ident_with_hyphen_prefix() {
1227        let tokens = CssTokenizer::collect("-webkit-flex");
1228        assert_eq!(tokens.len(), 1);
1229        assert!(matches!(&tokens[0], Token::Ident(s) if s == "-webkit-flex"));
1230    }
1231
1232    #[test]
1233    fn ident_with_double_hyphen() {
1234        // §4.3.9: `--` starts an ident sequence (custom properties).
1235        let tokens = CssTokenizer::collect("--my-var");
1236        assert_eq!(tokens.len(), 1);
1237        assert!(matches!(&tokens[0], Token::Ident(s) if s == "--my-var"));
1238    }
1239
1240    #[test]
1241    fn ident_with_underscore() {
1242        let tokens = CssTokenizer::collect("_private");
1243        assert_eq!(tokens.len(), 1);
1244        assert!(matches!(&tokens[0], Token::Ident(s) if s == "_private"));
1245    }
1246
1247    #[test]
1248    fn ident_with_escape() {
1249        // §4.3.7: `\26` → '&' (U+0026)
1250        let tokens = CssTokenizer::collect("color\\26 B");
1251        assert_eq!(tokens.len(), 1);
1252        assert!(matches!(&tokens[0], Token::Ident(s) if s == "color&B"));
1253    }
1254
1255    #[test]
1256    fn function_token() {
1257        let tokens = CssTokenizer::collect("translate(");
1258        assert_eq!(tokens.len(), 1);
1259        assert!(matches!(&tokens[0], Token::Function(s) if s == "translate"));
1260    }
1261
1262    #[test]
1263    fn at_keyword_token() {
1264        let tokens = CssTokenizer::collect("@media");
1265        assert_eq!(tokens.len(), 1);
1266        assert!(matches!(&tokens[0], Token::AtKeyword(s) if s == "media"));
1267    }
1268
1269    #[test]
1270    fn at_keyword_with_hyphen() {
1271        let tokens = CssTokenizer::collect("@-webkit-keyframes");
1272        assert_eq!(tokens.len(), 1);
1273        assert!(matches!(&tokens[0], Token::AtKeyword(s) if s == "-webkit-keyframes"));
1274    }
1275
1276    #[test]
1277    fn at_sign_alone_is_delim() {
1278        // §4.3.1: `@` not followed by ident-start → <delim-token>
1279        let tokens = CssTokenizer::collect("@ ");
1280        assert_eq!(tokens.len(), 2);
1281        assert!(matches!(tokens[0], Token::Delim('@')));
1282        assert!(matches!(tokens[1], Token::Whitespace));
1283    }
1284
1285    #[test]
1286    fn hash_id_type() {
1287        // §4.3.1 / §4.3.4: `#main` → <hash-token> with type "id"
1288        let tokens = CssTokenizer::collect("#main");
1289        assert_eq!(tokens.len(), 1);
1290        match &tokens[0] {
1291            Token::Hash(s, t) => {
1292                assert_eq!(s, "main");
1293                assert_eq!(*t, crate::types::HashType::Id);
1294            }
1295            _ => panic!("expected Hash, got {:?}", tokens[0]),
1296        }
1297    }
1298
1299    #[test]
1300    fn hash_unrestricted_type() {
1301        // §4.3.1 L801-826 + §4.3.4: `#123` → next `1` is an ident code
1302        // point (digits are ident code points per §4.2), so a hash-token
1303        // is created. `123` does not would-start-an-ident-sequence (digit
1304        // is not ident-start), so type is "unrestricted". The value is
1305        // "123" (consume_an_ident_sequence consumes all ident code points).
1306        let tokens = CssTokenizer::collect("#123");
1307        assert_eq!(tokens.len(), 1);
1308        match &tokens[0] {
1309            Token::Hash(s, t) => {
1310                assert_eq!(s, "123");
1311                assert_eq!(*t, crate::types::HashType::Unrestricted);
1312            }
1313            _ => panic!("expected Hash, got {:?}", tokens[0]),
1314        }
1315    }
1316
1317    #[test]
1318    fn hash_hex_color_is_unrestricted() {
1319        // §4.3.4: `#ff00aa` → starts with `f` which IS ident-start, so
1320        // this is actually type "id" (the hex digits form a valid ident).
1321        let tokens = CssTokenizer::collect("#ff00aa");
1322        assert_eq!(tokens.len(), 1);
1323        match &tokens[0] {
1324            Token::Hash(s, t) => {
1325                assert_eq!(s, "ff00aa");
1326                assert_eq!(*t, crate::types::HashType::Id);
1327            }
1328            _ => panic!("expected Hash, got {:?}", tokens[0]),
1329        }
1330    }
1331
1332    #[test]
1333    fn backslash_escape_in_ident() {
1334        // §4.3.7: `\` followed by non-hex → literal char. Use `z` (not
1335        // `b`, which is a hex digit and would decode to U+000B).
1336        let tokens = CssTokenizer::collect("a\\z");
1337        assert_eq!(tokens.len(), 1);
1338        assert!(matches!(&tokens[0], Token::Ident(s) if s == "az"));
1339    }
1340
1341    #[test]
1342    fn backslash_alone_is_delim() {
1343        // §4.3.1: `\` not followed by valid escape (EOF or newline) →
1344        // <delim-token>
1345        let tokens = CssTokenizer::collect("\\");
1346        assert_eq!(tokens.len(), 1);
1347        assert!(matches!(tokens[0], Token::Delim('\\')));
1348    }
1349
1350    #[test]
1351    fn backslash_newline_is_delim() {
1352        // §4.3.8: `\` followed by newline is not a valid escape.
1353        let tokens = CssTokenizer::collect("\\\n");
1354        assert_eq!(tokens.len(), 2);
1355        assert!(matches!(tokens[0], Token::Delim('\\')));
1356        assert!(matches!(tokens[1], Token::Whitespace));
1357    }
1358
1359    #[test]
1360    fn declaration_with_ident_and_value() {
1361        let tokens = CssTokenizer::collect("color: red");
1362        assert_eq!(tokens.len(), 4);
1363        assert!(matches!(&tokens[0], Token::Ident(s) if s == "color"));
1364        assert!(matches!(tokens[1], Token::Colon));
1365        assert!(matches!(tokens[2], Token::Whitespace));
1366        assert!(matches!(&tokens[3], Token::Ident(s) if s == "red"));
1367    }
1368
1369    #[test]
1370    fn unicode_escape_in_ident() {
1371        // §4.3.7 L1223-1225: `\000026` → '&' (U+0026); after 6 hex digits,
1372        // if next is whitespace, consume it. So `\000026 B` → the space is
1373        // consumed as trailing whitespace, `B` joins the ident sequence →
1374        // single token Ident("&B").
1375        let tokens = CssTokenizer::collect("\\000026 B");
1376        assert_eq!(tokens.len(), 1);
1377        assert!(matches!(&tokens[0], Token::Ident(s) if s == "&B"));
1378    }
1379
1380    #[test]
1381    fn null_escape_becomes_replacement_char() {
1382        // §4.3.7: `\0` → U+FFFD (NULL is not allowed)
1383        let tokens = CssTokenizer::collect("\\0");
1384        assert_eq!(tokens.len(), 1);
1385        assert!(matches!(&tokens[0], Token::Ident(s) if s == "\u{FFFD}"));
1386    }
1387
1388    #[test]
1389    fn surrogate_escape_becomes_replacement_char() {
1390        // §4.3.7: `\D800` → U+FFFD (surrogates not allowed)
1391        let tokens = CssTokenizer::collect("\\D800");
1392        assert_eq!(tokens.len(), 1);
1393        assert!(matches!(&tokens[0], Token::Ident(s) if s == "\u{FFFD}"));
1394    }
1395
1396    // ── C-2 Bug 1 回归测试: # Delim 回退 (§4.3.1 L824-826) ────────────
1397
1398    #[test]
1399    fn hash_alone_is_delim() {
1400        // §4.3.1 L824-826: `#` followed by EOF → next is not ident code
1401        // point, not valid escape → return delim-token with value `#`.
1402        let tokens = CssTokenizer::collect("#");
1403        assert_eq!(tokens.len(), 1);
1404        assert!(matches!(tokens[0], Token::Delim('#')));
1405    }
1406
1407    #[test]
1408    fn hash_followed_by_space_is_delim() {
1409        // §4.3.1 L824-826: `#` followed by space (not ident code point,
1410        // not valid escape) → delim-token with value `#`.
1411        let tokens = CssTokenizer::collect("# ");
1412        assert_eq!(tokens.len(), 2);
1413        assert!(matches!(tokens[0], Token::Delim('#')));
1414        assert!(matches!(tokens[1], Token::Whitespace));
1415    }
1416
1417    #[test]
1418    fn hash_followed_by_at_is_delim() {
1419        // §4.3.1 L824-826: `#` followed by `@` (not ident code point,
1420        // not valid escape) → delim-token with value `#`.
1421        let tokens = CssTokenizer::collect("#@");
1422        assert_eq!(tokens.len(), 2);
1423        assert!(matches!(tokens[0], Token::Delim('#')));
1424        assert!(matches!(tokens[1], Token::Delim('@')));
1425    }
1426
1427    // ── C-3 tests: §4.3.5 string + bad-string ────────────────────────
1428
1429    #[test]
1430    fn string_double_quoted() {
1431        // §4.3.5: "hello" → String("hello")
1432        let tokens = CssTokenizer::collect("\"hello\"");
1433        assert_eq!(tokens.len(), 1);
1434        assert!(matches!(&tokens[0], Token::String(s) if s == "hello"));
1435    }
1436
1437    #[test]
1438    fn string_single_quoted() {
1439        // §4.3.5: 'hello' → String("hello")
1440        let tokens = CssTokenizer::collect("'hello'");
1441        assert_eq!(tokens.len(), 1);
1442        assert!(matches!(&tokens[0], Token::String(s) if s == "hello"));
1443    }
1444
1445    #[test]
1446    fn string_with_escape() {
1447        // §4.3.5 + §4.3.7: "a\z" → String("az") (z is non-hex → literal)
1448        let tokens = CssTokenizer::collect("\"a\\z\"");
1449        assert_eq!(tokens.len(), 1);
1450        assert!(matches!(&tokens[0], Token::String(s) if s == "az"));
1451    }
1452
1453    #[test]
1454    fn string_unterminated_eof() {
1455        // §4.3.5 L1101-1104: EOF before closing quote → parse error,
1456        // return string-token with value accumulated so far.
1457        let tokens = CssTokenizer::collect("\"hello");
1458        assert_eq!(tokens.len(), 1);
1459        assert!(matches!(&tokens[0], Token::String(s) if s == "hello"));
1460    }
1461
1462    #[test]
1463    fn string_unescaped_newline_is_bad_string() {
1464        // §4.3.5 L1106-1110: unescaped newline → parse error, reconsume,
1465        // return bad-string. The reconsumed newline is then tokenized as
1466        // whitespace by the next consume_a_token call.
1467        let tokens = CssTokenizer::collect("\"\n");
1468        assert_eq!(tokens.len(), 2);
1469        assert!(matches!(tokens[0], Token::BadString));
1470        assert!(matches!(tokens[1], Token::Whitespace));
1471    }
1472
1473    #[test]
1474    fn string_line_continuation() {
1475        // §4.3.5 L1117-1119: backslash followed by newline → line
1476        // continuation (newline consumed, no char appended).
1477        let tokens = CssTokenizer::collect("\"a\\\nb\"");
1478        assert_eq!(tokens.len(), 1);
1479        assert!(matches!(&tokens[0], Token::String(s) if s == "ab"));
1480    }
1481
1482    #[test]
1483    fn string_hex_escape() {
1484        // §4.3.5 + §4.3.7: "\26" → String("&") (U+0026)
1485        let tokens = CssTokenizer::collect("\"\\26\"");
1486        assert_eq!(tokens.len(), 1);
1487        assert!(matches!(&tokens[0], Token::String(s) if s == "&"));
1488    }
1489
1490    // ── C-4 tests: §4.3.3/§4.3.10/§4.3.13 number/percentage/dimension ─
1491
1492    #[test]
1493    fn number_integer() {
1494        let tokens = CssTokenizer::collect("42");
1495        assert_eq!(tokens.len(), 1);
1496        assert_eq!(
1497            tokens[0],
1498            Token::Number(Numeric {
1499                value: 42.0,
1500                is_integer: true
1501            })
1502        );
1503    }
1504
1505    #[test]
1506    fn number_decimal() {
1507        let tokens = CssTokenizer::collect("3.5");
1508        assert_eq!(tokens.len(), 1);
1509        assert_eq!(
1510            tokens[0],
1511            Token::Number(Numeric {
1512                value: 3.5,
1513                is_integer: false
1514            })
1515        );
1516    }
1517
1518    #[test]
1519    fn number_signed() {
1520        let tokens = CssTokenizer::collect("-5");
1521        assert_eq!(tokens.len(), 1);
1522        assert_eq!(
1523            tokens[0],
1524            Token::Number(Numeric {
1525                value: -5.0,
1526                is_integer: true
1527            })
1528        );
1529    }
1530
1531    #[test]
1532    fn number_exponent() {
1533        // §4.3.13: 1e3 → 1.0 * 10^3 = 1000.0, type "number"
1534        let tokens = CssTokenizer::collect("1e3");
1535        assert_eq!(tokens.len(), 1);
1536        assert_eq!(
1537            tokens[0],
1538            Token::Number(Numeric {
1539                value: 1000.0,
1540                is_integer: false
1541            })
1542        );
1543    }
1544
1545    #[test]
1546    fn number_decimal_exponent() {
1547        // §4.3.13: 1.5e2 → 1.5 * 10^2 = 150.0, type "number"
1548        let tokens = CssTokenizer::collect("1.5e2");
1549        assert_eq!(tokens.len(), 1);
1550        assert_eq!(
1551            tokens[0],
1552            Token::Number(Numeric {
1553                value: 150.0,
1554                is_integer: false
1555            })
1556        );
1557    }
1558
1559    #[test]
1560    fn percentage_token() {
1561        let tokens = CssTokenizer::collect("50%");
1562        assert_eq!(tokens.len(), 1);
1563        assert_eq!(
1564            tokens[0],
1565            Token::Percentage(Numeric {
1566                value: 50.0,
1567                is_integer: true
1568            })
1569        );
1570    }
1571
1572    #[test]
1573    fn dimension_px() {
1574        let tokens = CssTokenizer::collect("10px");
1575        assert_eq!(tokens.len(), 1);
1576        assert_eq!(
1577            tokens[0],
1578            Token::Dimension(
1579                Numeric {
1580                    value: 10.0,
1581                    is_integer: true
1582                },
1583                "px".to_string(),
1584            )
1585        );
1586    }
1587
1588    #[test]
1589    fn dimension_em() {
1590        let tokens = CssTokenizer::collect("1.5em");
1591        assert_eq!(tokens.len(), 1);
1592        assert_eq!(
1593            tokens[0],
1594            Token::Dimension(
1595                Numeric {
1596                    value: 1.5,
1597                    is_integer: false
1598                },
1599                "em".to_string(),
1600            )
1601        );
1602    }
1603
1604    #[test]
1605    fn dimension_signed() {
1606        let tokens = CssTokenizer::collect("-30deg");
1607        assert_eq!(tokens.len(), 1);
1608        assert_eq!(
1609            tokens[0],
1610            Token::Dimension(
1611                Numeric {
1612                    value: -30.0,
1613                    is_integer: true
1614                },
1615                "deg".to_string(),
1616            )
1617        );
1618    }
1619
1620    #[test]
1621    fn plus_sign_number() {
1622        let tokens = CssTokenizer::collect("+5");
1623        assert_eq!(tokens.len(), 1);
1624        assert_eq!(
1625            tokens[0],
1626            Token::Number(Numeric {
1627                value: 5.0,
1628                is_integer: true
1629            })
1630        );
1631    }
1632
1633    #[test]
1634    fn dot_starts_number() {
1635        let tokens = CssTokenizer::collect(".5");
1636        assert_eq!(tokens.len(), 1);
1637        assert_eq!(
1638            tokens[0],
1639            Token::Number(Numeric {
1640                value: 0.5,
1641                is_integer: false
1642            })
1643        );
1644    }
1645
1646    // ── C-5 tests: §4.3.4 url( special case + §4.3.6 Url + §4.3.15 BadUrl ─
1647
1648    #[test]
1649    fn url_unquoted_simple() {
1650        // §4.3.6: url(foo) → Url("foo")
1651        let tokens = CssTokenizer::collect("url(foo)");
1652        assert_eq!(tokens.len(), 1);
1653        assert_eq!(tokens[0], Token::Url("foo".to_string()));
1654    }
1655
1656    #[test]
1657    fn url_unquoted_with_spaces() {
1658        // §4.3.6: leading/trailing whitespace consumed; url( foo ) → Url("foo")
1659        let tokens = CssTokenizer::collect("url( foo )");
1660        assert_eq!(tokens.len(), 1);
1661        assert_eq!(tokens[0], Token::Url("foo".to_string()));
1662    }
1663
1664    #[test]
1665    fn url_empty() {
1666        // §4.3.6: url() → Url("")
1667        let tokens = CssTokenizer::collect("url()");
1668        assert_eq!(tokens.len(), 1);
1669        assert_eq!(tokens[0], Token::Url(String::new()));
1670    }
1671
1672    #[test]
1673    fn url_eof_unterminated() {
1674        // §4.3.6 L1161-1164: EOF → parse error, return url-token with value so far
1675        let tokens = CssTokenizer::collect("url(foo");
1676        assert_eq!(tokens.len(), 1);
1677        assert_eq!(tokens[0], Token::Url("foo".to_string()));
1678    }
1679
1680    #[test]
1681    fn url_quoted_is_function() {
1682        // §4.3.4 L1058-1063: url("foo") → Function("url"); the body
1683        // `"foo")` remains in the stream → String("foo") + CloseParen.
1684        let tokens = CssTokenizer::collect("url(\"foo\")");
1685        assert_eq!(tokens.len(), 3);
1686        assert_eq!(tokens[0], Token::Function("url".to_string()));
1687        assert_eq!(tokens[1], Token::String("foo".to_string()));
1688        assert_eq!(tokens[2], Token::CloseParen);
1689    }
1690
1691    #[test]
1692    fn url_single_quoted_is_function() {
1693        // §4.3.4 L1058-1063: url('foo') → Function("url") + String + CloseParen
1694        let tokens = CssTokenizer::collect("url('foo')");
1695        assert_eq!(tokens.len(), 3);
1696        assert_eq!(tokens[0], Token::Function("url".to_string()));
1697        assert_eq!(tokens[1], Token::String("foo".to_string()));
1698        assert_eq!(tokens[2], Token::CloseParen);
1699    }
1700
1701    #[test]
1702    fn url_ws_then_quote_is_function() {
1703        // §4.3.4 L1058-1063: url( "foo") → whitespace+quote → Function.
1704        // L1056-1057 only collapses when *two* consecutive whitespace
1705        // follow `(`; here only one space precedes `"`, so it is NOT
1706        // consumed and becomes a Whitespace token.
1707        let tokens = CssTokenizer::collect("url( \"foo\")");
1708        assert_eq!(tokens.len(), 4);
1709        assert_eq!(tokens[0], Token::Function("url".to_string()));
1710        assert_eq!(tokens[1], Token::Whitespace);
1711        assert_eq!(tokens[2], Token::String("foo".to_string()));
1712        assert_eq!(tokens[3], Token::CloseParen);
1713    }
1714
1715    #[test]
1716    fn url_with_escape() {
1717        // §4.3.6 + §4.3.7: url(foo\29 bar) → \29 consumes trailing space,
1718        // decodes to `)`, then `bar` appended → Url("foo)bar")
1719        let tokens = CssTokenizer::collect("url(foo\\29 bar)");
1720        assert_eq!(tokens.len(), 1);
1721        assert_eq!(tokens[0], Token::Url("foo)bar".to_string()));
1722    }
1723
1724    #[test]
1725    fn url_bad_paren_in_unquoted() {
1726        // §4.3.6 L1177-1185: `(` in unquoted url → BadUrl
1727        let tokens = CssTokenizer::collect("url(foo(bar)");
1728        assert_eq!(tokens.len(), 1);
1729        assert_eq!(tokens[0], Token::BadUrl);
1730    }
1731
1732    #[test]
1733    fn url_bad_quote_in_unquoted() {
1734        // §4.3.6 L1177-1185: `"` in unquoted url → BadUrl
1735        let tokens = CssTokenizer::collect("url(foo\"bar)");
1736        assert_eq!(tokens.len(), 1);
1737        assert_eq!(tokens[0], Token::BadUrl);
1738    }
1739
1740    // ── C-6 tests: §4.3.1 unicode_ranges_allowed + §4.3.11 + §4.3.14 ──
1741
1742    #[test]
1743    fn unicode_range_disabled_by_default() {
1744        // §4.3.1 L782-783: default unicode_ranges_allowed=false.
1745        // `U+1234` → `U` ident-start → Ident("U") (`+` stops ident seq);
1746        // then `+1234` → starts_with_number (`+`+digit) → Number(1234).
1747        let tokens = CssTokenizer::collect("U+1234");
1748        assert_eq!(tokens.len(), 2);
1749        assert_eq!(tokens[0], Token::Ident("U".to_string()));
1750        assert_eq!(
1751            tokens[1],
1752            Token::Number(Numeric {
1753                value: 1234.0,
1754                is_integer: true
1755            })
1756        );
1757    }
1758
1759    #[test]
1760    fn unicode_range_simple() {
1761        // §4.3.14: U+1234 → UnicodeRange(0x1234, 0x1234)
1762        let mut tz = CssTokenizer::new("U+1234");
1763        tz.set_unicode_ranges_allowed(true);
1764        let t = tz.next_token().unwrap();
1765        assert_eq!(t, Token::UnicodeRange(Some(0x1234), Some(0x1234)));
1766    }
1767
1768    #[test]
1769    fn unicode_range_question_marks() {
1770        // §4.3.14 step 3: U+12?? → start=0x1200 (?→0), end=0x12FF (?→F)
1771        let mut tz = CssTokenizer::new("U+12??");
1772        tz.set_unicode_ranges_allowed(true);
1773        let t = tz.next_token().unwrap();
1774        assert_eq!(t, Token::UnicodeRange(Some(0x1200), Some(0x12FF)));
1775    }
1776
1777    #[test]
1778    fn unicode_range_range() {
1779        // §4.3.14 step 5: U+12-34FF → start=0x12, end=0x34FF
1780        let mut tz = CssTokenizer::new("U+12-34FF");
1781        tz.set_unicode_ranges_allowed(true);
1782        let t = tz.next_token().unwrap();
1783        assert_eq!(t, Token::UnicodeRange(Some(0x12), Some(0x34FF)));
1784    }
1785
1786    #[test]
1787    fn unicode_range_max_hex() {
1788        // §4.3.14: U+10FFFF → UnicodeRange(0x10FFFF, 0x10FFFF)
1789        let mut tz = CssTokenizer::new("U+10FFFF");
1790        tz.set_unicode_ranges_allowed(true);
1791        let t = tz.next_token().unwrap();
1792        assert_eq!(t, Token::UnicodeRange(Some(0x10FFFF), Some(0x10FFFF)));
1793    }
1794
1795    #[test]
1796    fn unicode_range_lowercase_u() {
1797        // §4.3.11: lowercase u also triggers; u+abc → UnicodeRange(0xABC, 0xABC)
1798        let mut tz = CssTokenizer::new("u+abc");
1799        tz.set_unicode_ranges_allowed(true);
1800        let t = tz.next_token().unwrap();
1801        assert_eq!(t, Token::UnicodeRange(Some(0xABC), Some(0xABC)));
1802    }
1803}