Skip to main content

muskitty_css_tokenizer/
impls.rs

1//! CSS Syntax tokenizer implementation.
2//!
3//! Implements the "Tokenizer Algorithms" of CSS Syntax Module Level 3 §4.3.
4//! The tokenizer is recursive-descent: [`CssTokenizer::next_token`] calls
5//! [`consume_a_token`](CssTokenizer::consume_a_token) (§4.3.1), which
6//! dispatches to sub-algorithms based on the current input code point.
7//!
8//! # Coverage
9//!
10//! All §4.3 sub-algorithms are implemented:
11//! - §4.3.1 Consume a token (full dispatch, incl. `unicode_ranges_allowed`)
12//! - §4.3.2 Consume comments
13//! - §4.3.3 Consume a numeric token
14//! - §4.3.4 Consume an ident-like token (incl. `url(` special case)
15//! - §4.3.5 Consume a string token
16//! - §4.3.6 Consume a url token
17//! - §4.3.7 Consume an escaped code point
18//! - §4.3.8 Check if two code points are a valid escape
19//! - §4.3.9 Check if three code points would start an ident sequence
20//! - §4.3.10 Check if three code points would start a number
21//! - §4.3.11 Check if three code points would start a unicode-range
22//! - §4.3.12 Consume an ident sequence
23//! - §4.3.13 Consume a number
24//! - §4.3.14 Consume a unicode-range token
25//! - §4.3.15 Consume the remnants of a bad url
26
27use crate::trait_def::Tokenizer;
28use crate::types::{HashType, Numeric, State, Token};
29
30/// The CSS tokenizer (§4.3).
31///
32/// Holds the preprocessed input stream (§5.3 normalization applied at
33/// construction), the current position, and EOF tracking. Sub-algorithms
34/// are implemented as methods on this struct so they share the input
35/// stream and position.
36pub struct CssTokenizer {
37    /// Input code points after §5.3 preprocessing (CR/LF/FF normalized to LF).
38    input: Vec<char>,
39    /// Current position in `input` (0-based index of the next code point
40    /// to consume). `pos == input.len()` means the stream is exhausted.
41    pos: usize,
42    /// Top-level state (§4.3.1 dispatch vs. EOF).
43    state: State,
44    /// Whether `<EOF-token>` has been emitted. After this is `true`,
45    /// [`next_token`](Tokenizer::next_token) returns `None`.
46    eof_emitted: bool,
47    /// §4.3.1 L782-783: `unicode_ranges_allowed` flag, default `false`.
48    /// When `true`, the `U+`/`u+` branch of §4.3.1 produces a
49    /// `<unicode-range-token>` (§4.3.14); otherwise `U`/`u` is tokenized
50    /// as an ident-like token.
51    unicode_ranges_allowed: bool,
52}
53
54impl CssTokenizer {
55    /// Construct a new tokenizer over `input`.
56    ///
57    /// Applies §5.3 input preprocessing: per the "input stream"
58    /// definition, U+000D CR followed by U+000A LF is collapsed to a
59    /// single LF, lone CR is replaced with LF, and U+000C FF is replaced
60    /// with LF. The tokenizer operates on this normalized stream.
61    pub fn new(input: &str) -> Self {
62        let input = preprocess_input(input);
63        Self {
64            input,
65            pos: 0,
66            state: State::Data,
67            eof_emitted: false,
68            unicode_ranges_allowed: false,
69        }
70    }
71
72    /// Peek at the code point at `offset` from the current position
73    /// without consuming it. Returns `None` if past end of input.
74    ///
75    /// Used by §4.3.8 / §4.3.9 / §4.3.10 / §4.3.11 "check if N code
76    /// points..." predicates, which must look ahead without advancing.
77    fn peek(&self, offset: usize) -> Option<char> {
78        self.input.get(self.pos + offset).copied()
79    }
80
81    /// Consume and return the next code point, advancing `pos`. Returns
82    /// `None` if at end of input.
83    ///
84    /// Implements the "consume the next input code point" primitive used
85    /// throughout §4.3.
86    fn consume(&mut self) -> Option<char> {
87        let c = self.input.get(self.pos).copied();
88        if c.is_some() {
89            self.pos += 1;
90        }
91        c
92    }
93
94    /// Re-consume the current code point: step `pos` back by one.
95    ///
96    /// Per §4.3.1 "reconsume the current input code point": the next
97    /// call to [`consume`](Self::consume) will return the same code point
98    /// that was just consumed. No-op if `pos == 0`.
99    fn reconsume(&mut self) {
100        if self.pos > 0 {
101            self.pos -= 1;
102        }
103    }
104
105    /// §4.3.1 Consume a token.
106    ///
107    /// This is the main entry point of the tokenizer. Dispatches based on
108    /// the current input code point to one of the sub-algorithms (§4.3.2
109    /// through §4.3.6) or emits a simple token directly. The optional
110    /// `unicode_ranges_allowed` flag (§4.3.1 L782-783, default `false`)
111    /// gates the `U+`/`u+` unicode-range branch.
112    fn consume_a_token(&mut self) -> Token {
113        // §4.3.2: If the next two input code points are U+002F SOLIDUS
114        // (`/`) and U+002A ASTERISK (`*`), consume comments and
115        // recursively consume the next token. We implement this as a
116        // loop: any leading run of `/* ... */` comments is skipped before
117        // producing the next real token.
118        loop {
119            let Some(c) = self.consume() else {
120                // §5.3: end of stream → <EOF-token>
121                self.eof_emitted = true;
122                self.state = State::Eof;
123                return Token::Eof;
124            };
125
126            // §4.3.2 comment dispatch: `/` followed by `*`.
127            if c == '/' && self.peek(0) == Some('*') {
128                self.consume(); // consume `*`
129                self.consume_comments_body();
130                // Loop back to consume the next token (which may itself
131                // be preceded by another comment).
132                continue;
133            }
134
135            return match c {
136                // §4.3.1: whitespace → <whitespace-token>
137                ' ' | '\t' | '\n' | '\r' | '\u{000C}' => {
138                    // Consume all consecutive whitespace.
139                    while let Some(next) = self.peek(0) {
140                        if matches!(next, ' ' | '\t' | '\n' | '\r' | '\u{000C}') {
141                            self.consume();
142                        } else {
143                            break;
144                        }
145                    }
146                    Token::Whitespace
147                }
148                // §4.3.1: `"` or `'` → consume a string token (§4.3.5)
149                '"' | '\'' => self.consume_a_string_token(c),
150                // §4.3.1: `#` → consume an ident-like token (§4.3.4) for hash
151                '#' => self.consume_a_hash_token(),
152                // §4.3.1: `(` → <(-token>
153                '(' => Token::OpenParen,
154                // §4.3.1: `)` → <)-token>
155                ')' => Token::CloseParen,
156                // §4.3.1: `+` or `-` → if would-start-a-number, consume a
157                // numeric token (§4.3.3); else <delim-token>.
158                // §4.3.1: `-` additionally: if next two are `-` and `>`
159                // (i.e. `-->`), consume them and return <CDC-token>.
160                // §4.3.1: `.` → if would-start-a-number, consume a numeric
161                // token; else <delim-token>.
162                '-' => {
163                    if self.starts_with_number(c) {
164                        self.reconsume();
165                        self.consume_a_numeric_token()
166                    } else if self.peek(0) == Some('-') && self.peek(1) == Some('>') {
167                        // §4.3.1: `-->` → <CDC-token>
168                        self.consume(); // consume second `-`
169                        self.consume(); // consume `>`
170                        Token::Cdc
171                    } else if self.would_start_ident_sequence_at(0) {
172                        // §4.3.1: `-` starting an ident sequence → reconsume
173                        // and consume an ident-like token.
174                        self.reconsume();
175                        self.consume_an_ident_like_token()
176                    } else {
177                        Token::Delim(c)
178                    }
179                }
180                '+' | '.' => {
181                    if self.starts_with_number(c) {
182                        self.reconsume();
183                        self.consume_a_numeric_token()
184                    } else {
185                        Token::Delim(c)
186                    }
187                }
188                // §4.3.1: `<` → if next three are `!`, `-`, `-` (i.e.
189                // `<!--`), consume them and return <CDO-token>. Otherwise
190                // emit <delim-token>.
191                '<' => {
192                    if self.peek(0) == Some('!')
193                        && self.peek(1) == Some('-')
194                        && self.peek(2) == Some('-')
195                    {
196                        // Consume `!`, `-`, `-`.
197                        self.consume();
198                        self.consume();
199                        self.consume();
200                        Token::Cdo
201                    } else {
202                        Token::Delim(c)
203                    }
204                }
205                // §4.3.1: `@` → if next would-start-an-ident-sequence,
206                // consume an ident-like token (§4.3.4) for at-keyword.
207                '@' => self.consume_an_at_keyword_token(),
208                // §4.3.1: `[` → <[-token>
209                '[' => Token::OpenBracket,
210                // §4.3.1: `\` → if would-start-an-escape (§4.3.8), reconsume
211                // and consume an ident-like token (§4.3.4); else parse error,
212                // <delim-token>.
213                '\\' => self.consume_ident_or_delim(),
214                // §4.3.1: `]` → <]-token>
215                ']' => Token::CloseBracket,
216                // §4.3.1: `{` → <{-token>
217                '{' => Token::OpenBrace,
218                // §4.3.1: `}` → <}-token>
219                '}' => Token::CloseBrace,
220                // §4.3.1: digit → reconsume and consume a numeric token (§4.3.3)
221                '0'..='9' => {
222                    self.reconsume();
223                    self.consume_a_numeric_token()
224                }
225                // §4.3.1: `U+` or `u+` → if would-start-a-unicode-range (§4.3.11),
226                // consume a unicode-range token (§4.3.14).
227                'u' | 'U' => self.consume_u_or_unicode_range(),
228                // §4.3.1: ident-start code point → reconsume and consume an
229                // ident-like token (§4.3.4)
230                _ if is_ident_start_code_point(c) => {
231                    self.reconsume();
232                    self.consume_an_ident_like_token()
233                }
234                // §4.3.1: `:` → <colon-token>
235                ':' => Token::Colon,
236                // §4.3.1: `;` → <semicolon-token>
237                ';' => Token::Semicolon,
238                // §4.3.1: `,` → <comma-token>
239                ',' => Token::Comma,
240                // §4.3.1: anything else → <delim-token>
241                _ => Token::Delim(c),
242            };
243        }
244    }
245
246    /// §4.3.2 Consume comments — body only.
247    ///
248    /// Precondition: the opening `/*` has already been consumed. Consume
249    /// code points until the closing `*/` or EOF. Per §4.3.2, an
250    /// unterminated comment (EOF before `*/`) is a parse error; we
251    /// silently consume to EOF in that case (no error reporting yet).
252    fn consume_comments_body(&mut self) {
253        loop {
254            match self.consume() {
255                None => return, // EOF in comment: parse error (unreported)
256                Some('*') => {
257                    if self.peek(0) == Some('/') {
258                        self.consume(); // consume `/`
259                        return;
260                    }
261                }
262                Some(_) => {}
263            }
264        }
265    }
266
267    // ── Sub-algorithm stubs (implemented in subsequent commits) ───────
268
269    /// §4.3.5 (L1081-1128) Consume a string token.
270    ///
271    /// `quote` is the ending code point (the opening quote `"` or `'`).
272    /// Returns a `string-token` (normal or EOF-terminated) or a
273    /// `bad-string-token` (unescaped newline).
274    ///
275    /// Per §4.3.5:
276    /// - ending code point (L1097-1099) → return string-token.
277    /// - EOF (L1101-1104) → parse error, return string-token.
278    /// - newline (L1106-1110) → parse error, reconsume, return bad-string.
279    /// - `\` (L1112-1124):
280    ///   - next EOF → do nothing (loop; next consume hits EOF → string).
281    ///   - next newline → consume it (line continuation).
282    ///   - otherwise (valid escape) → consume an escaped code point, append.
283    /// - anything else (L1126-1128) → append.
284    fn consume_a_string_token(&mut self, quote: char) -> Token {
285        let mut value = String::new();
286        loop {
287            match self.consume() {
288                None => {
289                    // §4.3.5 L1101-1104: EOF → parse error, return string-token.
290                    return Token::String(value);
291                }
292                Some(c) if c == quote => {
293                    // §4.3.5 L1097-1099: ending code point → return string-token.
294                    return Token::String(value);
295                }
296                Some('\n') => {
297                    // §4.3.5 L1106-1110: newline → parse error, reconsume, return bad-string.
298                    self.reconsume();
299                    return Token::BadString;
300                }
301                Some('\\') => {
302                    // §4.3.5 L1112-1124: escape handling.
303                    match self.peek(0) {
304                        None => {
305                            // §4.3.5 L1114-1115: next is EOF, do nothing.
306                            // Loop continues; next consume() returns None → String.
307                        }
308                        Some('\n') => {
309                            // §4.3.5 L1117-1119: next is newline, consume it (line continuation).
310                            self.consume();
311                        }
312                        Some(_) => {
313                            // §4.3.5 L1121-1124: valid escape, consume escaped code point.
314                            let escaped = self.consume_an_escaped_code_point();
315                            value.push(escaped);
316                        }
317                    }
318                }
319                Some(c) => {
320                    // §4.3.5 L1126-1128: anything else → append.
321                    value.push(c);
322                }
323            }
324        }
325    }
326
327    /// §4.3.1 (L801-826) `#` branch: Consume a hash token.
328    ///
329    /// Precondition: `#` has been consumed. Per §4.3.1 L801-826:
330    /// - If the next code point is an ident code point OR the next two
331    ///   are a valid escape, then create a `hash-token` (type "id" if
332    ///   would-start-ident-sequence, else "unrestricted"), consume an
333    ///   ident sequence for the value, and return it.
334    /// - Otherwise, return a `delim-token` with value `#`.
335    fn consume_a_hash_token(&mut self) -> Token {
336        // §4.3.1 L803-805: next is ident code point OR next two are valid escape.
337        let next_is_ident = self.peek(0).is_some_and(is_ident_code_point);
338        let next_is_valid_escape = self.is_valid_escape_at(0);
339        if !next_is_ident && !next_is_valid_escape {
340            // §4.3.1 L824-826: return delim-token with value `#`.
341            return Token::Delim('#');
342        }
343        let hash_type = if self.would_start_ident_sequence_at(0) {
344            HashType::Id
345        } else {
346            HashType::Unrestricted
347        };
348        let name = self.consume_an_ident_sequence();
349        Token::Hash(name, hash_type)
350    }
351
352    /// §4.3.3 (L1011-1042) Consume a numeric token.
353    ///
354    /// Returns a `number-token`, `percentage-token`, or `dimension-token`.
355    /// Per §4.3.3:
356    /// 1. Consume a number (§4.3.13).
357    /// 2. If next 3 would start an ident sequence → dimension-token (consume
358    ///    an ident sequence for the unit).
359    /// 3. Else if next is `%` → percentage-token (consume `%`).
360    /// 4. Else → number-token.
361    fn consume_a_numeric_token(&mut self) -> Token {
362        let (value, is_integer) = self.consume_a_number();
363        let numeric = Numeric { value, is_integer };
364        // §4.3.3 L1019-1029: dimension
365        if self.would_start_ident_sequence_at(0) {
366            let unit = self.consume_an_ident_sequence();
367            return Token::Dimension(numeric, unit);
368        }
369        // §4.3.3 L1032-1036: percentage
370        if self.peek(0) == Some('%') {
371            self.consume();
372            return Token::Percentage(numeric);
373        }
374        // §4.3.3 L1040-1042: number
375        Token::Number(numeric)
376    }
377
378    /// §4.3.13 (L1415-1483) Consume a number.
379    ///
380    /// Returns `(value, is_integer)` where `is_integer` is true when the
381    /// source representation had no fractional part and no exponent (type
382    /// "integer"), false when it had either (type "number"). The sign is
383    /// included in `value`.
384    ///
385    /// Per §4.3.13:
386    /// 1. type = "integer"; number_part = ""; exponent_part = "".
387    /// 2. If next is `+`/`-`, consume, append to number_part.
388    /// 3. While next is digit, consume, append to number_part.
389    /// 4. If next 2 are `.` + digit: consume `.`, consume digits; type = "number".
390    /// 5. If next 2/3 are `e`/`E` + optional `+`/`-` + digit: consume `e`/`E`,
391    ///    optional sign, digits into exponent_part; type = "number".
392    /// 6. value = parse number_part; if exponent_part non-empty, value *= 10^exp.
393    /// 7. Return value, type, sign.
394    fn consume_a_number(&mut self) -> (f64, bool) {
395        let mut type_is_number = false;
396        let mut number_part = String::new();
397        let mut exponent_part = String::new();
398
399        // §4.3.13 L1436-1440: sign
400        if self.peek(0) == Some('+') || self.peek(0) == Some('-') {
401            number_part.push(self.consume().unwrap());
402        }
403
404        // §4.3.13 L1442-1444: integer digits
405        while let Some(d) = self.peek(0) {
406            if is_digit(d) {
407                number_part.push(d);
408                self.consume();
409            } else {
410                break;
411            }
412        }
413
414        // §4.3.13 L1446-1454: fraction part
415        if self.peek(0) == Some('.') && matches!(self.peek(1), Some(d) if is_digit(d)) {
416            number_part.push(self.consume().unwrap()); // consume `.`
417            type_is_number = true;
418            while let Some(d) = self.peek(0) {
419                if is_digit(d) {
420                    number_part.push(d);
421                    self.consume();
422                } else {
423                    break;
424                }
425            }
426        }
427
428        // §4.3.13 L1457-1469: exponent part
429        if matches!(self.peek(0), Some('e') | Some('E')) {
430            let p1 = self.peek(1);
431            let p2 = self.peek(2);
432            let valid_exp = match p1 {
433                Some('+') | Some('-') => matches!(p2, Some(d) if is_digit(d)),
434                Some(d) if is_digit(d) => true,
435                _ => false,
436            };
437            if valid_exp {
438                self.consume(); // consume `e`/`E`
439                if self.peek(0) == Some('+') || self.peek(0) == Some('-') {
440                    exponent_part.push(self.consume().unwrap());
441                }
442                while let Some(d) = self.peek(0) {
443                    if is_digit(d) {
444                        exponent_part.push(d);
445                        self.consume();
446                    } else {
447                        break;
448                    }
449                }
450                type_is_number = true;
451            }
452        }
453
454        // §4.3.13 L1472-1480: compute value
455        let mut value: f64 = number_part.parse().unwrap_or(0.0);
456        if !exponent_part.is_empty() {
457            let exp: i32 = exponent_part.parse().unwrap_or(0);
458            value *= 10.0f64.powi(exp);
459        }
460
461        (value, !type_is_number)
462    }
463
464    /// §4.3.4 (L1045-1078) Consume an ident-like token.
465    ///
466    /// Precondition: the current position is at the start of an ident
467    /// sequence (the caller has not consumed anything; or has reconsumed
468    /// the first code point). Returns an `ident-token`, `function-token`,
469    /// `url-token`, or `bad-url-token`.
470    ///
471    /// Per §4.3.4:
472    /// 1. Consume an ident sequence (§4.3.12), yielding `name`.
473    /// 2. If `name` is ASCII-case-insensitive "url" and next is `(`:
474    ///    consume `(`, collapse leading whitespace pairs, then if the
475    ///    next 1-2 code points are a quote (or whitespace+quote) return
476    ///    a `function-token`; otherwise consume a url token (§4.3.6).
477    /// 3. Else if next is `(`: consume it, return `function-token`.
478    /// 4. Else return `ident-token`.
479    fn consume_an_ident_like_token(&mut self) -> Token {
480        let name = self.consume_an_ident_sequence();
481        // §4.3.4 L1053-1066: url( special case
482        if name.eq_ignore_ascii_case("url") && self.peek(0) == Some('(') {
483            self.consume(); // consume `(`
484                            // §4.3.4 L1056-1057: while next two are whitespace, consume one
485            while self.peek(0).is_some_and(is_whitespace) && self.peek(1).is_some_and(is_whitespace)
486            {
487                self.consume();
488            }
489            // §4.3.4 L1058-1063: if next 1-2 are " / ' / ws+" / ws+' → Function
490            let p0 = self.peek(0);
491            let p1 = self.peek(1);
492            let is_quote_case = matches!(p0, Some('"') | Some('\''))
493                || (p0.is_some_and(is_whitespace) && matches!(p1, Some('"') | Some('\'')));
494            if is_quote_case {
495                return Token::Function(name);
496            }
497            // §4.3.4 L1064-1066: otherwise consume a url token
498            return self.consume_a_url_token();
499        }
500        // §4.3.4 L1068-1073: ordinary function
501        if self.peek(0) == Some('(') {
502            self.consume();
503            return Token::Function(name);
504        }
505        // §4.3.4 L1075-1078: ident
506        Token::Ident(name)
507    }
508
509    /// §4.3.4 (at-keyword branch) Consume an at-keyword token.
510    ///
511    /// Precondition: `@` has been consumed. Per §4.3.1, if the next code
512    /// points would start an ident sequence, consume it and return an
513    /// `<at-keyword-token>`; otherwise return a `<delim-token>` with `@`.
514    fn consume_an_at_keyword_token(&mut self) -> Token {
515        if self.would_start_ident_sequence_at(0) {
516            let name = self.consume_an_ident_sequence();
517            Token::AtKeyword(name)
518        } else {
519            Token::Delim('@')
520        }
521    }
522
523    /// §4.3.1 (backslash branch): if would-start-an-escape, consume an
524    /// ident-like token; else emit `<delim-token>`.
525    fn consume_ident_or_delim(&mut self) -> Token {
526        if self.is_valid_escape_next() {
527            self.reconsume();
528            self.consume_an_ident_like_token()
529        } else {
530            Token::Delim('\\')
531        }
532    }
533
534    /// §4.3.6 (L1132-1203) Consume a url token.
535    ///
536    /// Precondition: `url(` has been consumed by the caller (§4.3.4
537    /// L1053-1066). This algorithm consumes the "unquoted" url body and
538    /// returns a `url-token` or `bad-url-token`. A quoted value like
539    /// `url("foo")` is handled by §4.3.4 as a `function-token` and never
540    /// reaches here.
541    ///
542    /// Per §4.3.6:
543    /// - `)` → return url-token.
544    /// - EOF → parse error, return url-token.
545    /// - whitespace → consume trailing whitespace; if next is `)` or EOF
546    ///   return url-token, else bad-url.
547    /// - `"` / `'` / `(` / non-printable → parse error, bad-url.
548    /// - `\` → if valid escape, consume escaped and append; else bad-url.
549    /// - anything else → append.
550    fn consume_a_url_token(&mut self) -> Token {
551        let mut value = String::new();
552        // §4.3.6 L1151: consume as much whitespace as possible
553        while self.peek(0).is_some_and(is_whitespace) {
554            self.consume();
555        }
556        loop {
557            match self.consume() {
558                // §4.3.6 L1157-1159: `)` → return url-token
559                Some(')') => return Token::Url(value),
560                // §4.3.6 L1161-1164: EOF → parse error, return url-token
561                None => return Token::Url(value),
562                // §4.3.6 L1166-1175: whitespace → consume trailing ws, then check
563                Some(c) if is_whitespace(c) => {
564                    while self.peek(0).is_some_and(is_whitespace) {
565                        self.consume();
566                    }
567                    match self.peek(0) {
568                        Some(')') => {
569                            self.consume();
570                            return Token::Url(value);
571                        }
572                        None => return Token::Url(value), // parse error
573                        _ => {
574                            self.consume_the_remnants_of_a_bad_url();
575                            return Token::BadUrl;
576                        }
577                    }
578                }
579                // §4.3.6 L1177-1185: " / ' / ( / non-printable → bad url
580                Some(c) if c == '"' || c == '\'' || c == '(' || is_non_printable(c) => {
581                    self.consume_the_remnants_of_a_bad_url();
582                    return Token::BadUrl;
583                }
584                // §4.3.6 L1187-1197: `\`
585                Some('\\') => {
586                    if self.is_valid_escape_next() {
587                        let escaped = self.consume_an_escaped_code_point();
588                        value.push(escaped);
589                    } else {
590                        self.consume_the_remnants_of_a_bad_url();
591                        return Token::BadUrl;
592                    }
593                }
594                // §4.3.6 L1199-1202: anything else → append
595                Some(c) => value.push(c),
596            }
597        }
598    }
599
600    /// §4.3.15 (L1551-1577) Consume the remnants of a bad url.
601    ///
602    /// Consumes code points until `)` or EOF, allowing an escaped `)`
603    /// (`\)`) to be consumed without ending the bad-url. Per §4.3.15:
604    /// - `)` or EOF → return.
605    /// - valid escape (stream starts with `\` + non-newline non-EOF) →
606    ///   consume an escaped code point.
607    /// - anything else → do nothing.
608    fn consume_the_remnants_of_a_bad_url(&mut self) {
609        loop {
610            match self.consume() {
611                // §4.3.15 L1563-1566: `)` or EOF → return
612                Some(')') | None => return,
613                // §4.3.15 L1568-1572: valid escape → consume escaped code point
614                Some('\\') if self.is_valid_escape_next() => {
615                    let _ = self.consume_an_escaped_code_point();
616                }
617                // §4.3.15 L1574-1576: anything else → do nothing
618                Some(_) => {}
619            }
620        }
621    }
622
623    /// §4.3.1 (L960-972) U/u branch: if `unicode_ranges_allowed` is true
624    /// and the stream would start a unicode-range (§4.3.11), consume a
625    /// unicode-range token (§4.3.14); otherwise consume an ident-like token.
626    ///
627    /// Precondition: `u`/`U` has been consumed, `pos` points just past it.
628    /// This method reconsumes back to `u` before dispatching.
629    fn consume_u_or_unicode_range(&mut self) -> Token {
630        self.reconsume(); // pos back to `u`/`U`
631                          // §4.3.1 L963-967: unicode_ranges_allowed && would-start-unicode-range
632        if self.unicode_ranges_allowed && self.would_start_unicode_range_at(0) {
633            return self.consume_a_unicode_range_token();
634        }
635        // §4.3.1 L969-972: otherwise ident-like
636        self.consume_an_ident_like_token()
637    }
638
639    /// §4.3.11 (L1356-1378) Check if three code points would start a
640    /// unicode-range, examining code points starting at `offset` from the
641    /// current position.
642    ///
643    /// Per §4.3.11: true iff first is `U`/`u`, second is `+`, third is `?`
644    /// or a hex digit.
645    fn would_start_unicode_range_at(&self, offset: usize) -> bool {
646        let first = match self.peek(offset) {
647            Some(c) => c,
648            None => return false,
649        };
650        if !matches!(first, 'U' | 'u') {
651            return false;
652        }
653        if self.peek(offset + 1) != Some('+') {
654            return false;
655        }
656        match self.peek(offset + 2) {
657            Some('?') => true,
658            Some(c) if is_hex_digit(c) => true,
659            _ => false,
660        }
661    }
662
663    /// §4.3.14 (L1487-1548) Consume a unicode-range token.
664    ///
665    /// Precondition: the stream starts with `u`/`U` + `+` + (`?` or hex
666    /// digit) (i.e. [`would_start_unicode_range_at`](Self::would_start_unicode_range_at)
667    /// returns true at the current position).
668    ///
669    /// Returns `UnicodeRange(Some(start), Some(end))`.
670    ///
671    /// Per §4.3.14:
672    /// 1. Consume and discard `u`/`U` and `+`.
673    /// 2. Consume up to 6 hex digits; if fewer than 6, consume `?` up to a
674    ///    total of 6 → `first_segment`.
675    /// 3. If `first_segment` contains `?`: start = `?`→`0`, end = `?`→`F`.
676    /// 4. Else start = hex value of `first_segment`.
677    /// 5. If next two are `-` + hex digit: consume `-`, consume up to 6 hex
678    ///    → end.
679    /// 6. Else end = start.
680    fn consume_a_unicode_range_token(&mut self) -> Token {
681        // §4.3.14 step 1: consume and discard `u`/`U` + `+`
682        self.consume(); // `u`/`U`
683        self.consume(); // `+`
684
685        // §4.3.14 step 2: consume up to 6 hex digits
686        let mut first_segment = String::new();
687        while first_segment.len() < 6 {
688            match self.peek(0) {
689                Some(c) if is_hex_digit(c) => {
690                    first_segment.push(c);
691                    self.consume();
692                }
693                _ => break,
694            }
695        }
696        // then consume `?` up to a total of 6
697        while first_segment.len() < 6 {
698            match self.peek(0) {
699                Some('?') => {
700                    first_segment.push('?');
701                    self.consume();
702                }
703                _ => break,
704            }
705        }
706
707        // §4.3.14 step 3: if first_segment contains `?`
708        if first_segment.contains('?') {
709            let start_str: String = first_segment
710                .chars()
711                .map(|c| if c == '?' { '0' } else { c })
712                .collect();
713            let start = u32::from_str_radix(&start_str, 16).unwrap_or(0);
714            let end_str: String = first_segment
715                .chars()
716                .map(|c| if c == '?' { 'F' } else { c })
717                .collect();
718            let end = u32::from_str_radix(&end_str, 16).unwrap_or(0);
719            return Token::UnicodeRange(Some(start), Some(end));
720        }
721
722        // §4.3.14 step 4: first_segment as hex → start
723        let start = u32::from_str_radix(&first_segment, 16).unwrap_or(0);
724
725        // §4.3.14 step 5: if next two are `-` + hex digit
726        if self.peek(0) == Some('-') && self.peek(1).is_some_and(is_hex_digit) {
727            self.consume(); // consume `-`
728            let mut end_segment = String::new();
729            while end_segment.len() < 6 {
730                match self.peek(0) {
731                    Some(c) if is_hex_digit(c) => {
732                        end_segment.push(c);
733                        self.consume();
734                    }
735                    _ => break,
736                }
737            }
738            let end = u32::from_str_radix(&end_segment, 16).unwrap_or(0);
739            return Token::UnicodeRange(Some(start), Some(end));
740        }
741
742        // §4.3.14 step 6: otherwise start == end
743        Token::UnicodeRange(Some(start), Some(start))
744    }
745
746    // ── §4.3.12 Consume an ident sequence ────────────────────────────
747
748    /// §4.3.12 Consume an ident sequence.
749    ///
750    /// Repeatedly consume code points as long as they form an ident
751    /// sequence: ident code points, valid escapes (consumed via
752    /// §4.3.7), or `\`-`-` (for `--`-prefixed names). Returns the
753    /// decoded name with escapes resolved.
754    ///
755    /// Per §4.3.12, the loop terminates when:
756    /// - EOF is reached, or
757    /// - the next code point is not an ident code point, not a valid
758    ///   escape, and not part of a `\`-`-` continuation.
759    fn consume_an_ident_sequence(&mut self) -> String {
760        let mut result = String::new();
761        loop {
762            let Some(c) = self.consume() else {
763                return result;
764            };
765            if is_ident_code_point(c) {
766                result.push(c);
767            } else if c == '\\' && self.is_valid_escape_next() {
768                // §4.3.12: valid escape → consume the escaped code point
769                // and append it.
770                let escaped = self.consume_an_escaped_code_point();
771                result.push(escaped);
772            } else {
773                // Not part of the ident sequence: reconsume and stop.
774                self.reconsume();
775                return result;
776            }
777        }
778    }
779
780    // ── §4.3.7 Consume an escaped code point ────────────────────────
781
782    /// §4.3.7 Consume an escaped code point.
783    ///
784    /// Precondition: the `\` has been consumed. Returns the decoded code
785    /// point.
786    ///
787    /// Per §4.3.7 (L1207-1240):
788    /// - EOF (L1233-1236) → parse error, return U+FFFD REPLACEMENT CHARACTER.
789    /// - newline → parse error, reconsume (invalid escape; the caller
790    ///   should have checked §4.3.8 first, but we handle defensively).
791    /// - hex digit(s) (L1219-1231) → consume 1-6 hex digits, optionally
792    ///   followed by whitespace, and return the code point with that
793    ///   value. U+0000 NULL or out-of-range → replacement character U+FFFD.
794    /// - anything else (L1238-1240) → return the consumed code point.
795    fn consume_an_escaped_code_point(&mut self) -> char {
796        let Some(c) = self.consume() else {
797            // §4.3.7 L1233-1236: EOF → parse error, return U+FFFD.
798            return '\u{FFFD}';
799        };
800        if is_hex_digit(c) {
801            // §4.3.7 step 4: consume up to 5 more hex digits (total ≤ 6).
802            let mut hex = String::new();
803            hex.push(c);
804            while hex.len() < 6 {
805                match self.peek(0) {
806                    Some(next) if is_hex_digit(next) => {
807                        hex.push(next);
808                        self.consume();
809                    }
810                    _ => break,
811                }
812            }
813            // §4.3.7 step 4: consume one trailing whitespace.
814            if self.peek(0) == Some(' ') || self.peek(0) == Some('\t') || self.peek(0) == Some('\n')
815            {
816                self.consume();
817            }
818            // Parse the hex value.
819            let value = u32::from_str_radix(&hex, 16).unwrap_or(0);
820            // §4.3.7 step 4: 0 or > 0x10FFFF → U+FFFD. Also surrogates
821            // (0xD800–0xDFFF) → U+FFFD.
822            if value == 0 || value > 0x10FFFF || (0xD800..=0xDFFF).contains(&value) {
823                '\u{FFFD}'
824            } else {
825                char::from_u32(value).unwrap_or('\u{FFFD}')
826            }
827        } else if c == '\n' {
828            // §4.3.7 step 3: newline after `\` is a parse error. This
829            // should not happen if the caller checked §4.3.8, but handle
830            // defensively by returning `\` and reconsuming the newline.
831            self.reconsume();
832            '\\'
833        } else {
834            // §4.3.7 step 5: any other code point → return it.
835            c
836        }
837    }
838
839    // ── §4.3.8 / §4.3.9 Predicates ──────────────────────────────────
840
841    /// §4.3.8 Check if two code points are a valid escape.
842    ///
843    /// Precondition: the `\` has been consumed. Returns true if the next
844    /// code point forms a valid escape with the consumed `\` (i.e. the
845    /// next code point is not a newline).
846    ///
847    /// §4.3.8: EOF as the second code point returns **true** — only a
848    /// newline invalidates the escape. The escaped-EOF path then hits
849    /// §4.3.7's EOF branch and yields U+FFFD (the WPT
850    /// css/css-syntax/escaped-eof.html semantics).
851    fn is_valid_escape_next(&self) -> bool {
852        match self.peek(0) {
853            None => true,
854            Some('\n') => false,
855            Some(_) => true,
856        }
857    }
858
859    /// §4.3.9 Check if three code points would start an ident sequence,
860    /// examining code points starting at `offset` from the current
861    /// position.
862    ///
863    /// Per §4.3.9:
864    /// - U+002D HYPHEN-MINUS (`-`):
865    ///   - If the next is also `-`, true (start of `--...`).
866    ///   - If the next is an ident-start code point (or a valid escape),
867    ///     true.
868    ///   - Else false.
869    /// - U+005C REVERSE SOLIDUS (`\`): true if valid escape.
870    /// - ident-start code point: true.
871    /// - Else false.
872    fn would_start_ident_sequence_at(&self, offset: usize) -> bool {
873        let first = match self.peek(offset) {
874            Some(c) => c,
875            None => return false,
876        };
877        match first {
878            '-' => {
879                // §4.3.9: if next is `-`, true.
880                match self.peek(offset + 1) {
881                    Some('-') => true,
882                    Some(next) if is_ident_start_code_point(next) => true,
883                    Some('\\') => {
884                        // §4.3.9: if `\` and the following forms a valid escape.
885                        self.is_valid_escape_at(offset + 1)
886                    }
887                    _ => false,
888                }
889            }
890            '\\' => self.is_valid_escape_at(offset),
891            _ if is_ident_start_code_point(first) => true,
892            _ => false,
893        }
894    }
895
896    /// §4.3.8 Check if the code point at `offset` starts a valid escape
897    /// (i.e. `offset` points at `\` and `offset+1` is not newline/EOF).
898    fn is_valid_escape_at(&self, offset: usize) -> bool {
899        if self.peek(offset) != Some('\\') {
900            return false;
901        }
902        match self.peek(offset + 1) {
903            // §4.3.8: EOF as the second code point is a valid escape.
904            None => true,
905            Some('\n') => false,
906            Some(_) => true,
907        }
908    }
909
910    // ── Predicates (stubs returning false; implemented in C-4/C-6) ──
911
912    /// §4.3.10 (L1307-1352) Check if three code points would start a number.
913    ///
914    /// `first` is the code point already consumed (the caller passes it
915    /// so the predicate can examine the next two without reconsume). The
916    /// three code points examined are `first`, `peek(0)`, `peek(1)`.
917    ///
918    /// Per §4.3.10:
919    /// - `+`/`-`: second is digit → true; second is `.` and third is digit
920    ///   → true; else false.
921    /// - `.`: second is digit → true; else false.
922    /// - digit: true.
923    /// - anything else: false.
924    fn starts_with_number(&self, first: char) -> bool {
925        match first {
926            '+' | '-' => match self.peek(0) {
927                Some(d) if is_digit(d) => true,
928                Some('.') => matches!(self.peek(1), Some(d) if is_digit(d)),
929                _ => false,
930            },
931            '.' => matches!(self.peek(0), Some(d) if is_digit(d)),
932            d if is_digit(d) => true,
933            _ => false,
934        }
935    }
936}
937
938// ── Free functions (§4.2 code point classes) ──────────────────────────
939
940/// §4.2 Whether `c` is an ident-start code point.
941///
942/// An ident-start code point is an ASCII letter, a non-ASCII ident code
943/// point, or U+005F LOW LINE (`_`).
944fn is_ident_start_code_point(c: char) -> bool {
945    matches!(c, 'A'..='Z' | 'a'..='z' | '_') || is_non_ascii_ident_code_point(c)
946}
947
948/// §4.2 Whether `c` is an ident code point (allowed after the first).
949///
950/// An ident code point is an ident-start code point, or an ASCII digit,
951/// or U+002D HYPHEN-MINUS (`-`).
952fn is_ident_code_point(c: char) -> bool {
953    matches!(c, '0'..='9' | '-') || is_ident_start_code_point(c)
954}
955
956/// §4.2 Whether `c` is a non-ASCII ident code point.
957///
958/// The spec defines these as an explicit allowlist of ranges (aligned
959/// with the code points allowed in HTML custom element names) — *not*
960/// "any code point ≥ U+0080". Notably U+0080–U+00B6 and U+00B8–U+00BF
961/// are excluded (WPT css/css-syntax/non-ascii-codepoints.html).
962fn is_non_ascii_ident_code_point(c: char) -> bool {
963    matches!(c,
964        '\u{00B7}'
965        | '\u{00C0}'..='\u{00D6}'
966        | '\u{00D8}'..='\u{00F6}'
967        | '\u{00F8}'..='\u{037D}'
968        | '\u{037F}'..='\u{1FFF}'
969        | '\u{200C}' | '\u{200D}'
970        | '\u{203F}' | '\u{2040}'
971        | '\u{2070}'..='\u{218F}'
972        | '\u{2C00}'..='\u{2FEF}'
973        | '\u{3001}'..='\u{D7FF}'
974        | '\u{F900}'..='\u{FDCF}'
975        | '\u{FDF0}'..='\u{FFFD}'
976        | '\u{10000}'..=char::MAX)
977}
978
979/// §4.2 Whether `c` is a digit (ASCII 0-9).
980fn is_digit(c: char) -> bool {
981    c.is_ascii_digit()
982}
983
984/// §4.2 Whether `c` is a hex digit (ASCII 0-9, A-F, a-f).
985fn is_hex_digit(c: char) -> bool {
986    c.is_ascii_hexdigit()
987}
988
989/// §4.2 Whether `c` is a whitespace code point.
990///
991/// Per §4.2: U+0009 TAB, U+000A LF, U+000C FF, U+000D CR, U+0020 SPACE.
992/// (After §5.3 preprocessing, U+000C FF and lone U+000D CR have been
993/// normalized to U+000A LF, but the helper follows the full §4.2
994/// definition for correctness when called on arbitrary input.)
995fn is_whitespace(c: char) -> bool {
996    matches!(c, '\t' | '\n' | '\u{000C}' | '\r' | ' ')
997}
998
999/// §4.2 Whether `c` is a non-printable code point.
1000///
1001/// Per §4.2: U+0000-U+0008, U+000B, U+000E-U+001F, U+007F-U+009F.
1002fn is_non_printable(c: char) -> bool {
1003    matches!(
1004        c,
1005        '\u{0000}'..='\u{0008}' | '\u{000B}' | '\u{000E}'..='\u{001F}' | '\u{007F}'..='\u{009F}'
1006    )
1007}
1008
1009/// §5.3 input preprocessing.
1010///
1011/// Per the "input stream" definition in §5.3:
1012/// - U+000D CR followed by U+000A LF → single U+000A LF
1013/// - lone U+000D CR → U+000A LF
1014/// - U+000C FF → U+000A LF
1015///
1016/// The tokenizer operates on this normalized stream.
1017fn preprocess_input(s: &str) -> Vec<char> {
1018    let mut out = Vec::with_capacity(s.len());
1019    let mut chars = s.chars().peekable();
1020    while let Some(c) = chars.next() {
1021        match c {
1022            '\r' => {
1023                if chars.peek() == Some(&'\n') {
1024                    chars.next();
1025                }
1026                out.push('\n');
1027            }
1028            '\u{000C}' => out.push('\n'),
1029            // §5.3: NULL is replaced with U+FFFD during preprocessing.
1030            '\u{0000}' => out.push('\u{FFFD}'),
1031            _ => out.push(c),
1032        }
1033    }
1034    out
1035}
1036
1037impl Tokenizer for CssTokenizer {
1038    fn next_token(&mut self) -> Option<Token> {
1039        if self.eof_emitted {
1040            return None;
1041        }
1042        Some(self.consume_a_token())
1043    }
1044
1045    fn state(&self) -> State {
1046        self.state
1047    }
1048
1049    fn set_state(&mut self, state: State) {
1050        self.state = state;
1051        if state == State::Data {
1052            // Resetting to Data after EOF re-enables the iterator; useful
1053            // for `reset()` callers that want to re-tokenize.
1054            self.eof_emitted = false;
1055        }
1056    }
1057
1058    fn reset(&mut self) {
1059        self.pos = 0;
1060        self.state = State::Data;
1061        self.eof_emitted = false;
1062    }
1063
1064    fn set_unicode_ranges_allowed(&mut self, allowed: bool) {
1065        self.unicode_ranges_allowed = allowed;
1066    }
1067
1068    fn position(&self) -> usize {
1069        self.pos
1070    }
1071}
1072
1073// ── Test helpers exposed for unit tests in this file ──────────────────
1074
1075#[cfg(test)]
1076impl CssTokenizer {
1077    /// Tokenize the entire input and return all tokens except the
1078    /// trailing `<EOF-token>`. Test-only convenience.
1079    fn collect(input: &str) -> Vec<Token> {
1080        let mut tz = Self::new(input);
1081        let mut out = Vec::new();
1082        while let Some(t) = tz.next_token() {
1083            if matches!(t, Token::Eof) {
1084                break;
1085            }
1086            out.push(t);
1087        }
1088        out
1089    }
1090}
1091
1092#[cfg(test)]
1093mod tests {
1094    use super::*;
1095
1096    #[test]
1097    fn empty_input_emits_eof_only() {
1098        let mut tz = CssTokenizer::new("");
1099        assert!(matches!(tz.next_token(), Some(Token::Eof)));
1100        assert!(tz.next_token().is_none());
1101    }
1102
1103    #[test]
1104    fn whitespace_collapses_runs() {
1105        let tokens = CssTokenizer::collect("   \t\n  ");
1106        assert_eq!(tokens.len(), 1, "whitespace run → single token");
1107        assert!(matches!(tokens[0], Token::Whitespace));
1108    }
1109
1110    #[test]
1111    fn simple_punctuation_tokens() {
1112        let tokens = CssTokenizer::collect(":;,.{}[]()");
1113        assert_eq!(tokens.len(), 10);
1114        assert!(matches!(tokens[0], Token::Colon));
1115        assert!(matches!(tokens[1], Token::Semicolon));
1116        assert!(matches!(tokens[2], Token::Comma));
1117        assert!(matches!(tokens[3], Token::Delim('.')));
1118        assert!(matches!(tokens[4], Token::OpenBrace));
1119        assert!(matches!(tokens[5], Token::CloseBrace));
1120        assert!(matches!(tokens[6], Token::OpenBracket));
1121        assert!(matches!(tokens[7], Token::CloseBracket));
1122        assert!(matches!(tokens[8], Token::OpenParen));
1123        assert!(matches!(tokens[9], Token::CloseParen));
1124    }
1125
1126    #[test]
1127    fn close_paren_is_emitted() {
1128        let tokens = CssTokenizer::collect(")");
1129        assert_eq!(tokens.len(), 1);
1130        assert!(matches!(tokens[0], Token::CloseParen));
1131    }
1132
1133    #[test]
1134    fn unknown_delim_chars() {
1135        let tokens = CssTokenizer::collect("^~`>");
1136        assert_eq!(tokens.len(), 4);
1137        assert!(matches!(tokens[0], Token::Delim('^')));
1138        assert!(matches!(tokens[1], Token::Delim('~')));
1139        assert!(matches!(tokens[2], Token::Delim('`')));
1140        assert!(matches!(tokens[3], Token::Delim('>')));
1141    }
1142
1143    #[test]
1144    fn preprocessing_normalizes_cr_lf_ff() {
1145        // CR LF → single LF
1146        let tz = CssTokenizer::new("a\r\nb");
1147        assert_eq!(tz.input, vec!['a', '\n', 'b']);
1148        // Lone CR → LF
1149        let tz = CssTokenizer::new("a\rb");
1150        assert_eq!(tz.input, vec!['a', '\n', 'b']);
1151        // FF → LF
1152        let tz = CssTokenizer::new("a\u{000C}b");
1153        assert_eq!(tz.input, vec!['a', '\n', 'b']);
1154    }
1155
1156    #[test]
1157    fn eof_emitted_once() {
1158        // C-1: use a non-ident-start, non-digit char to avoid the
1159        // ident-like / numeric branches (which are stubbed with todo!()
1160        // until C-2 / C-4).
1161        let mut tz = CssTokenizer::new(",");
1162        let t1 = tz.next_token();
1163        let t2 = tz.next_token();
1164        let t3 = tz.next_token();
1165        assert!(matches!(t1, Some(Token::Comma)));
1166        assert!(matches!(t2, Some(Token::Eof)));
1167        assert!(t3.is_none());
1168    }
1169
1170    #[test]
1171    fn block_comment_is_skipped() {
1172        // §4.3.2: `/* ... */` is consumed and no token is emitted.
1173        let tokens = CssTokenizer::collect("/* hello */:/*x*/;");
1174        assert_eq!(tokens.len(), 2);
1175        assert!(matches!(tokens[0], Token::Colon));
1176        assert!(matches!(tokens[1], Token::Semicolon));
1177    }
1178
1179    #[test]
1180    fn unterminated_comment_consumes_to_eof() {
1181        // §4.3.2: EOF before `*/` is a parse error; we consume to EOF.
1182        let tokens = CssTokenizer::collect("/* unfinished");
1183        assert_eq!(tokens.len(), 0, "unterminated comment → no tokens");
1184    }
1185
1186    #[test]
1187    fn comment_between_tokens() {
1188        // C-2 not yet: `a`/`b` would be ident tokens (todo!()). Verify
1189        // comment is consumed without affecting surrounding simple tokens
1190        // by using only punctuation.
1191        let tokens = CssTokenizer::collect(":/*c*/;");
1192        assert_eq!(tokens.len(), 2);
1193        assert!(matches!(tokens[0], Token::Colon));
1194        assert!(matches!(tokens[1], Token::Semicolon));
1195    }
1196
1197    #[test]
1198    fn cdo_token_emitted() {
1199        // §4.3.1: `<!--` → <CDO-token>
1200        let tokens = CssTokenizer::collect("<!--");
1201        assert_eq!(tokens.len(), 1);
1202        assert!(matches!(tokens[0], Token::Cdo));
1203    }
1204
1205    #[test]
1206    fn cdc_token_emitted() {
1207        // §4.3.1: `-->` → <CDC-token>
1208        let tokens = CssTokenizer::collect("-->");
1209        assert_eq!(tokens.len(), 1);
1210        assert!(matches!(tokens[0], Token::Cdc));
1211    }
1212
1213    #[test]
1214    fn cdo_cdc_wrapping_stylesheet() {
1215        // Legacy CSS 1/2.1 stylesheet wrapping.
1216        // `<!-- : ; -->`
1217        // tokens: Cdo, WS, Colon, WS, Semicolon, WS, Cdc = 7
1218        let tokens = CssTokenizer::collect("<!-- : ; -->");
1219        assert_eq!(tokens.len(), 7);
1220        assert!(matches!(tokens[0], Token::Cdo));
1221        assert!(matches!(tokens[1], Token::Whitespace));
1222        assert!(matches!(tokens[2], Token::Colon));
1223        assert!(matches!(tokens[3], Token::Whitespace));
1224        assert!(matches!(tokens[4], Token::Semicolon));
1225        assert!(matches!(tokens[5], Token::Whitespace));
1226        assert!(matches!(tokens[6], Token::Cdc));
1227    }
1228
1229    #[test]
1230    fn less_than_alone_is_delim() {
1231        // §4.3.1: `<` not followed by `!-` → <delim-token>
1232        let tokens = CssTokenizer::collect("< ");
1233        assert_eq!(tokens.len(), 2);
1234        assert!(matches!(tokens[0], Token::Delim('<')));
1235        assert!(matches!(tokens[1], Token::Whitespace));
1236    }
1237
1238    #[test]
1239    fn hyphen_alone_is_delim() {
1240        // §4.3.1: `-` not followed by number/ident/`->` → <delim-token>
1241        let tokens = CssTokenizer::collect("- ");
1242        assert_eq!(tokens.len(), 2);
1243        assert!(matches!(tokens[0], Token::Delim('-')));
1244        assert!(matches!(tokens[1], Token::Whitespace));
1245    }
1246
1247    // ── C-2 tests: ident / function / at-keyword / hash ──────────────
1248
1249    #[test]
1250    fn simple_ident() {
1251        let tokens = CssTokenizer::collect("color");
1252        assert_eq!(tokens.len(), 1);
1253        assert!(matches!(&tokens[0], Token::Ident(s) if s == "color"));
1254    }
1255
1256    #[test]
1257    fn ident_with_hyphen_prefix() {
1258        let tokens = CssTokenizer::collect("-webkit-flex");
1259        assert_eq!(tokens.len(), 1);
1260        assert!(matches!(&tokens[0], Token::Ident(s) if s == "-webkit-flex"));
1261    }
1262
1263    #[test]
1264    fn ident_with_double_hyphen() {
1265        // §4.3.9: `--` starts an ident sequence (custom properties).
1266        let tokens = CssTokenizer::collect("--my-var");
1267        assert_eq!(tokens.len(), 1);
1268        assert!(matches!(&tokens[0], Token::Ident(s) if s == "--my-var"));
1269    }
1270
1271    #[test]
1272    fn ident_with_underscore() {
1273        let tokens = CssTokenizer::collect("_private");
1274        assert_eq!(tokens.len(), 1);
1275        assert!(matches!(&tokens[0], Token::Ident(s) if s == "_private"));
1276    }
1277
1278    #[test]
1279    fn ident_with_escape() {
1280        // §4.3.7: `\26` → '&' (U+0026)
1281        let tokens = CssTokenizer::collect("color\\26 B");
1282        assert_eq!(tokens.len(), 1);
1283        assert!(matches!(&tokens[0], Token::Ident(s) if s == "color&B"));
1284    }
1285
1286    #[test]
1287    fn function_token() {
1288        let tokens = CssTokenizer::collect("translate(");
1289        assert_eq!(tokens.len(), 1);
1290        assert!(matches!(&tokens[0], Token::Function(s) if s == "translate"));
1291    }
1292
1293    #[test]
1294    fn at_keyword_token() {
1295        let tokens = CssTokenizer::collect("@media");
1296        assert_eq!(tokens.len(), 1);
1297        assert!(matches!(&tokens[0], Token::AtKeyword(s) if s == "media"));
1298    }
1299
1300    #[test]
1301    fn at_keyword_with_hyphen() {
1302        let tokens = CssTokenizer::collect("@-webkit-keyframes");
1303        assert_eq!(tokens.len(), 1);
1304        assert!(matches!(&tokens[0], Token::AtKeyword(s) if s == "-webkit-keyframes"));
1305    }
1306
1307    #[test]
1308    fn at_sign_alone_is_delim() {
1309        // §4.3.1: `@` not followed by ident-start → <delim-token>
1310        let tokens = CssTokenizer::collect("@ ");
1311        assert_eq!(tokens.len(), 2);
1312        assert!(matches!(tokens[0], Token::Delim('@')));
1313        assert!(matches!(tokens[1], Token::Whitespace));
1314    }
1315
1316    #[test]
1317    fn hash_id_type() {
1318        // §4.3.1 / §4.3.4: `#main` → <hash-token> with type "id"
1319        let tokens = CssTokenizer::collect("#main");
1320        assert_eq!(tokens.len(), 1);
1321        match &tokens[0] {
1322            Token::Hash(s, t) => {
1323                assert_eq!(s, "main");
1324                assert_eq!(*t, crate::types::HashType::Id);
1325            }
1326            _ => panic!("expected Hash, got {:?}", tokens[0]),
1327        }
1328    }
1329
1330    #[test]
1331    fn hash_unrestricted_type() {
1332        // §4.3.1 L801-826 + §4.3.4: `#123` → next `1` is an ident code
1333        // point (digits are ident code points per §4.2), so a hash-token
1334        // is created. `123` does not would-start-an-ident-sequence (digit
1335        // is not ident-start), so type is "unrestricted". The value is
1336        // "123" (consume_an_ident_sequence consumes all ident code points).
1337        let tokens = CssTokenizer::collect("#123");
1338        assert_eq!(tokens.len(), 1);
1339        match &tokens[0] {
1340            Token::Hash(s, t) => {
1341                assert_eq!(s, "123");
1342                assert_eq!(*t, crate::types::HashType::Unrestricted);
1343            }
1344            _ => panic!("expected Hash, got {:?}", tokens[0]),
1345        }
1346    }
1347
1348    #[test]
1349    fn hash_hex_color_is_unrestricted() {
1350        // §4.3.4: `#ff00aa` → starts with `f` which IS ident-start, so
1351        // this is actually type "id" (the hex digits form a valid ident).
1352        let tokens = CssTokenizer::collect("#ff00aa");
1353        assert_eq!(tokens.len(), 1);
1354        match &tokens[0] {
1355            Token::Hash(s, t) => {
1356                assert_eq!(s, "ff00aa");
1357                assert_eq!(*t, crate::types::HashType::Id);
1358            }
1359            _ => panic!("expected Hash, got {:?}", tokens[0]),
1360        }
1361    }
1362
1363    #[test]
1364    fn backslash_escape_in_ident() {
1365        // §4.3.7: `\` followed by non-hex → literal char. Use `z` (not
1366        // `b`, which is a hex digit and would decode to U+000B).
1367        let tokens = CssTokenizer::collect("a\\z");
1368        assert_eq!(tokens.len(), 1);
1369        assert!(matches!(&tokens[0], Token::Ident(s) if s == "az"));
1370    }
1371
1372    #[test]
1373    fn backslash_at_eof_yields_fffd_ident() {
1374        // §4.3.8: EOF as the second code point is a valid escape;
1375        // §4.3.7: escaped-EOF returns U+FFFD. So a lone `\` at EOF
1376        // produces an <ident-token> whose value is U+FFFD (WPT
1377        // css/css-syntax/escaped-eof.html: "Escaped EOF turns into a
1378        // U+FFFD in an ident token").
1379        let tokens = CssTokenizer::collect("\\");
1380        assert_eq!(tokens.len(), 1);
1381        assert_eq!(tokens[0], Token::Ident("\u{FFFD}".to_string()));
1382    }
1383
1384    #[test]
1385    fn backslash_newline_is_delim() {
1386        // §4.3.8: `\` followed by newline is not a valid escape.
1387        let tokens = CssTokenizer::collect("\\\n");
1388        assert_eq!(tokens.len(), 2);
1389        assert!(matches!(tokens[0], Token::Delim('\\')));
1390        assert!(matches!(tokens[1], Token::Whitespace));
1391    }
1392
1393    #[test]
1394    fn declaration_with_ident_and_value() {
1395        let tokens = CssTokenizer::collect("color: red");
1396        assert_eq!(tokens.len(), 4);
1397        assert!(matches!(&tokens[0], Token::Ident(s) if s == "color"));
1398        assert!(matches!(tokens[1], Token::Colon));
1399        assert!(matches!(tokens[2], Token::Whitespace));
1400        assert!(matches!(&tokens[3], Token::Ident(s) if s == "red"));
1401    }
1402
1403    #[test]
1404    fn unicode_escape_in_ident() {
1405        // §4.3.7 L1223-1225: `\000026` → '&' (U+0026); after 6 hex digits,
1406        // if next is whitespace, consume it. So `\000026 B` → the space is
1407        // consumed as trailing whitespace, `B` joins the ident sequence →
1408        // single token Ident("&B").
1409        let tokens = CssTokenizer::collect("\\000026 B");
1410        assert_eq!(tokens.len(), 1);
1411        assert!(matches!(&tokens[0], Token::Ident(s) if s == "&B"));
1412    }
1413
1414    #[test]
1415    fn null_escape_becomes_replacement_char() {
1416        // §4.3.7: `\0` → U+FFFD (NULL is not allowed)
1417        let tokens = CssTokenizer::collect("\\0");
1418        assert_eq!(tokens.len(), 1);
1419        assert!(matches!(&tokens[0], Token::Ident(s) if s == "\u{FFFD}"));
1420    }
1421
1422    #[test]
1423    fn surrogate_escape_becomes_replacement_char() {
1424        // §4.3.7: `\D800` → U+FFFD (surrogates not allowed)
1425        let tokens = CssTokenizer::collect("\\D800");
1426        assert_eq!(tokens.len(), 1);
1427        assert!(matches!(&tokens[0], Token::Ident(s) if s == "\u{FFFD}"));
1428    }
1429
1430    // ── C-2 Bug 1 回归测试: # Delim 回退 (§4.3.1 L824-826) ────────────
1431
1432    #[test]
1433    fn hash_alone_is_delim() {
1434        // §4.3.1 L824-826: `#` followed by EOF → next is not ident code
1435        // point, not valid escape → return delim-token with value `#`.
1436        let tokens = CssTokenizer::collect("#");
1437        assert_eq!(tokens.len(), 1);
1438        assert!(matches!(tokens[0], Token::Delim('#')));
1439    }
1440
1441    #[test]
1442    fn hash_followed_by_space_is_delim() {
1443        // §4.3.1 L824-826: `#` followed by space (not ident code point,
1444        // not valid escape) → delim-token with value `#`.
1445        let tokens = CssTokenizer::collect("# ");
1446        assert_eq!(tokens.len(), 2);
1447        assert!(matches!(tokens[0], Token::Delim('#')));
1448        assert!(matches!(tokens[1], Token::Whitespace));
1449    }
1450
1451    #[test]
1452    fn hash_followed_by_at_is_delim() {
1453        // §4.3.1 L824-826: `#` followed by `@` (not ident code point,
1454        // not valid escape) → delim-token with value `#`.
1455        let tokens = CssTokenizer::collect("#@");
1456        assert_eq!(tokens.len(), 2);
1457        assert!(matches!(tokens[0], Token::Delim('#')));
1458        assert!(matches!(tokens[1], Token::Delim('@')));
1459    }
1460
1461    // ── C-3 tests: §4.3.5 string + bad-string ────────────────────────
1462
1463    #[test]
1464    fn string_double_quoted() {
1465        // §4.3.5: "hello" → String("hello")
1466        let tokens = CssTokenizer::collect("\"hello\"");
1467        assert_eq!(tokens.len(), 1);
1468        assert!(matches!(&tokens[0], Token::String(s) if s == "hello"));
1469    }
1470
1471    #[test]
1472    fn string_single_quoted() {
1473        // §4.3.5: 'hello' → String("hello")
1474        let tokens = CssTokenizer::collect("'hello'");
1475        assert_eq!(tokens.len(), 1);
1476        assert!(matches!(&tokens[0], Token::String(s) if s == "hello"));
1477    }
1478
1479    #[test]
1480    fn string_with_escape() {
1481        // §4.3.5 + §4.3.7: "a\z" → String("az") (z is non-hex → literal)
1482        let tokens = CssTokenizer::collect("\"a\\z\"");
1483        assert_eq!(tokens.len(), 1);
1484        assert!(matches!(&tokens[0], Token::String(s) if s == "az"));
1485    }
1486
1487    #[test]
1488    fn string_unterminated_eof() {
1489        // §4.3.5 L1101-1104: EOF before closing quote → parse error,
1490        // return string-token with value accumulated so far.
1491        let tokens = CssTokenizer::collect("\"hello");
1492        assert_eq!(tokens.len(), 1);
1493        assert!(matches!(&tokens[0], Token::String(s) if s == "hello"));
1494    }
1495
1496    #[test]
1497    fn string_unescaped_newline_is_bad_string() {
1498        // §4.3.5 L1106-1110: unescaped newline → parse error, reconsume,
1499        // return bad-string. The reconsumed newline is then tokenized as
1500        // whitespace by the next consume_a_token call.
1501        let tokens = CssTokenizer::collect("\"\n");
1502        assert_eq!(tokens.len(), 2);
1503        assert!(matches!(tokens[0], Token::BadString));
1504        assert!(matches!(tokens[1], Token::Whitespace));
1505    }
1506
1507    #[test]
1508    fn string_line_continuation() {
1509        // §4.3.5 L1117-1119: backslash followed by newline → line
1510        // continuation (newline consumed, no char appended).
1511        let tokens = CssTokenizer::collect("\"a\\\nb\"");
1512        assert_eq!(tokens.len(), 1);
1513        assert!(matches!(&tokens[0], Token::String(s) if s == "ab"));
1514    }
1515
1516    #[test]
1517    fn string_hex_escape() {
1518        // §4.3.5 + §4.3.7: "\26" → String("&") (U+0026)
1519        let tokens = CssTokenizer::collect("\"\\26\"");
1520        assert_eq!(tokens.len(), 1);
1521        assert!(matches!(&tokens[0], Token::String(s) if s == "&"));
1522    }
1523
1524    // ── C-4 tests: §4.3.3/§4.3.10/§4.3.13 number/percentage/dimension ─
1525
1526    #[test]
1527    fn number_integer() {
1528        let tokens = CssTokenizer::collect("42");
1529        assert_eq!(tokens.len(), 1);
1530        assert_eq!(
1531            tokens[0],
1532            Token::Number(Numeric {
1533                value: 42.0,
1534                is_integer: true
1535            })
1536        );
1537    }
1538
1539    #[test]
1540    fn number_decimal() {
1541        let tokens = CssTokenizer::collect("3.5");
1542        assert_eq!(tokens.len(), 1);
1543        assert_eq!(
1544            tokens[0],
1545            Token::Number(Numeric {
1546                value: 3.5,
1547                is_integer: false
1548            })
1549        );
1550    }
1551
1552    #[test]
1553    fn number_signed() {
1554        let tokens = CssTokenizer::collect("-5");
1555        assert_eq!(tokens.len(), 1);
1556        assert_eq!(
1557            tokens[0],
1558            Token::Number(Numeric {
1559                value: -5.0,
1560                is_integer: true
1561            })
1562        );
1563    }
1564
1565    #[test]
1566    fn number_exponent() {
1567        // §4.3.13: 1e3 → 1.0 * 10^3 = 1000.0, type "number"
1568        let tokens = CssTokenizer::collect("1e3");
1569        assert_eq!(tokens.len(), 1);
1570        assert_eq!(
1571            tokens[0],
1572            Token::Number(Numeric {
1573                value: 1000.0,
1574                is_integer: false
1575            })
1576        );
1577    }
1578
1579    #[test]
1580    fn number_decimal_exponent() {
1581        // §4.3.13: 1.5e2 → 1.5 * 10^2 = 150.0, type "number"
1582        let tokens = CssTokenizer::collect("1.5e2");
1583        assert_eq!(tokens.len(), 1);
1584        assert_eq!(
1585            tokens[0],
1586            Token::Number(Numeric {
1587                value: 150.0,
1588                is_integer: false
1589            })
1590        );
1591    }
1592
1593    #[test]
1594    fn percentage_token() {
1595        let tokens = CssTokenizer::collect("50%");
1596        assert_eq!(tokens.len(), 1);
1597        assert_eq!(
1598            tokens[0],
1599            Token::Percentage(Numeric {
1600                value: 50.0,
1601                is_integer: true
1602            })
1603        );
1604    }
1605
1606    #[test]
1607    fn dimension_px() {
1608        let tokens = CssTokenizer::collect("10px");
1609        assert_eq!(tokens.len(), 1);
1610        assert_eq!(
1611            tokens[0],
1612            Token::Dimension(
1613                Numeric {
1614                    value: 10.0,
1615                    is_integer: true
1616                },
1617                "px".to_string(),
1618            )
1619        );
1620    }
1621
1622    #[test]
1623    fn dimension_em() {
1624        let tokens = CssTokenizer::collect("1.5em");
1625        assert_eq!(tokens.len(), 1);
1626        assert_eq!(
1627            tokens[0],
1628            Token::Dimension(
1629                Numeric {
1630                    value: 1.5,
1631                    is_integer: false
1632                },
1633                "em".to_string(),
1634            )
1635        );
1636    }
1637
1638    #[test]
1639    fn dimension_signed() {
1640        let tokens = CssTokenizer::collect("-30deg");
1641        assert_eq!(tokens.len(), 1);
1642        assert_eq!(
1643            tokens[0],
1644            Token::Dimension(
1645                Numeric {
1646                    value: -30.0,
1647                    is_integer: true
1648                },
1649                "deg".to_string(),
1650            )
1651        );
1652    }
1653
1654    #[test]
1655    fn plus_sign_number() {
1656        let tokens = CssTokenizer::collect("+5");
1657        assert_eq!(tokens.len(), 1);
1658        assert_eq!(
1659            tokens[0],
1660            Token::Number(Numeric {
1661                value: 5.0,
1662                is_integer: true
1663            })
1664        );
1665    }
1666
1667    #[test]
1668    fn dot_starts_number() {
1669        let tokens = CssTokenizer::collect(".5");
1670        assert_eq!(tokens.len(), 1);
1671        assert_eq!(
1672            tokens[0],
1673            Token::Number(Numeric {
1674                value: 0.5,
1675                is_integer: false
1676            })
1677        );
1678    }
1679
1680    // ── C-5 tests: §4.3.4 url( special case + §4.3.6 Url + §4.3.15 BadUrl ─
1681
1682    #[test]
1683    fn url_unquoted_simple() {
1684        // §4.3.6: url(foo) → Url("foo")
1685        let tokens = CssTokenizer::collect("url(foo)");
1686        assert_eq!(tokens.len(), 1);
1687        assert_eq!(tokens[0], Token::Url("foo".to_string()));
1688    }
1689
1690    #[test]
1691    fn url_unquoted_with_spaces() {
1692        // §4.3.6: leading/trailing whitespace consumed; url( foo ) → Url("foo")
1693        let tokens = CssTokenizer::collect("url( foo )");
1694        assert_eq!(tokens.len(), 1);
1695        assert_eq!(tokens[0], Token::Url("foo".to_string()));
1696    }
1697
1698    #[test]
1699    fn url_empty() {
1700        // §4.3.6: url() → Url("")
1701        let tokens = CssTokenizer::collect("url()");
1702        assert_eq!(tokens.len(), 1);
1703        assert_eq!(tokens[0], Token::Url(String::new()));
1704    }
1705
1706    #[test]
1707    fn url_eof_unterminated() {
1708        // §4.3.6 L1161-1164: EOF → parse error, return url-token with value so far
1709        let tokens = CssTokenizer::collect("url(foo");
1710        assert_eq!(tokens.len(), 1);
1711        assert_eq!(tokens[0], Token::Url("foo".to_string()));
1712    }
1713
1714    #[test]
1715    fn url_quoted_is_function() {
1716        // §4.3.4 L1058-1063: url("foo") → Function("url"); the body
1717        // `"foo")` remains in the stream → String("foo") + CloseParen.
1718        let tokens = CssTokenizer::collect("url(\"foo\")");
1719        assert_eq!(tokens.len(), 3);
1720        assert_eq!(tokens[0], Token::Function("url".to_string()));
1721        assert_eq!(tokens[1], Token::String("foo".to_string()));
1722        assert_eq!(tokens[2], Token::CloseParen);
1723    }
1724
1725    #[test]
1726    fn url_single_quoted_is_function() {
1727        // §4.3.4 L1058-1063: url('foo') → Function("url") + String + CloseParen
1728        let tokens = CssTokenizer::collect("url('foo')");
1729        assert_eq!(tokens.len(), 3);
1730        assert_eq!(tokens[0], Token::Function("url".to_string()));
1731        assert_eq!(tokens[1], Token::String("foo".to_string()));
1732        assert_eq!(tokens[2], Token::CloseParen);
1733    }
1734
1735    #[test]
1736    fn url_ws_then_quote_is_function() {
1737        // §4.3.4 L1058-1063: url( "foo") → whitespace+quote → Function.
1738        // L1056-1057 only collapses when *two* consecutive whitespace
1739        // follow `(`; here only one space precedes `"`, so it is NOT
1740        // consumed and becomes a Whitespace token.
1741        let tokens = CssTokenizer::collect("url( \"foo\")");
1742        assert_eq!(tokens.len(), 4);
1743        assert_eq!(tokens[0], Token::Function("url".to_string()));
1744        assert_eq!(tokens[1], Token::Whitespace);
1745        assert_eq!(tokens[2], Token::String("foo".to_string()));
1746        assert_eq!(tokens[3], Token::CloseParen);
1747    }
1748
1749    #[test]
1750    fn url_with_escape() {
1751        // §4.3.6 + §4.3.7: url(foo\29 bar) → \29 consumes trailing space,
1752        // decodes to `)`, then `bar` appended → Url("foo)bar")
1753        let tokens = CssTokenizer::collect("url(foo\\29 bar)");
1754        assert_eq!(tokens.len(), 1);
1755        assert_eq!(tokens[0], Token::Url("foo)bar".to_string()));
1756    }
1757
1758    #[test]
1759    fn url_bad_paren_in_unquoted() {
1760        // §4.3.6 L1177-1185: `(` in unquoted url → BadUrl
1761        let tokens = CssTokenizer::collect("url(foo(bar)");
1762        assert_eq!(tokens.len(), 1);
1763        assert_eq!(tokens[0], Token::BadUrl);
1764    }
1765
1766    #[test]
1767    fn url_bad_quote_in_unquoted() {
1768        // §4.3.6 L1177-1185: `"` in unquoted url → BadUrl
1769        let tokens = CssTokenizer::collect("url(foo\"bar)");
1770        assert_eq!(tokens.len(), 1);
1771        assert_eq!(tokens[0], Token::BadUrl);
1772    }
1773
1774    // ── C-6 tests: §4.3.1 unicode_ranges_allowed + §4.3.11 + §4.3.14 ──
1775
1776    #[test]
1777    fn unicode_range_disabled_by_default() {
1778        // §4.3.1 L782-783: default unicode_ranges_allowed=false.
1779        // `U+1234` → `U` ident-start → Ident("U") (`+` stops ident seq);
1780        // then `+1234` → starts_with_number (`+`+digit) → Number(1234).
1781        let tokens = CssTokenizer::collect("U+1234");
1782        assert_eq!(tokens.len(), 2);
1783        assert_eq!(tokens[0], Token::Ident("U".to_string()));
1784        assert_eq!(
1785            tokens[1],
1786            Token::Number(Numeric {
1787                value: 1234.0,
1788                is_integer: true
1789            })
1790        );
1791    }
1792
1793    #[test]
1794    fn unicode_range_simple() {
1795        // §4.3.14: U+1234 → UnicodeRange(0x1234, 0x1234)
1796        let mut tz = CssTokenizer::new("U+1234");
1797        tz.set_unicode_ranges_allowed(true);
1798        let t = tz.next_token().unwrap();
1799        assert_eq!(t, Token::UnicodeRange(Some(0x1234), Some(0x1234)));
1800    }
1801
1802    #[test]
1803    fn unicode_range_question_marks() {
1804        // §4.3.14 step 3: U+12?? → start=0x1200 (?→0), end=0x12FF (?→F)
1805        let mut tz = CssTokenizer::new("U+12??");
1806        tz.set_unicode_ranges_allowed(true);
1807        let t = tz.next_token().unwrap();
1808        assert_eq!(t, Token::UnicodeRange(Some(0x1200), Some(0x12FF)));
1809    }
1810
1811    #[test]
1812    fn unicode_range_range() {
1813        // §4.3.14 step 5: U+12-34FF → start=0x12, end=0x34FF
1814        let mut tz = CssTokenizer::new("U+12-34FF");
1815        tz.set_unicode_ranges_allowed(true);
1816        let t = tz.next_token().unwrap();
1817        assert_eq!(t, Token::UnicodeRange(Some(0x12), Some(0x34FF)));
1818    }
1819
1820    #[test]
1821    fn unicode_range_max_hex() {
1822        // §4.3.14: U+10FFFF → UnicodeRange(0x10FFFF, 0x10FFFF)
1823        let mut tz = CssTokenizer::new("U+10FFFF");
1824        tz.set_unicode_ranges_allowed(true);
1825        let t = tz.next_token().unwrap();
1826        assert_eq!(t, Token::UnicodeRange(Some(0x10FFFF), Some(0x10FFFF)));
1827    }
1828
1829    #[test]
1830    fn unicode_range_lowercase_u() {
1831        // §4.3.11: lowercase u also triggers; u+abc → UnicodeRange(0xABC, 0xABC)
1832        let mut tz = CssTokenizer::new("u+abc");
1833        tz.set_unicode_ranges_allowed(true);
1834        let t = tz.next_token().unwrap();
1835        assert_eq!(t, Token::UnicodeRange(Some(0xABC), Some(0xABC)));
1836    }
1837}