muskitty_css_tokenizer/impls.rs
1//! CSS Syntax tokenizer implementation.
2//!
3//! Implements the "Tokenizer Algorithms" of CSS Syntax Module Level 3 §4.3.
4//! The tokenizer is recursive-descent: [`CssTokenizer::next_token`] calls
5//! [`consume_a_token`](CssTokenizer::consume_a_token) (§4.3.1), which
6//! dispatches to sub-algorithms based on the current input code point.
7//!
8//! # Coverage
9//!
10//! All §4.3 sub-algorithms are implemented:
11//! - §4.3.1 Consume a token (full dispatch, incl. `unicode_ranges_allowed`)
12//! - §4.3.2 Consume comments
13//! - §4.3.3 Consume a numeric token
14//! - §4.3.4 Consume an ident-like token (incl. `url(` special case)
15//! - §4.3.5 Consume a string token
16//! - §4.3.6 Consume a url token
17//! - §4.3.7 Consume an escaped code point
18//! - §4.3.8 Check if two code points are a valid escape
19//! - §4.3.9 Check if three code points would start an ident sequence
20//! - §4.3.10 Check if three code points would start a number
21//! - §4.3.11 Check if three code points would start a unicode-range
22//! - §4.3.12 Consume an ident sequence
23//! - §4.3.13 Consume a number
24//! - §4.3.14 Consume a unicode-range token
25//! - §4.3.15 Consume the remnants of a bad url
26
27use crate::trait_def::Tokenizer;
28use crate::types::{HashType, Numeric, State, Token};
29
30/// The CSS tokenizer (§4.3).
31///
32/// Holds the preprocessed input stream (§5.3 normalization applied at
33/// construction), the current position, and EOF tracking. Sub-algorithms
34/// are implemented as methods on this struct so they share the input
35/// stream and position.
36pub struct CssTokenizer {
37 /// Input code points after §5.3 preprocessing (CR/LF/FF normalized to LF).
38 input: Vec<char>,
39 /// Current position in `input` (0-based index of the next code point
40 /// to consume). `pos == input.len()` means the stream is exhausted.
41 pos: usize,
42 /// Top-level state (§4.3.1 dispatch vs. EOF).
43 state: State,
44 /// Whether `<EOF-token>` has been emitted. After this is `true`,
45 /// [`next_token`](Tokenizer::next_token) returns `None`.
46 eof_emitted: bool,
47 /// §4.3.1 L782-783: `unicode_ranges_allowed` flag, default `false`.
48 /// When `true`, the `U+`/`u+` branch of §4.3.1 produces a
49 /// `<unicode-range-token>` (§4.3.14); otherwise `U`/`u` is tokenized
50 /// as an ident-like token.
51 unicode_ranges_allowed: bool,
52}
53
54impl CssTokenizer {
55 /// Construct a new tokenizer over `input`.
56 ///
57 /// Applies §5.3 input preprocessing: per the "input stream"
58 /// definition, U+000D CR followed by U+000A LF is collapsed to a
59 /// single LF, lone CR is replaced with LF, and U+000C FF is replaced
60 /// with LF. The tokenizer operates on this normalized stream.
61 pub fn new(input: &str) -> Self {
62 let input = preprocess_input(input);
63 Self {
64 input,
65 pos: 0,
66 state: State::Data,
67 eof_emitted: false,
68 unicode_ranges_allowed: false,
69 }
70 }
71
72 /// Peek at the code point at `offset` from the current position
73 /// without consuming it. Returns `None` if past end of input.
74 ///
75 /// Used by §4.3.8 / §4.3.9 / §4.3.10 / §4.3.11 "check if N code
76 /// points..." predicates, which must look ahead without advancing.
77 fn peek(&self, offset: usize) -> Option<char> {
78 self.input.get(self.pos + offset).copied()
79 }
80
81 /// Consume and return the next code point, advancing `pos`. Returns
82 /// `None` if at end of input.
83 ///
84 /// Implements the "consume the next input code point" primitive used
85 /// throughout §4.3.
86 fn consume(&mut self) -> Option<char> {
87 let c = self.input.get(self.pos).copied();
88 if c.is_some() {
89 self.pos += 1;
90 }
91 c
92 }
93
94 /// Re-consume the current code point: step `pos` back by one.
95 ///
96 /// Per §4.3.1 "reconsume the current input code point": the next
97 /// call to [`consume`](Self::consume) will return the same code point
98 /// that was just consumed. No-op if `pos == 0`.
99 fn reconsume(&mut self) {
100 if self.pos > 0 {
101 self.pos -= 1;
102 }
103 }
104
105 /// §4.3.1 Consume a token.
106 ///
107 /// This is the main entry point of the tokenizer. Dispatches based on
108 /// the current input code point to one of the sub-algorithms (§4.3.2
109 /// through §4.3.6) or emits a simple token directly. The optional
110 /// `unicode_ranges_allowed` flag (§4.3.1 L782-783, default `false`)
111 /// gates the `U+`/`u+` unicode-range branch.
112 fn consume_a_token(&mut self) -> Token {
113 // §4.3.2: If the next two input code points are U+002F SOLIDUS
114 // (`/`) and U+002A ASTERISK (`*`), consume comments and
115 // recursively consume the next token. We implement this as a
116 // loop: any leading run of `/* ... */` comments is skipped before
117 // producing the next real token.
118 loop {
119 let Some(c) = self.consume() else {
120 // §5.3: end of stream → <EOF-token>
121 self.eof_emitted = true;
122 self.state = State::Eof;
123 return Token::Eof;
124 };
125
126 // §4.3.2 comment dispatch: `/` followed by `*`.
127 if c == '/' && self.peek(0) == Some('*') {
128 self.consume(); // consume `*`
129 self.consume_comments_body();
130 // Loop back to consume the next token (which may itself
131 // be preceded by another comment).
132 continue;
133 }
134
135 return match c {
136 // §4.3.1: whitespace → <whitespace-token>
137 ' ' | '\t' | '\n' | '\r' | '\u{000C}' => {
138 // Consume all consecutive whitespace.
139 while let Some(next) = self.peek(0) {
140 if matches!(next, ' ' | '\t' | '\n' | '\r' | '\u{000C}') {
141 self.consume();
142 } else {
143 break;
144 }
145 }
146 Token::Whitespace
147 }
148 // §4.3.1: `"` or `'` → consume a string token (§4.3.5)
149 '"' | '\'' => self.consume_a_string_token(c),
150 // §4.3.1: `#` → consume an ident-like token (§4.3.4) for hash
151 '#' => self.consume_a_hash_token(),
152 // §4.3.1: `(` → <(-token>
153 '(' => Token::OpenParen,
154 // §4.3.1: `)` → <)-token>
155 ')' => Token::CloseParen,
156 // §4.3.1: `+` or `-` → if would-start-a-number, consume a
157 // numeric token (§4.3.3); else <delim-token>.
158 // §4.3.1: `-` additionally: if next two are `-` and `>`
159 // (i.e. `-->`), consume them and return <CDC-token>.
160 // §4.3.1: `.` → if would-start-a-number, consume a numeric
161 // token; else <delim-token>.
162 '-' => {
163 if self.starts_with_number(c) {
164 self.reconsume();
165 self.consume_a_numeric_token()
166 } else if self.peek(0) == Some('-') && self.peek(1) == Some('>') {
167 // §4.3.1: `-->` → <CDC-token>
168 self.consume(); // consume second `-`
169 self.consume(); // consume `>`
170 Token::Cdc
171 } else if self.would_start_ident_sequence_at(0) {
172 // §4.3.1: `-` starting an ident sequence → reconsume
173 // and consume an ident-like token.
174 self.reconsume();
175 self.consume_an_ident_like_token()
176 } else {
177 Token::Delim(c)
178 }
179 }
180 '+' | '.' => {
181 if self.starts_with_number(c) {
182 self.reconsume();
183 self.consume_a_numeric_token()
184 } else {
185 Token::Delim(c)
186 }
187 }
188 // §4.3.1: `<` → if next three are `!`, `-`, `-` (i.e.
189 // `<!--`), consume them and return <CDO-token>. Otherwise
190 // emit <delim-token>.
191 '<' => {
192 if self.peek(0) == Some('!')
193 && self.peek(1) == Some('-')
194 && self.peek(2) == Some('-')
195 {
196 // Consume `!`, `-`, `-`.
197 self.consume();
198 self.consume();
199 self.consume();
200 Token::Cdo
201 } else {
202 Token::Delim(c)
203 }
204 }
205 // §4.3.1: `@` → if next would-start-an-ident-sequence,
206 // consume an ident-like token (§4.3.4) for at-keyword.
207 '@' => self.consume_an_at_keyword_token(),
208 // §4.3.1: `[` → <[-token>
209 '[' => Token::OpenBracket,
210 // §4.3.1: `\` → if would-start-an-escape (§4.3.8), reconsume
211 // and consume an ident-like token (§4.3.4); else parse error,
212 // <delim-token>.
213 '\\' => self.consume_ident_or_delim(),
214 // §4.3.1: `]` → <]-token>
215 ']' => Token::CloseBracket,
216 // §4.3.1: `{` → <{-token>
217 '{' => Token::OpenBrace,
218 // §4.3.1: `}` → <}-token>
219 '}' => Token::CloseBrace,
220 // §4.3.1: digit → reconsume and consume a numeric token (§4.3.3)
221 '0'..='9' => {
222 self.reconsume();
223 self.consume_a_numeric_token()
224 }
225 // §4.3.1: `U+` or `u+` → if would-start-a-unicode-range (§4.3.11),
226 // consume a unicode-range token (§4.3.14).
227 'u' | 'U' => self.consume_u_or_unicode_range(),
228 // §4.3.1: ident-start code point → reconsume and consume an
229 // ident-like token (§4.3.4)
230 _ if is_ident_start_code_point(c) => {
231 self.reconsume();
232 self.consume_an_ident_like_token()
233 }
234 // §4.3.1: `:` → <colon-token>
235 ':' => Token::Colon,
236 // §4.3.1: `;` → <semicolon-token>
237 ';' => Token::Semicolon,
238 // §4.3.1: `,` → <comma-token>
239 ',' => Token::Comma,
240 // §4.3.1: anything else → <delim-token>
241 _ => Token::Delim(c),
242 };
243 }
244 }
245
246 /// §4.3.2 Consume comments — body only.
247 ///
248 /// Precondition: the opening `/*` has already been consumed. Consume
249 /// code points until the closing `*/` or EOF. Per §4.3.2, an
250 /// unterminated comment (EOF before `*/`) is a parse error; we
251 /// silently consume to EOF in that case (no error reporting yet).
252 fn consume_comments_body(&mut self) {
253 loop {
254 match self.consume() {
255 None => return, // EOF in comment: parse error (unreported)
256 Some('*') => {
257 if self.peek(0) == Some('/') {
258 self.consume(); // consume `/`
259 return;
260 }
261 }
262 Some(_) => {}
263 }
264 }
265 }
266
267 // ── Sub-algorithm stubs (implemented in subsequent commits) ───────
268
269 /// §4.3.5 (L1081-1128) Consume a string token.
270 ///
271 /// `quote` is the ending code point (the opening quote `"` or `'`).
272 /// Returns a `string-token` (normal or EOF-terminated) or a
273 /// `bad-string-token` (unescaped newline).
274 ///
275 /// Per §4.3.5:
276 /// - ending code point (L1097-1099) → return string-token.
277 /// - EOF (L1101-1104) → parse error, return string-token.
278 /// - newline (L1106-1110) → parse error, reconsume, return bad-string.
279 /// - `\` (L1112-1124):
280 /// - next EOF → do nothing (loop; next consume hits EOF → string).
281 /// - next newline → consume it (line continuation).
282 /// - otherwise (valid escape) → consume an escaped code point, append.
283 /// - anything else (L1126-1128) → append.
284 fn consume_a_string_token(&mut self, quote: char) -> Token {
285 let mut value = String::new();
286 loop {
287 match self.consume() {
288 None => {
289 // §4.3.5 L1101-1104: EOF → parse error, return string-token.
290 return Token::String(value);
291 }
292 Some(c) if c == quote => {
293 // §4.3.5 L1097-1099: ending code point → return string-token.
294 return Token::String(value);
295 }
296 Some('\n') => {
297 // §4.3.5 L1106-1110: newline → parse error, reconsume, return bad-string.
298 self.reconsume();
299 return Token::BadString;
300 }
301 Some('\\') => {
302 // §4.3.5 L1112-1124: escape handling.
303 match self.peek(0) {
304 None => {
305 // §4.3.5 L1114-1115: next is EOF, do nothing.
306 // Loop continues; next consume() returns None → String.
307 }
308 Some('\n') => {
309 // §4.3.5 L1117-1119: next is newline, consume it (line continuation).
310 self.consume();
311 }
312 Some(_) => {
313 // §4.3.5 L1121-1124: valid escape, consume escaped code point.
314 let escaped = self.consume_an_escaped_code_point();
315 value.push(escaped);
316 }
317 }
318 }
319 Some(c) => {
320 // §4.3.5 L1126-1128: anything else → append.
321 value.push(c);
322 }
323 }
324 }
325 }
326
327 /// §4.3.1 (L801-826) `#` branch: Consume a hash token.
328 ///
329 /// Precondition: `#` has been consumed. Per §4.3.1 L801-826:
330 /// - If the next code point is an ident code point OR the next two
331 /// are a valid escape, then create a `hash-token` (type "id" if
332 /// would-start-ident-sequence, else "unrestricted"), consume an
333 /// ident sequence for the value, and return it.
334 /// - Otherwise, return a `delim-token` with value `#`.
335 fn consume_a_hash_token(&mut self) -> Token {
336 // §4.3.1 L803-805: next is ident code point OR next two are valid escape.
337 let next_is_ident = self.peek(0).is_some_and(is_ident_code_point);
338 let next_is_valid_escape = self.is_valid_escape_at(0);
339 if !next_is_ident && !next_is_valid_escape {
340 // §4.3.1 L824-826: return delim-token with value `#`.
341 return Token::Delim('#');
342 }
343 let hash_type = if self.would_start_ident_sequence_at(0) {
344 HashType::Id
345 } else {
346 HashType::Unrestricted
347 };
348 let name = self.consume_an_ident_sequence();
349 Token::Hash(name, hash_type)
350 }
351
352 /// §4.3.3 (L1011-1042) Consume a numeric token.
353 ///
354 /// Returns a `number-token`, `percentage-token`, or `dimension-token`.
355 /// Per §4.3.3:
356 /// 1. Consume a number (§4.3.13).
357 /// 2. If next 3 would start an ident sequence → dimension-token (consume
358 /// an ident sequence for the unit).
359 /// 3. Else if next is `%` → percentage-token (consume `%`).
360 /// 4. Else → number-token.
361 fn consume_a_numeric_token(&mut self) -> Token {
362 let (value, is_integer) = self.consume_a_number();
363 let numeric = Numeric { value, is_integer };
364 // §4.3.3 L1019-1029: dimension
365 if self.would_start_ident_sequence_at(0) {
366 let unit = self.consume_an_ident_sequence();
367 return Token::Dimension(numeric, unit);
368 }
369 // §4.3.3 L1032-1036: percentage
370 if self.peek(0) == Some('%') {
371 self.consume();
372 return Token::Percentage(numeric);
373 }
374 // §4.3.3 L1040-1042: number
375 Token::Number(numeric)
376 }
377
378 /// §4.3.13 (L1415-1483) Consume a number.
379 ///
380 /// Returns `(value, is_integer)` where `is_integer` is true when the
381 /// source representation had no fractional part and no exponent (type
382 /// "integer"), false when it had either (type "number"). The sign is
383 /// included in `value`.
384 ///
385 /// Per §4.3.13:
386 /// 1. type = "integer"; number_part = ""; exponent_part = "".
387 /// 2. If next is `+`/`-`, consume, append to number_part.
388 /// 3. While next is digit, consume, append to number_part.
389 /// 4. If next 2 are `.` + digit: consume `.`, consume digits; type = "number".
390 /// 5. If next 2/3 are `e`/`E` + optional `+`/`-` + digit: consume `e`/`E`,
391 /// optional sign, digits into exponent_part; type = "number".
392 /// 6. value = parse number_part; if exponent_part non-empty, value *= 10^exp.
393 /// 7. Return value, type, sign.
394 fn consume_a_number(&mut self) -> (f64, bool) {
395 let mut type_is_number = false;
396 let mut number_part = String::new();
397 let mut exponent_part = String::new();
398
399 // §4.3.13 L1436-1440: sign
400 if self.peek(0) == Some('+') || self.peek(0) == Some('-') {
401 number_part.push(self.consume().unwrap());
402 }
403
404 // §4.3.13 L1442-1444: integer digits
405 while let Some(d) = self.peek(0) {
406 if is_digit(d) {
407 number_part.push(d);
408 self.consume();
409 } else {
410 break;
411 }
412 }
413
414 // §4.3.13 L1446-1454: fraction part
415 if self.peek(0) == Some('.') && matches!(self.peek(1), Some(d) if is_digit(d)) {
416 number_part.push(self.consume().unwrap()); // consume `.`
417 type_is_number = true;
418 while let Some(d) = self.peek(0) {
419 if is_digit(d) {
420 number_part.push(d);
421 self.consume();
422 } else {
423 break;
424 }
425 }
426 }
427
428 // §4.3.13 L1457-1469: exponent part
429 if matches!(self.peek(0), Some('e') | Some('E')) {
430 let p1 = self.peek(1);
431 let p2 = self.peek(2);
432 let valid_exp = match p1 {
433 Some('+') | Some('-') => matches!(p2, Some(d) if is_digit(d)),
434 Some(d) if is_digit(d) => true,
435 _ => false,
436 };
437 if valid_exp {
438 self.consume(); // consume `e`/`E`
439 if self.peek(0) == Some('+') || self.peek(0) == Some('-') {
440 exponent_part.push(self.consume().unwrap());
441 }
442 while let Some(d) = self.peek(0) {
443 if is_digit(d) {
444 exponent_part.push(d);
445 self.consume();
446 } else {
447 break;
448 }
449 }
450 type_is_number = true;
451 }
452 }
453
454 // §4.3.13 L1472-1480: compute value
455 let mut value: f64 = number_part.parse().unwrap_or(0.0);
456 if !exponent_part.is_empty() {
457 let exp: i32 = exponent_part.parse().unwrap_or(0);
458 value *= 10.0f64.powi(exp);
459 }
460
461 (value, !type_is_number)
462 }
463
464 /// §4.3.4 (L1045-1078) Consume an ident-like token.
465 ///
466 /// Precondition: the current position is at the start of an ident
467 /// sequence (the caller has not consumed anything; or has reconsumed
468 /// the first code point). Returns an `ident-token`, `function-token`,
469 /// `url-token`, or `bad-url-token`.
470 ///
471 /// Per §4.3.4:
472 /// 1. Consume an ident sequence (§4.3.12), yielding `name`.
473 /// 2. If `name` is ASCII-case-insensitive "url" and next is `(`:
474 /// consume `(`, collapse leading whitespace pairs, then if the
475 /// next 1-2 code points are a quote (or whitespace+quote) return
476 /// a `function-token`; otherwise consume a url token (§4.3.6).
477 /// 3. Else if next is `(`: consume it, return `function-token`.
478 /// 4. Else return `ident-token`.
479 fn consume_an_ident_like_token(&mut self) -> Token {
480 let name = self.consume_an_ident_sequence();
481 // §4.3.4 L1053-1066: url( special case
482 if name.eq_ignore_ascii_case("url") && self.peek(0) == Some('(') {
483 self.consume(); // consume `(`
484 // §4.3.4 L1056-1057: while next two are whitespace, consume one
485 while self.peek(0).is_some_and(is_whitespace) && self.peek(1).is_some_and(is_whitespace)
486 {
487 self.consume();
488 }
489 // §4.3.4 L1058-1063: if next 1-2 are " / ' / ws+" / ws+' → Function
490 let p0 = self.peek(0);
491 let p1 = self.peek(1);
492 let is_quote_case = matches!(p0, Some('"') | Some('\''))
493 || (p0.is_some_and(is_whitespace) && matches!(p1, Some('"') | Some('\'')));
494 if is_quote_case {
495 return Token::Function(name);
496 }
497 // §4.3.4 L1064-1066: otherwise consume a url token
498 return self.consume_a_url_token();
499 }
500 // §4.3.4 L1068-1073: ordinary function
501 if self.peek(0) == Some('(') {
502 self.consume();
503 return Token::Function(name);
504 }
505 // §4.3.4 L1075-1078: ident
506 Token::Ident(name)
507 }
508
509 /// §4.3.4 (at-keyword branch) Consume an at-keyword token.
510 ///
511 /// Precondition: `@` has been consumed. Per §4.3.1, if the next code
512 /// points would start an ident sequence, consume it and return an
513 /// `<at-keyword-token>`; otherwise return a `<delim-token>` with `@`.
514 fn consume_an_at_keyword_token(&mut self) -> Token {
515 if self.would_start_ident_sequence_at(0) {
516 let name = self.consume_an_ident_sequence();
517 Token::AtKeyword(name)
518 } else {
519 Token::Delim('@')
520 }
521 }
522
523 /// §4.3.1 (backslash branch): if would-start-an-escape, consume an
524 /// ident-like token; else emit `<delim-token>`.
525 fn consume_ident_or_delim(&mut self) -> Token {
526 if self.is_valid_escape_next() {
527 self.reconsume();
528 self.consume_an_ident_like_token()
529 } else {
530 Token::Delim('\\')
531 }
532 }
533
534 /// §4.3.6 (L1132-1203) Consume a url token.
535 ///
536 /// Precondition: `url(` has been consumed by the caller (§4.3.4
537 /// L1053-1066). This algorithm consumes the "unquoted" url body and
538 /// returns a `url-token` or `bad-url-token`. A quoted value like
539 /// `url("foo")` is handled by §4.3.4 as a `function-token` and never
540 /// reaches here.
541 ///
542 /// Per §4.3.6:
543 /// - `)` → return url-token.
544 /// - EOF → parse error, return url-token.
545 /// - whitespace → consume trailing whitespace; if next is `)` or EOF
546 /// return url-token, else bad-url.
547 /// - `"` / `'` / `(` / non-printable → parse error, bad-url.
548 /// - `\` → if valid escape, consume escaped and append; else bad-url.
549 /// - anything else → append.
550 fn consume_a_url_token(&mut self) -> Token {
551 let mut value = String::new();
552 // §4.3.6 L1151: consume as much whitespace as possible
553 while self.peek(0).is_some_and(is_whitespace) {
554 self.consume();
555 }
556 loop {
557 match self.consume() {
558 // §4.3.6 L1157-1159: `)` → return url-token
559 Some(')') => return Token::Url(value),
560 // §4.3.6 L1161-1164: EOF → parse error, return url-token
561 None => return Token::Url(value),
562 // §4.3.6 L1166-1175: whitespace → consume trailing ws, then check
563 Some(c) if is_whitespace(c) => {
564 while self.peek(0).is_some_and(is_whitespace) {
565 self.consume();
566 }
567 match self.peek(0) {
568 Some(')') => {
569 self.consume();
570 return Token::Url(value);
571 }
572 None => return Token::Url(value), // parse error
573 _ => {
574 self.consume_the_remnants_of_a_bad_url();
575 return Token::BadUrl;
576 }
577 }
578 }
579 // §4.3.6 L1177-1185: " / ' / ( / non-printable → bad url
580 Some(c) if c == '"' || c == '\'' || c == '(' || is_non_printable(c) => {
581 self.consume_the_remnants_of_a_bad_url();
582 return Token::BadUrl;
583 }
584 // §4.3.6 L1187-1197: `\`
585 Some('\\') => {
586 if self.is_valid_escape_next() {
587 let escaped = self.consume_an_escaped_code_point();
588 value.push(escaped);
589 } else {
590 self.consume_the_remnants_of_a_bad_url();
591 return Token::BadUrl;
592 }
593 }
594 // §4.3.6 L1199-1202: anything else → append
595 Some(c) => value.push(c),
596 }
597 }
598 }
599
600 /// §4.3.15 (L1551-1577) Consume the remnants of a bad url.
601 ///
602 /// Consumes code points until `)` or EOF, allowing an escaped `)`
603 /// (`\)`) to be consumed without ending the bad-url. Per §4.3.15:
604 /// - `)` or EOF → return.
605 /// - valid escape (stream starts with `\` + non-newline non-EOF) →
606 /// consume an escaped code point.
607 /// - anything else → do nothing.
608 fn consume_the_remnants_of_a_bad_url(&mut self) {
609 loop {
610 match self.consume() {
611 // §4.3.15 L1563-1566: `)` or EOF → return
612 Some(')') | None => return,
613 // §4.3.15 L1568-1572: valid escape → consume escaped code point
614 Some('\\') if self.is_valid_escape_next() => {
615 let _ = self.consume_an_escaped_code_point();
616 }
617 // §4.3.15 L1574-1576: anything else → do nothing
618 Some(_) => {}
619 }
620 }
621 }
622
623 /// §4.3.1 (L960-972) U/u branch: if `unicode_ranges_allowed` is true
624 /// and the stream would start a unicode-range (§4.3.11), consume a
625 /// unicode-range token (§4.3.14); otherwise consume an ident-like token.
626 ///
627 /// Precondition: `u`/`U` has been consumed, `pos` points just past it.
628 /// This method reconsumes back to `u` before dispatching.
629 fn consume_u_or_unicode_range(&mut self) -> Token {
630 self.reconsume(); // pos back to `u`/`U`
631 // §4.3.1 L963-967: unicode_ranges_allowed && would-start-unicode-range
632 if self.unicode_ranges_allowed && self.would_start_unicode_range_at(0) {
633 return self.consume_a_unicode_range_token();
634 }
635 // §4.3.1 L969-972: otherwise ident-like
636 self.consume_an_ident_like_token()
637 }
638
639 /// §4.3.11 (L1356-1378) Check if three code points would start a
640 /// unicode-range, examining code points starting at `offset` from the
641 /// current position.
642 ///
643 /// Per §4.3.11: true iff first is `U`/`u`, second is `+`, third is `?`
644 /// or a hex digit.
645 fn would_start_unicode_range_at(&self, offset: usize) -> bool {
646 let first = match self.peek(offset) {
647 Some(c) => c,
648 None => return false,
649 };
650 if !matches!(first, 'U' | 'u') {
651 return false;
652 }
653 if self.peek(offset + 1) != Some('+') {
654 return false;
655 }
656 match self.peek(offset + 2) {
657 Some('?') => true,
658 Some(c) if is_hex_digit(c) => true,
659 _ => false,
660 }
661 }
662
663 /// §4.3.14 (L1487-1548) Consume a unicode-range token.
664 ///
665 /// Precondition: the stream starts with `u`/`U` + `+` + (`?` or hex
666 /// digit) (i.e. [`would_start_unicode_range_at`](Self::would_start_unicode_range_at)
667 /// returns true at the current position).
668 ///
669 /// Returns `UnicodeRange(Some(start), Some(end))`.
670 ///
671 /// Per §4.3.14:
672 /// 1. Consume and discard `u`/`U` and `+`.
673 /// 2. Consume up to 6 hex digits; if fewer than 6, consume `?` up to a
674 /// total of 6 → `first_segment`.
675 /// 3. If `first_segment` contains `?`: start = `?`→`0`, end = `?`→`F`.
676 /// 4. Else start = hex value of `first_segment`.
677 /// 5. If next two are `-` + hex digit: consume `-`, consume up to 6 hex
678 /// → end.
679 /// 6. Else end = start.
680 fn consume_a_unicode_range_token(&mut self) -> Token {
681 // §4.3.14 step 1: consume and discard `u`/`U` + `+`
682 self.consume(); // `u`/`U`
683 self.consume(); // `+`
684
685 // §4.3.14 step 2: consume up to 6 hex digits
686 let mut first_segment = String::new();
687 while first_segment.len() < 6 {
688 match self.peek(0) {
689 Some(c) if is_hex_digit(c) => {
690 first_segment.push(c);
691 self.consume();
692 }
693 _ => break,
694 }
695 }
696 // then consume `?` up to a total of 6
697 while first_segment.len() < 6 {
698 match self.peek(0) {
699 Some('?') => {
700 first_segment.push('?');
701 self.consume();
702 }
703 _ => break,
704 }
705 }
706
707 // §4.3.14 step 3: if first_segment contains `?`
708 if first_segment.contains('?') {
709 let start_str: String = first_segment
710 .chars()
711 .map(|c| if c == '?' { '0' } else { c })
712 .collect();
713 let start = u32::from_str_radix(&start_str, 16).unwrap_or(0);
714 let end_str: String = first_segment
715 .chars()
716 .map(|c| if c == '?' { 'F' } else { c })
717 .collect();
718 let end = u32::from_str_radix(&end_str, 16).unwrap_or(0);
719 return Token::UnicodeRange(Some(start), Some(end));
720 }
721
722 // §4.3.14 step 4: first_segment as hex → start
723 let start = u32::from_str_radix(&first_segment, 16).unwrap_or(0);
724
725 // §4.3.14 step 5: if next two are `-` + hex digit
726 if self.peek(0) == Some('-') && self.peek(1).is_some_and(is_hex_digit) {
727 self.consume(); // consume `-`
728 let mut end_segment = String::new();
729 while end_segment.len() < 6 {
730 match self.peek(0) {
731 Some(c) if is_hex_digit(c) => {
732 end_segment.push(c);
733 self.consume();
734 }
735 _ => break,
736 }
737 }
738 let end = u32::from_str_radix(&end_segment, 16).unwrap_or(0);
739 return Token::UnicodeRange(Some(start), Some(end));
740 }
741
742 // §4.3.14 step 6: otherwise start == end
743 Token::UnicodeRange(Some(start), Some(start))
744 }
745
746 // ── §4.3.12 Consume an ident sequence ────────────────────────────
747
748 /// §4.3.12 Consume an ident sequence.
749 ///
750 /// Repeatedly consume code points as long as they form an ident
751 /// sequence: ident code points, valid escapes (consumed via
752 /// §4.3.7), or `\`-`-` (for `--`-prefixed names). Returns the
753 /// decoded name with escapes resolved.
754 ///
755 /// Per §4.3.12, the loop terminates when:
756 /// - EOF is reached, or
757 /// - the next code point is not an ident code point, not a valid
758 /// escape, and not part of a `\`-`-` continuation.
759 fn consume_an_ident_sequence(&mut self) -> String {
760 let mut result = String::new();
761 loop {
762 let Some(c) = self.consume() else {
763 return result;
764 };
765 if is_ident_code_point(c) {
766 result.push(c);
767 } else if c == '\\' && self.is_valid_escape_next() {
768 // §4.3.12: valid escape → consume the escaped code point
769 // and append it.
770 let escaped = self.consume_an_escaped_code_point();
771 result.push(escaped);
772 } else {
773 // Not part of the ident sequence: reconsume and stop.
774 self.reconsume();
775 return result;
776 }
777 }
778 }
779
780 // ── §4.3.7 Consume an escaped code point ────────────────────────
781
782 /// §4.3.7 Consume an escaped code point.
783 ///
784 /// Precondition: the `\` has been consumed. Returns the decoded code
785 /// point.
786 ///
787 /// Per §4.3.7 (L1207-1240):
788 /// - EOF (L1233-1236) → parse error, return U+FFFD REPLACEMENT CHARACTER.
789 /// - newline → parse error, reconsume (invalid escape; the caller
790 /// should have checked §4.3.8 first, but we handle defensively).
791 /// - hex digit(s) (L1219-1231) → consume 1-6 hex digits, optionally
792 /// followed by whitespace, and return the code point with that
793 /// value. U+0000 NULL or out-of-range → replacement character U+FFFD.
794 /// - anything else (L1238-1240) → return the consumed code point.
795 fn consume_an_escaped_code_point(&mut self) -> char {
796 let Some(c) = self.consume() else {
797 // §4.3.7 L1233-1236: EOF → parse error, return U+FFFD.
798 return '\u{FFFD}';
799 };
800 if is_hex_digit(c) {
801 // §4.3.7 step 4: consume up to 5 more hex digits (total ≤ 6).
802 let mut hex = String::new();
803 hex.push(c);
804 while hex.len() < 6 {
805 match self.peek(0) {
806 Some(next) if is_hex_digit(next) => {
807 hex.push(next);
808 self.consume();
809 }
810 _ => break,
811 }
812 }
813 // §4.3.7 step 4: consume one trailing whitespace.
814 if self.peek(0) == Some(' ') || self.peek(0) == Some('\t') || self.peek(0) == Some('\n')
815 {
816 self.consume();
817 }
818 // Parse the hex value.
819 let value = u32::from_str_radix(&hex, 16).unwrap_or(0);
820 // §4.3.7 step 4: 0 or > 0x10FFFF → U+FFFD. Also surrogates
821 // (0xD800–0xDFFF) → U+FFFD.
822 if value == 0 || value > 0x10FFFF || (0xD800..=0xDFFF).contains(&value) {
823 '\u{FFFD}'
824 } else {
825 char::from_u32(value).unwrap_or('\u{FFFD}')
826 }
827 } else if c == '\n' {
828 // §4.3.7 step 3: newline after `\` is a parse error. This
829 // should not happen if the caller checked §4.3.8, but handle
830 // defensively by returning `\` and reconsuming the newline.
831 self.reconsume();
832 '\\'
833 } else {
834 // §4.3.7 step 5: any other code point → return it.
835 c
836 }
837 }
838
839 // ── §4.3.8 / §4.3.9 Predicates ──────────────────────────────────
840
841 /// §4.3.8 Check if two code points are a valid escape.
842 ///
843 /// Precondition: the `\` has been consumed. Returns true if the next
844 /// code point forms a valid escape with the consumed `\` (i.e. the
845 /// next code point is not a newline).
846 ///
847 /// §4.3.8: EOF as the second code point returns **true** — only a
848 /// newline invalidates the escape. The escaped-EOF path then hits
849 /// §4.3.7's EOF branch and yields U+FFFD (the WPT
850 /// css/css-syntax/escaped-eof.html semantics).
851 fn is_valid_escape_next(&self) -> bool {
852 match self.peek(0) {
853 None => true,
854 Some('\n') => false,
855 Some(_) => true,
856 }
857 }
858
859 /// §4.3.9 Check if three code points would start an ident sequence,
860 /// examining code points starting at `offset` from the current
861 /// position.
862 ///
863 /// Per §4.3.9:
864 /// - U+002D HYPHEN-MINUS (`-`):
865 /// - If the next is also `-`, true (start of `--...`).
866 /// - If the next is an ident-start code point (or a valid escape),
867 /// true.
868 /// - Else false.
869 /// - U+005C REVERSE SOLIDUS (`\`): true if valid escape.
870 /// - ident-start code point: true.
871 /// - Else false.
872 fn would_start_ident_sequence_at(&self, offset: usize) -> bool {
873 let first = match self.peek(offset) {
874 Some(c) => c,
875 None => return false,
876 };
877 match first {
878 '-' => {
879 // §4.3.9: if next is `-`, true.
880 match self.peek(offset + 1) {
881 Some('-') => true,
882 Some(next) if is_ident_start_code_point(next) => true,
883 Some('\\') => {
884 // §4.3.9: if `\` and the following forms a valid escape.
885 self.is_valid_escape_at(offset + 1)
886 }
887 _ => false,
888 }
889 }
890 '\\' => self.is_valid_escape_at(offset),
891 _ if is_ident_start_code_point(first) => true,
892 _ => false,
893 }
894 }
895
896 /// §4.3.8 Check if the code point at `offset` starts a valid escape
897 /// (i.e. `offset` points at `\` and `offset+1` is not newline/EOF).
898 fn is_valid_escape_at(&self, offset: usize) -> bool {
899 if self.peek(offset) != Some('\\') {
900 return false;
901 }
902 match self.peek(offset + 1) {
903 // §4.3.8: EOF as the second code point is a valid escape.
904 None => true,
905 Some('\n') => false,
906 Some(_) => true,
907 }
908 }
909
910 // ── Predicates (stubs returning false; implemented in C-4/C-6) ──
911
912 /// §4.3.10 (L1307-1352) Check if three code points would start a number.
913 ///
914 /// `first` is the code point already consumed (the caller passes it
915 /// so the predicate can examine the next two without reconsume). The
916 /// three code points examined are `first`, `peek(0)`, `peek(1)`.
917 ///
918 /// Per §4.3.10:
919 /// - `+`/`-`: second is digit → true; second is `.` and third is digit
920 /// → true; else false.
921 /// - `.`: second is digit → true; else false.
922 /// - digit: true.
923 /// - anything else: false.
924 fn starts_with_number(&self, first: char) -> bool {
925 match first {
926 '+' | '-' => match self.peek(0) {
927 Some(d) if is_digit(d) => true,
928 Some('.') => matches!(self.peek(1), Some(d) if is_digit(d)),
929 _ => false,
930 },
931 '.' => matches!(self.peek(0), Some(d) if is_digit(d)),
932 d if is_digit(d) => true,
933 _ => false,
934 }
935 }
936}
937
938// ── Free functions (§4.2 code point classes) ──────────────────────────
939
940/// §4.2 Whether `c` is an ident-start code point.
941///
942/// An ident-start code point is an ASCII letter, a non-ASCII ident code
943/// point, or U+005F LOW LINE (`_`).
944fn is_ident_start_code_point(c: char) -> bool {
945 matches!(c, 'A'..='Z' | 'a'..='z' | '_') || is_non_ascii_ident_code_point(c)
946}
947
948/// §4.2 Whether `c` is an ident code point (allowed after the first).
949///
950/// An ident code point is an ident-start code point, or an ASCII digit,
951/// or U+002D HYPHEN-MINUS (`-`).
952fn is_ident_code_point(c: char) -> bool {
953 matches!(c, '0'..='9' | '-') || is_ident_start_code_point(c)
954}
955
956/// §4.2 Whether `c` is a non-ASCII ident code point.
957///
958/// The spec defines these as an explicit allowlist of ranges (aligned
959/// with the code points allowed in HTML custom element names) — *not*
960/// "any code point ≥ U+0080". Notably U+0080–U+00B6 and U+00B8–U+00BF
961/// are excluded (WPT css/css-syntax/non-ascii-codepoints.html).
962fn is_non_ascii_ident_code_point(c: char) -> bool {
963 matches!(c,
964 '\u{00B7}'
965 | '\u{00C0}'..='\u{00D6}'
966 | '\u{00D8}'..='\u{00F6}'
967 | '\u{00F8}'..='\u{037D}'
968 | '\u{037F}'..='\u{1FFF}'
969 | '\u{200C}' | '\u{200D}'
970 | '\u{203F}' | '\u{2040}'
971 | '\u{2070}'..='\u{218F}'
972 | '\u{2C00}'..='\u{2FEF}'
973 | '\u{3001}'..='\u{D7FF}'
974 | '\u{F900}'..='\u{FDCF}'
975 | '\u{FDF0}'..='\u{FFFD}'
976 | '\u{10000}'..=char::MAX)
977}
978
979/// §4.2 Whether `c` is a digit (ASCII 0-9).
980fn is_digit(c: char) -> bool {
981 c.is_ascii_digit()
982}
983
984/// §4.2 Whether `c` is a hex digit (ASCII 0-9, A-F, a-f).
985fn is_hex_digit(c: char) -> bool {
986 c.is_ascii_hexdigit()
987}
988
989/// §4.2 Whether `c` is a whitespace code point.
990///
991/// Per §4.2: U+0009 TAB, U+000A LF, U+000C FF, U+000D CR, U+0020 SPACE.
992/// (After §5.3 preprocessing, U+000C FF and lone U+000D CR have been
993/// normalized to U+000A LF, but the helper follows the full §4.2
994/// definition for correctness when called on arbitrary input.)
995fn is_whitespace(c: char) -> bool {
996 matches!(c, '\t' | '\n' | '\u{000C}' | '\r' | ' ')
997}
998
999/// §4.2 Whether `c` is a non-printable code point.
1000///
1001/// Per §4.2: U+0000-U+0008, U+000B, U+000E-U+001F, U+007F-U+009F.
1002fn is_non_printable(c: char) -> bool {
1003 matches!(
1004 c,
1005 '\u{0000}'..='\u{0008}' | '\u{000B}' | '\u{000E}'..='\u{001F}' | '\u{007F}'..='\u{009F}'
1006 )
1007}
1008
1009/// §5.3 input preprocessing.
1010///
1011/// Per the "input stream" definition in §5.3:
1012/// - U+000D CR followed by U+000A LF → single U+000A LF
1013/// - lone U+000D CR → U+000A LF
1014/// - U+000C FF → U+000A LF
1015///
1016/// The tokenizer operates on this normalized stream.
1017fn preprocess_input(s: &str) -> Vec<char> {
1018 let mut out = Vec::with_capacity(s.len());
1019 let mut chars = s.chars().peekable();
1020 while let Some(c) = chars.next() {
1021 match c {
1022 '\r' => {
1023 if chars.peek() == Some(&'\n') {
1024 chars.next();
1025 }
1026 out.push('\n');
1027 }
1028 '\u{000C}' => out.push('\n'),
1029 // §5.3: NULL is replaced with U+FFFD during preprocessing.
1030 '\u{0000}' => out.push('\u{FFFD}'),
1031 _ => out.push(c),
1032 }
1033 }
1034 out
1035}
1036
1037impl Tokenizer for CssTokenizer {
1038 fn next_token(&mut self) -> Option<Token> {
1039 if self.eof_emitted {
1040 return None;
1041 }
1042 Some(self.consume_a_token())
1043 }
1044
1045 fn state(&self) -> State {
1046 self.state
1047 }
1048
1049 fn set_state(&mut self, state: State) {
1050 self.state = state;
1051 if state == State::Data {
1052 // Resetting to Data after EOF re-enables the iterator; useful
1053 // for `reset()` callers that want to re-tokenize.
1054 self.eof_emitted = false;
1055 }
1056 }
1057
1058 fn reset(&mut self) {
1059 self.pos = 0;
1060 self.state = State::Data;
1061 self.eof_emitted = false;
1062 }
1063
1064 fn set_unicode_ranges_allowed(&mut self, allowed: bool) {
1065 self.unicode_ranges_allowed = allowed;
1066 }
1067
1068 fn position(&self) -> usize {
1069 self.pos
1070 }
1071}
1072
1073// ── Test helpers exposed for unit tests in this file ──────────────────
1074
1075#[cfg(test)]
1076impl CssTokenizer {
1077 /// Tokenize the entire input and return all tokens except the
1078 /// trailing `<EOF-token>`. Test-only convenience.
1079 fn collect(input: &str) -> Vec<Token> {
1080 let mut tz = Self::new(input);
1081 let mut out = Vec::new();
1082 while let Some(t) = tz.next_token() {
1083 if matches!(t, Token::Eof) {
1084 break;
1085 }
1086 out.push(t);
1087 }
1088 out
1089 }
1090}
1091
1092#[cfg(test)]
1093mod tests {
1094 use super::*;
1095
1096 #[test]
1097 fn empty_input_emits_eof_only() {
1098 let mut tz = CssTokenizer::new("");
1099 assert!(matches!(tz.next_token(), Some(Token::Eof)));
1100 assert!(tz.next_token().is_none());
1101 }
1102
1103 #[test]
1104 fn whitespace_collapses_runs() {
1105 let tokens = CssTokenizer::collect(" \t\n ");
1106 assert_eq!(tokens.len(), 1, "whitespace run → single token");
1107 assert!(matches!(tokens[0], Token::Whitespace));
1108 }
1109
1110 #[test]
1111 fn simple_punctuation_tokens() {
1112 let tokens = CssTokenizer::collect(":;,.{}[]()");
1113 assert_eq!(tokens.len(), 10);
1114 assert!(matches!(tokens[0], Token::Colon));
1115 assert!(matches!(tokens[1], Token::Semicolon));
1116 assert!(matches!(tokens[2], Token::Comma));
1117 assert!(matches!(tokens[3], Token::Delim('.')));
1118 assert!(matches!(tokens[4], Token::OpenBrace));
1119 assert!(matches!(tokens[5], Token::CloseBrace));
1120 assert!(matches!(tokens[6], Token::OpenBracket));
1121 assert!(matches!(tokens[7], Token::CloseBracket));
1122 assert!(matches!(tokens[8], Token::OpenParen));
1123 assert!(matches!(tokens[9], Token::CloseParen));
1124 }
1125
1126 #[test]
1127 fn close_paren_is_emitted() {
1128 let tokens = CssTokenizer::collect(")");
1129 assert_eq!(tokens.len(), 1);
1130 assert!(matches!(tokens[0], Token::CloseParen));
1131 }
1132
1133 #[test]
1134 fn unknown_delim_chars() {
1135 let tokens = CssTokenizer::collect("^~`>");
1136 assert_eq!(tokens.len(), 4);
1137 assert!(matches!(tokens[0], Token::Delim('^')));
1138 assert!(matches!(tokens[1], Token::Delim('~')));
1139 assert!(matches!(tokens[2], Token::Delim('`')));
1140 assert!(matches!(tokens[3], Token::Delim('>')));
1141 }
1142
1143 #[test]
1144 fn preprocessing_normalizes_cr_lf_ff() {
1145 // CR LF → single LF
1146 let tz = CssTokenizer::new("a\r\nb");
1147 assert_eq!(tz.input, vec!['a', '\n', 'b']);
1148 // Lone CR → LF
1149 let tz = CssTokenizer::new("a\rb");
1150 assert_eq!(tz.input, vec!['a', '\n', 'b']);
1151 // FF → LF
1152 let tz = CssTokenizer::new("a\u{000C}b");
1153 assert_eq!(tz.input, vec!['a', '\n', 'b']);
1154 }
1155
1156 #[test]
1157 fn eof_emitted_once() {
1158 // C-1: use a non-ident-start, non-digit char to avoid the
1159 // ident-like / numeric branches (which are stubbed with todo!()
1160 // until C-2 / C-4).
1161 let mut tz = CssTokenizer::new(",");
1162 let t1 = tz.next_token();
1163 let t2 = tz.next_token();
1164 let t3 = tz.next_token();
1165 assert!(matches!(t1, Some(Token::Comma)));
1166 assert!(matches!(t2, Some(Token::Eof)));
1167 assert!(t3.is_none());
1168 }
1169
1170 #[test]
1171 fn block_comment_is_skipped() {
1172 // §4.3.2: `/* ... */` is consumed and no token is emitted.
1173 let tokens = CssTokenizer::collect("/* hello */:/*x*/;");
1174 assert_eq!(tokens.len(), 2);
1175 assert!(matches!(tokens[0], Token::Colon));
1176 assert!(matches!(tokens[1], Token::Semicolon));
1177 }
1178
1179 #[test]
1180 fn unterminated_comment_consumes_to_eof() {
1181 // §4.3.2: EOF before `*/` is a parse error; we consume to EOF.
1182 let tokens = CssTokenizer::collect("/* unfinished");
1183 assert_eq!(tokens.len(), 0, "unterminated comment → no tokens");
1184 }
1185
1186 #[test]
1187 fn comment_between_tokens() {
1188 // C-2 not yet: `a`/`b` would be ident tokens (todo!()). Verify
1189 // comment is consumed without affecting surrounding simple tokens
1190 // by using only punctuation.
1191 let tokens = CssTokenizer::collect(":/*c*/;");
1192 assert_eq!(tokens.len(), 2);
1193 assert!(matches!(tokens[0], Token::Colon));
1194 assert!(matches!(tokens[1], Token::Semicolon));
1195 }
1196
1197 #[test]
1198 fn cdo_token_emitted() {
1199 // §4.3.1: `<!--` → <CDO-token>
1200 let tokens = CssTokenizer::collect("<!--");
1201 assert_eq!(tokens.len(), 1);
1202 assert!(matches!(tokens[0], Token::Cdo));
1203 }
1204
1205 #[test]
1206 fn cdc_token_emitted() {
1207 // §4.3.1: `-->` → <CDC-token>
1208 let tokens = CssTokenizer::collect("-->");
1209 assert_eq!(tokens.len(), 1);
1210 assert!(matches!(tokens[0], Token::Cdc));
1211 }
1212
1213 #[test]
1214 fn cdo_cdc_wrapping_stylesheet() {
1215 // Legacy CSS 1/2.1 stylesheet wrapping.
1216 // `<!-- : ; -->`
1217 // tokens: Cdo, WS, Colon, WS, Semicolon, WS, Cdc = 7
1218 let tokens = CssTokenizer::collect("<!-- : ; -->");
1219 assert_eq!(tokens.len(), 7);
1220 assert!(matches!(tokens[0], Token::Cdo));
1221 assert!(matches!(tokens[1], Token::Whitespace));
1222 assert!(matches!(tokens[2], Token::Colon));
1223 assert!(matches!(tokens[3], Token::Whitespace));
1224 assert!(matches!(tokens[4], Token::Semicolon));
1225 assert!(matches!(tokens[5], Token::Whitespace));
1226 assert!(matches!(tokens[6], Token::Cdc));
1227 }
1228
1229 #[test]
1230 fn less_than_alone_is_delim() {
1231 // §4.3.1: `<` not followed by `!-` → <delim-token>
1232 let tokens = CssTokenizer::collect("< ");
1233 assert_eq!(tokens.len(), 2);
1234 assert!(matches!(tokens[0], Token::Delim('<')));
1235 assert!(matches!(tokens[1], Token::Whitespace));
1236 }
1237
1238 #[test]
1239 fn hyphen_alone_is_delim() {
1240 // §4.3.1: `-` not followed by number/ident/`->` → <delim-token>
1241 let tokens = CssTokenizer::collect("- ");
1242 assert_eq!(tokens.len(), 2);
1243 assert!(matches!(tokens[0], Token::Delim('-')));
1244 assert!(matches!(tokens[1], Token::Whitespace));
1245 }
1246
1247 // ── C-2 tests: ident / function / at-keyword / hash ──────────────
1248
1249 #[test]
1250 fn simple_ident() {
1251 let tokens = CssTokenizer::collect("color");
1252 assert_eq!(tokens.len(), 1);
1253 assert!(matches!(&tokens[0], Token::Ident(s) if s == "color"));
1254 }
1255
1256 #[test]
1257 fn ident_with_hyphen_prefix() {
1258 let tokens = CssTokenizer::collect("-webkit-flex");
1259 assert_eq!(tokens.len(), 1);
1260 assert!(matches!(&tokens[0], Token::Ident(s) if s == "-webkit-flex"));
1261 }
1262
1263 #[test]
1264 fn ident_with_double_hyphen() {
1265 // §4.3.9: `--` starts an ident sequence (custom properties).
1266 let tokens = CssTokenizer::collect("--my-var");
1267 assert_eq!(tokens.len(), 1);
1268 assert!(matches!(&tokens[0], Token::Ident(s) if s == "--my-var"));
1269 }
1270
1271 #[test]
1272 fn ident_with_underscore() {
1273 let tokens = CssTokenizer::collect("_private");
1274 assert_eq!(tokens.len(), 1);
1275 assert!(matches!(&tokens[0], Token::Ident(s) if s == "_private"));
1276 }
1277
1278 #[test]
1279 fn ident_with_escape() {
1280 // §4.3.7: `\26` → '&' (U+0026)
1281 let tokens = CssTokenizer::collect("color\\26 B");
1282 assert_eq!(tokens.len(), 1);
1283 assert!(matches!(&tokens[0], Token::Ident(s) if s == "color&B"));
1284 }
1285
1286 #[test]
1287 fn function_token() {
1288 let tokens = CssTokenizer::collect("translate(");
1289 assert_eq!(tokens.len(), 1);
1290 assert!(matches!(&tokens[0], Token::Function(s) if s == "translate"));
1291 }
1292
1293 #[test]
1294 fn at_keyword_token() {
1295 let tokens = CssTokenizer::collect("@media");
1296 assert_eq!(tokens.len(), 1);
1297 assert!(matches!(&tokens[0], Token::AtKeyword(s) if s == "media"));
1298 }
1299
1300 #[test]
1301 fn at_keyword_with_hyphen() {
1302 let tokens = CssTokenizer::collect("@-webkit-keyframes");
1303 assert_eq!(tokens.len(), 1);
1304 assert!(matches!(&tokens[0], Token::AtKeyword(s) if s == "-webkit-keyframes"));
1305 }
1306
1307 #[test]
1308 fn at_sign_alone_is_delim() {
1309 // §4.3.1: `@` not followed by ident-start → <delim-token>
1310 let tokens = CssTokenizer::collect("@ ");
1311 assert_eq!(tokens.len(), 2);
1312 assert!(matches!(tokens[0], Token::Delim('@')));
1313 assert!(matches!(tokens[1], Token::Whitespace));
1314 }
1315
1316 #[test]
1317 fn hash_id_type() {
1318 // §4.3.1 / §4.3.4: `#main` → <hash-token> with type "id"
1319 let tokens = CssTokenizer::collect("#main");
1320 assert_eq!(tokens.len(), 1);
1321 match &tokens[0] {
1322 Token::Hash(s, t) => {
1323 assert_eq!(s, "main");
1324 assert_eq!(*t, crate::types::HashType::Id);
1325 }
1326 _ => panic!("expected Hash, got {:?}", tokens[0]),
1327 }
1328 }
1329
1330 #[test]
1331 fn hash_unrestricted_type() {
1332 // §4.3.1 L801-826 + §4.3.4: `#123` → next `1` is an ident code
1333 // point (digits are ident code points per §4.2), so a hash-token
1334 // is created. `123` does not would-start-an-ident-sequence (digit
1335 // is not ident-start), so type is "unrestricted". The value is
1336 // "123" (consume_an_ident_sequence consumes all ident code points).
1337 let tokens = CssTokenizer::collect("#123");
1338 assert_eq!(tokens.len(), 1);
1339 match &tokens[0] {
1340 Token::Hash(s, t) => {
1341 assert_eq!(s, "123");
1342 assert_eq!(*t, crate::types::HashType::Unrestricted);
1343 }
1344 _ => panic!("expected Hash, got {:?}", tokens[0]),
1345 }
1346 }
1347
1348 #[test]
1349 fn hash_hex_color_is_unrestricted() {
1350 // §4.3.4: `#ff00aa` → starts with `f` which IS ident-start, so
1351 // this is actually type "id" (the hex digits form a valid ident).
1352 let tokens = CssTokenizer::collect("#ff00aa");
1353 assert_eq!(tokens.len(), 1);
1354 match &tokens[0] {
1355 Token::Hash(s, t) => {
1356 assert_eq!(s, "ff00aa");
1357 assert_eq!(*t, crate::types::HashType::Id);
1358 }
1359 _ => panic!("expected Hash, got {:?}", tokens[0]),
1360 }
1361 }
1362
1363 #[test]
1364 fn backslash_escape_in_ident() {
1365 // §4.3.7: `\` followed by non-hex → literal char. Use `z` (not
1366 // `b`, which is a hex digit and would decode to U+000B).
1367 let tokens = CssTokenizer::collect("a\\z");
1368 assert_eq!(tokens.len(), 1);
1369 assert!(matches!(&tokens[0], Token::Ident(s) if s == "az"));
1370 }
1371
1372 #[test]
1373 fn backslash_at_eof_yields_fffd_ident() {
1374 // §4.3.8: EOF as the second code point is a valid escape;
1375 // §4.3.7: escaped-EOF returns U+FFFD. So a lone `\` at EOF
1376 // produces an <ident-token> whose value is U+FFFD (WPT
1377 // css/css-syntax/escaped-eof.html: "Escaped EOF turns into a
1378 // U+FFFD in an ident token").
1379 let tokens = CssTokenizer::collect("\\");
1380 assert_eq!(tokens.len(), 1);
1381 assert_eq!(tokens[0], Token::Ident("\u{FFFD}".to_string()));
1382 }
1383
1384 #[test]
1385 fn backslash_newline_is_delim() {
1386 // §4.3.8: `\` followed by newline is not a valid escape.
1387 let tokens = CssTokenizer::collect("\\\n");
1388 assert_eq!(tokens.len(), 2);
1389 assert!(matches!(tokens[0], Token::Delim('\\')));
1390 assert!(matches!(tokens[1], Token::Whitespace));
1391 }
1392
1393 #[test]
1394 fn declaration_with_ident_and_value() {
1395 let tokens = CssTokenizer::collect("color: red");
1396 assert_eq!(tokens.len(), 4);
1397 assert!(matches!(&tokens[0], Token::Ident(s) if s == "color"));
1398 assert!(matches!(tokens[1], Token::Colon));
1399 assert!(matches!(tokens[2], Token::Whitespace));
1400 assert!(matches!(&tokens[3], Token::Ident(s) if s == "red"));
1401 }
1402
1403 #[test]
1404 fn unicode_escape_in_ident() {
1405 // §4.3.7 L1223-1225: `\000026` → '&' (U+0026); after 6 hex digits,
1406 // if next is whitespace, consume it. So `\000026 B` → the space is
1407 // consumed as trailing whitespace, `B` joins the ident sequence →
1408 // single token Ident("&B").
1409 let tokens = CssTokenizer::collect("\\000026 B");
1410 assert_eq!(tokens.len(), 1);
1411 assert!(matches!(&tokens[0], Token::Ident(s) if s == "&B"));
1412 }
1413
1414 #[test]
1415 fn null_escape_becomes_replacement_char() {
1416 // §4.3.7: `\0` → U+FFFD (NULL is not allowed)
1417 let tokens = CssTokenizer::collect("\\0");
1418 assert_eq!(tokens.len(), 1);
1419 assert!(matches!(&tokens[0], Token::Ident(s) if s == "\u{FFFD}"));
1420 }
1421
1422 #[test]
1423 fn surrogate_escape_becomes_replacement_char() {
1424 // §4.3.7: `\D800` → U+FFFD (surrogates not allowed)
1425 let tokens = CssTokenizer::collect("\\D800");
1426 assert_eq!(tokens.len(), 1);
1427 assert!(matches!(&tokens[0], Token::Ident(s) if s == "\u{FFFD}"));
1428 }
1429
1430 // ── C-2 Bug 1 回归测试: # Delim 回退 (§4.3.1 L824-826) ────────────
1431
1432 #[test]
1433 fn hash_alone_is_delim() {
1434 // §4.3.1 L824-826: `#` followed by EOF → next is not ident code
1435 // point, not valid escape → return delim-token with value `#`.
1436 let tokens = CssTokenizer::collect("#");
1437 assert_eq!(tokens.len(), 1);
1438 assert!(matches!(tokens[0], Token::Delim('#')));
1439 }
1440
1441 #[test]
1442 fn hash_followed_by_space_is_delim() {
1443 // §4.3.1 L824-826: `#` followed by space (not ident code point,
1444 // not valid escape) → delim-token with value `#`.
1445 let tokens = CssTokenizer::collect("# ");
1446 assert_eq!(tokens.len(), 2);
1447 assert!(matches!(tokens[0], Token::Delim('#')));
1448 assert!(matches!(tokens[1], Token::Whitespace));
1449 }
1450
1451 #[test]
1452 fn hash_followed_by_at_is_delim() {
1453 // §4.3.1 L824-826: `#` followed by `@` (not ident code point,
1454 // not valid escape) → delim-token with value `#`.
1455 let tokens = CssTokenizer::collect("#@");
1456 assert_eq!(tokens.len(), 2);
1457 assert!(matches!(tokens[0], Token::Delim('#')));
1458 assert!(matches!(tokens[1], Token::Delim('@')));
1459 }
1460
1461 // ── C-3 tests: §4.3.5 string + bad-string ────────────────────────
1462
1463 #[test]
1464 fn string_double_quoted() {
1465 // §4.3.5: "hello" → String("hello")
1466 let tokens = CssTokenizer::collect("\"hello\"");
1467 assert_eq!(tokens.len(), 1);
1468 assert!(matches!(&tokens[0], Token::String(s) if s == "hello"));
1469 }
1470
1471 #[test]
1472 fn string_single_quoted() {
1473 // §4.3.5: 'hello' → String("hello")
1474 let tokens = CssTokenizer::collect("'hello'");
1475 assert_eq!(tokens.len(), 1);
1476 assert!(matches!(&tokens[0], Token::String(s) if s == "hello"));
1477 }
1478
1479 #[test]
1480 fn string_with_escape() {
1481 // §4.3.5 + §4.3.7: "a\z" → String("az") (z is non-hex → literal)
1482 let tokens = CssTokenizer::collect("\"a\\z\"");
1483 assert_eq!(tokens.len(), 1);
1484 assert!(matches!(&tokens[0], Token::String(s) if s == "az"));
1485 }
1486
1487 #[test]
1488 fn string_unterminated_eof() {
1489 // §4.3.5 L1101-1104: EOF before closing quote → parse error,
1490 // return string-token with value accumulated so far.
1491 let tokens = CssTokenizer::collect("\"hello");
1492 assert_eq!(tokens.len(), 1);
1493 assert!(matches!(&tokens[0], Token::String(s) if s == "hello"));
1494 }
1495
1496 #[test]
1497 fn string_unescaped_newline_is_bad_string() {
1498 // §4.3.5 L1106-1110: unescaped newline → parse error, reconsume,
1499 // return bad-string. The reconsumed newline is then tokenized as
1500 // whitespace by the next consume_a_token call.
1501 let tokens = CssTokenizer::collect("\"\n");
1502 assert_eq!(tokens.len(), 2);
1503 assert!(matches!(tokens[0], Token::BadString));
1504 assert!(matches!(tokens[1], Token::Whitespace));
1505 }
1506
1507 #[test]
1508 fn string_line_continuation() {
1509 // §4.3.5 L1117-1119: backslash followed by newline → line
1510 // continuation (newline consumed, no char appended).
1511 let tokens = CssTokenizer::collect("\"a\\\nb\"");
1512 assert_eq!(tokens.len(), 1);
1513 assert!(matches!(&tokens[0], Token::String(s) if s == "ab"));
1514 }
1515
1516 #[test]
1517 fn string_hex_escape() {
1518 // §4.3.5 + §4.3.7: "\26" → String("&") (U+0026)
1519 let tokens = CssTokenizer::collect("\"\\26\"");
1520 assert_eq!(tokens.len(), 1);
1521 assert!(matches!(&tokens[0], Token::String(s) if s == "&"));
1522 }
1523
1524 // ── C-4 tests: §4.3.3/§4.3.10/§4.3.13 number/percentage/dimension ─
1525
1526 #[test]
1527 fn number_integer() {
1528 let tokens = CssTokenizer::collect("42");
1529 assert_eq!(tokens.len(), 1);
1530 assert_eq!(
1531 tokens[0],
1532 Token::Number(Numeric {
1533 value: 42.0,
1534 is_integer: true
1535 })
1536 );
1537 }
1538
1539 #[test]
1540 fn number_decimal() {
1541 let tokens = CssTokenizer::collect("3.5");
1542 assert_eq!(tokens.len(), 1);
1543 assert_eq!(
1544 tokens[0],
1545 Token::Number(Numeric {
1546 value: 3.5,
1547 is_integer: false
1548 })
1549 );
1550 }
1551
1552 #[test]
1553 fn number_signed() {
1554 let tokens = CssTokenizer::collect("-5");
1555 assert_eq!(tokens.len(), 1);
1556 assert_eq!(
1557 tokens[0],
1558 Token::Number(Numeric {
1559 value: -5.0,
1560 is_integer: true
1561 })
1562 );
1563 }
1564
1565 #[test]
1566 fn number_exponent() {
1567 // §4.3.13: 1e3 → 1.0 * 10^3 = 1000.0, type "number"
1568 let tokens = CssTokenizer::collect("1e3");
1569 assert_eq!(tokens.len(), 1);
1570 assert_eq!(
1571 tokens[0],
1572 Token::Number(Numeric {
1573 value: 1000.0,
1574 is_integer: false
1575 })
1576 );
1577 }
1578
1579 #[test]
1580 fn number_decimal_exponent() {
1581 // §4.3.13: 1.5e2 → 1.5 * 10^2 = 150.0, type "number"
1582 let tokens = CssTokenizer::collect("1.5e2");
1583 assert_eq!(tokens.len(), 1);
1584 assert_eq!(
1585 tokens[0],
1586 Token::Number(Numeric {
1587 value: 150.0,
1588 is_integer: false
1589 })
1590 );
1591 }
1592
1593 #[test]
1594 fn percentage_token() {
1595 let tokens = CssTokenizer::collect("50%");
1596 assert_eq!(tokens.len(), 1);
1597 assert_eq!(
1598 tokens[0],
1599 Token::Percentage(Numeric {
1600 value: 50.0,
1601 is_integer: true
1602 })
1603 );
1604 }
1605
1606 #[test]
1607 fn dimension_px() {
1608 let tokens = CssTokenizer::collect("10px");
1609 assert_eq!(tokens.len(), 1);
1610 assert_eq!(
1611 tokens[0],
1612 Token::Dimension(
1613 Numeric {
1614 value: 10.0,
1615 is_integer: true
1616 },
1617 "px".to_string(),
1618 )
1619 );
1620 }
1621
1622 #[test]
1623 fn dimension_em() {
1624 let tokens = CssTokenizer::collect("1.5em");
1625 assert_eq!(tokens.len(), 1);
1626 assert_eq!(
1627 tokens[0],
1628 Token::Dimension(
1629 Numeric {
1630 value: 1.5,
1631 is_integer: false
1632 },
1633 "em".to_string(),
1634 )
1635 );
1636 }
1637
1638 #[test]
1639 fn dimension_signed() {
1640 let tokens = CssTokenizer::collect("-30deg");
1641 assert_eq!(tokens.len(), 1);
1642 assert_eq!(
1643 tokens[0],
1644 Token::Dimension(
1645 Numeric {
1646 value: -30.0,
1647 is_integer: true
1648 },
1649 "deg".to_string(),
1650 )
1651 );
1652 }
1653
1654 #[test]
1655 fn plus_sign_number() {
1656 let tokens = CssTokenizer::collect("+5");
1657 assert_eq!(tokens.len(), 1);
1658 assert_eq!(
1659 tokens[0],
1660 Token::Number(Numeric {
1661 value: 5.0,
1662 is_integer: true
1663 })
1664 );
1665 }
1666
1667 #[test]
1668 fn dot_starts_number() {
1669 let tokens = CssTokenizer::collect(".5");
1670 assert_eq!(tokens.len(), 1);
1671 assert_eq!(
1672 tokens[0],
1673 Token::Number(Numeric {
1674 value: 0.5,
1675 is_integer: false
1676 })
1677 );
1678 }
1679
1680 // ── C-5 tests: §4.3.4 url( special case + §4.3.6 Url + §4.3.15 BadUrl ─
1681
1682 #[test]
1683 fn url_unquoted_simple() {
1684 // §4.3.6: url(foo) → Url("foo")
1685 let tokens = CssTokenizer::collect("url(foo)");
1686 assert_eq!(tokens.len(), 1);
1687 assert_eq!(tokens[0], Token::Url("foo".to_string()));
1688 }
1689
1690 #[test]
1691 fn url_unquoted_with_spaces() {
1692 // §4.3.6: leading/trailing whitespace consumed; url( foo ) → Url("foo")
1693 let tokens = CssTokenizer::collect("url( foo )");
1694 assert_eq!(tokens.len(), 1);
1695 assert_eq!(tokens[0], Token::Url("foo".to_string()));
1696 }
1697
1698 #[test]
1699 fn url_empty() {
1700 // §4.3.6: url() → Url("")
1701 let tokens = CssTokenizer::collect("url()");
1702 assert_eq!(tokens.len(), 1);
1703 assert_eq!(tokens[0], Token::Url(String::new()));
1704 }
1705
1706 #[test]
1707 fn url_eof_unterminated() {
1708 // §4.3.6 L1161-1164: EOF → parse error, return url-token with value so far
1709 let tokens = CssTokenizer::collect("url(foo");
1710 assert_eq!(tokens.len(), 1);
1711 assert_eq!(tokens[0], Token::Url("foo".to_string()));
1712 }
1713
1714 #[test]
1715 fn url_quoted_is_function() {
1716 // §4.3.4 L1058-1063: url("foo") → Function("url"); the body
1717 // `"foo")` remains in the stream → String("foo") + CloseParen.
1718 let tokens = CssTokenizer::collect("url(\"foo\")");
1719 assert_eq!(tokens.len(), 3);
1720 assert_eq!(tokens[0], Token::Function("url".to_string()));
1721 assert_eq!(tokens[1], Token::String("foo".to_string()));
1722 assert_eq!(tokens[2], Token::CloseParen);
1723 }
1724
1725 #[test]
1726 fn url_single_quoted_is_function() {
1727 // §4.3.4 L1058-1063: url('foo') → Function("url") + String + CloseParen
1728 let tokens = CssTokenizer::collect("url('foo')");
1729 assert_eq!(tokens.len(), 3);
1730 assert_eq!(tokens[0], Token::Function("url".to_string()));
1731 assert_eq!(tokens[1], Token::String("foo".to_string()));
1732 assert_eq!(tokens[2], Token::CloseParen);
1733 }
1734
1735 #[test]
1736 fn url_ws_then_quote_is_function() {
1737 // §4.3.4 L1058-1063: url( "foo") → whitespace+quote → Function.
1738 // L1056-1057 only collapses when *two* consecutive whitespace
1739 // follow `(`; here only one space precedes `"`, so it is NOT
1740 // consumed and becomes a Whitespace token.
1741 let tokens = CssTokenizer::collect("url( \"foo\")");
1742 assert_eq!(tokens.len(), 4);
1743 assert_eq!(tokens[0], Token::Function("url".to_string()));
1744 assert_eq!(tokens[1], Token::Whitespace);
1745 assert_eq!(tokens[2], Token::String("foo".to_string()));
1746 assert_eq!(tokens[3], Token::CloseParen);
1747 }
1748
1749 #[test]
1750 fn url_with_escape() {
1751 // §4.3.6 + §4.3.7: url(foo\29 bar) → \29 consumes trailing space,
1752 // decodes to `)`, then `bar` appended → Url("foo)bar")
1753 let tokens = CssTokenizer::collect("url(foo\\29 bar)");
1754 assert_eq!(tokens.len(), 1);
1755 assert_eq!(tokens[0], Token::Url("foo)bar".to_string()));
1756 }
1757
1758 #[test]
1759 fn url_bad_paren_in_unquoted() {
1760 // §4.3.6 L1177-1185: `(` in unquoted url → BadUrl
1761 let tokens = CssTokenizer::collect("url(foo(bar)");
1762 assert_eq!(tokens.len(), 1);
1763 assert_eq!(tokens[0], Token::BadUrl);
1764 }
1765
1766 #[test]
1767 fn url_bad_quote_in_unquoted() {
1768 // §4.3.6 L1177-1185: `"` in unquoted url → BadUrl
1769 let tokens = CssTokenizer::collect("url(foo\"bar)");
1770 assert_eq!(tokens.len(), 1);
1771 assert_eq!(tokens[0], Token::BadUrl);
1772 }
1773
1774 // ── C-6 tests: §4.3.1 unicode_ranges_allowed + §4.3.11 + §4.3.14 ──
1775
1776 #[test]
1777 fn unicode_range_disabled_by_default() {
1778 // §4.3.1 L782-783: default unicode_ranges_allowed=false.
1779 // `U+1234` → `U` ident-start → Ident("U") (`+` stops ident seq);
1780 // then `+1234` → starts_with_number (`+`+digit) → Number(1234).
1781 let tokens = CssTokenizer::collect("U+1234");
1782 assert_eq!(tokens.len(), 2);
1783 assert_eq!(tokens[0], Token::Ident("U".to_string()));
1784 assert_eq!(
1785 tokens[1],
1786 Token::Number(Numeric {
1787 value: 1234.0,
1788 is_integer: true
1789 })
1790 );
1791 }
1792
1793 #[test]
1794 fn unicode_range_simple() {
1795 // §4.3.14: U+1234 → UnicodeRange(0x1234, 0x1234)
1796 let mut tz = CssTokenizer::new("U+1234");
1797 tz.set_unicode_ranges_allowed(true);
1798 let t = tz.next_token().unwrap();
1799 assert_eq!(t, Token::UnicodeRange(Some(0x1234), Some(0x1234)));
1800 }
1801
1802 #[test]
1803 fn unicode_range_question_marks() {
1804 // §4.3.14 step 3: U+12?? → start=0x1200 (?→0), end=0x12FF (?→F)
1805 let mut tz = CssTokenizer::new("U+12??");
1806 tz.set_unicode_ranges_allowed(true);
1807 let t = tz.next_token().unwrap();
1808 assert_eq!(t, Token::UnicodeRange(Some(0x1200), Some(0x12FF)));
1809 }
1810
1811 #[test]
1812 fn unicode_range_range() {
1813 // §4.3.14 step 5: U+12-34FF → start=0x12, end=0x34FF
1814 let mut tz = CssTokenizer::new("U+12-34FF");
1815 tz.set_unicode_ranges_allowed(true);
1816 let t = tz.next_token().unwrap();
1817 assert_eq!(t, Token::UnicodeRange(Some(0x12), Some(0x34FF)));
1818 }
1819
1820 #[test]
1821 fn unicode_range_max_hex() {
1822 // §4.3.14: U+10FFFF → UnicodeRange(0x10FFFF, 0x10FFFF)
1823 let mut tz = CssTokenizer::new("U+10FFFF");
1824 tz.set_unicode_ranges_allowed(true);
1825 let t = tz.next_token().unwrap();
1826 assert_eq!(t, Token::UnicodeRange(Some(0x10FFFF), Some(0x10FFFF)));
1827 }
1828
1829 #[test]
1830 fn unicode_range_lowercase_u() {
1831 // §4.3.11: lowercase u also triggers; u+abc → UnicodeRange(0xABC, 0xABC)
1832 let mut tz = CssTokenizer::new("u+abc");
1833 tz.set_unicode_ranges_allowed(true);
1834 let t = tz.next_token().unwrap();
1835 assert_eq!(t, Token::UnicodeRange(Some(0xABC), Some(0xABC)));
1836 }
1837}