muskitty_css_tokenizer/impls.rs
1//! CSS Syntax tokenizer implementation.
2//!
3//! Implements the "Tokenizer Algorithms" of CSS Syntax Module Level 3 §4.3.
4//! The tokenizer is recursive-descent: [`CssTokenizer::next_token`] calls
5//! [`consume_a_token`](CssTokenizer::consume_a_token) (§4.3.1), which
6//! dispatches to sub-algorithms based on the current input code point.
7//!
8//! # Coverage
9//!
10//! All §4.3 sub-algorithms are implemented:
11//! - §4.3.1 Consume a token (full dispatch, incl. `unicode_ranges_allowed`)
12//! - §4.3.2 Consume comments
13//! - §4.3.3 Consume a numeric token
14//! - §4.3.4 Consume an ident-like token (incl. `url(` special case)
15//! - §4.3.5 Consume a string token
16//! - §4.3.6 Consume a url token
17//! - §4.3.7 Consume an escaped code point
18//! - §4.3.8 Check if two code points are a valid escape
19//! - §4.3.9 Check if three code points would start an ident sequence
20//! - §4.3.10 Check if three code points would start a number
21//! - §4.3.11 Check if three code points would start a unicode-range
22//! - §4.3.12 Consume an ident sequence
23//! - §4.3.13 Consume a number
24//! - §4.3.14 Consume a unicode-range token
25//! - §4.3.15 Consume the remnants of a bad url
26
27use crate::trait_def::Tokenizer;
28use crate::types::{HashType, Numeric, State, Token};
29
30/// The CSS tokenizer (§4.3).
31///
32/// Holds the preprocessed input stream (§5.3 normalization applied at
33/// construction), the current position, and EOF tracking. Sub-algorithms
34/// are implemented as methods on this struct so they share the input
35/// stream and position.
36pub struct CssTokenizer {
37 /// Input code points after §5.3 preprocessing (CR/LF/FF normalized to LF).
38 input: Vec<char>,
39 /// Current position in `input` (0-based index of the next code point
40 /// to consume). `pos == input.len()` means the stream is exhausted.
41 pos: usize,
42 /// Top-level state (§4.3.1 dispatch vs. EOF).
43 state: State,
44 /// Whether `<EOF-token>` has been emitted. After this is `true`,
45 /// [`next_token`](Tokenizer::next_token) returns `None`.
46 eof_emitted: bool,
47 /// §4.3.1 L782-783: `unicode_ranges_allowed` flag, default `false`.
48 /// When `true`, the `U+`/`u+` branch of §4.3.1 produces a
49 /// `<unicode-range-token>` (§4.3.14); otherwise `U`/`u` is tokenized
50 /// as an ident-like token.
51 unicode_ranges_allowed: bool,
52}
53
54impl CssTokenizer {
55 /// Construct a new tokenizer over `input`.
56 ///
57 /// Applies §5.3 input preprocessing: per the "input stream"
58 /// definition, U+000D CR followed by U+000A LF is collapsed to a
59 /// single LF, lone CR is replaced with LF, and U+000C FF is replaced
60 /// with LF. The tokenizer operates on this normalized stream.
61 pub fn new(input: &str) -> Self {
62 let input = preprocess_input(input);
63 Self {
64 input,
65 pos: 0,
66 state: State::Data,
67 eof_emitted: false,
68 unicode_ranges_allowed: false,
69 }
70 }
71
72 /// Peek at the code point at `offset` from the current position
73 /// without consuming it. Returns `None` if past end of input.
74 ///
75 /// Used by §4.3.8 / §4.3.9 / §4.3.10 / §4.3.11 "check if N code
76 /// points..." predicates, which must look ahead without advancing.
77 fn peek(&self, offset: usize) -> Option<char> {
78 self.input.get(self.pos + offset).copied()
79 }
80
81 /// Consume and return the next code point, advancing `pos`. Returns
82 /// `None` if at end of input.
83 ///
84 /// Implements the "consume the next input code point" primitive used
85 /// throughout §4.3.
86 fn consume(&mut self) -> Option<char> {
87 let c = self.input.get(self.pos).copied();
88 if c.is_some() {
89 self.pos += 1;
90 }
91 c
92 }
93
94 /// Re-consume the current code point: step `pos` back by one.
95 ///
96 /// Per §4.3.1 "reconsume the current input code point": the next
97 /// call to [`consume`](Self::consume) will return the same code point
98 /// that was just consumed. No-op if `pos == 0`.
99 fn reconsume(&mut self) {
100 if self.pos > 0 {
101 self.pos -= 1;
102 }
103 }
104
105 /// §4.3.1 Consume a token.
106 ///
107 /// This is the main entry point of the tokenizer. Dispatches based on
108 /// the current input code point to one of the sub-algorithms (§4.3.2
109 /// through §4.3.6) or emits a simple token directly. The optional
110 /// `unicode_ranges_allowed` flag (§4.3.1 L782-783, default `false`)
111 /// gates the `U+`/`u+` unicode-range branch.
112 fn consume_a_token(&mut self) -> Token {
113 // §4.3.2: If the next two input code points are U+002F SOLIDUS
114 // (`/`) and U+002A ASTERISK (`*`), consume comments and
115 // recursively consume the next token. We implement this as a
116 // loop: any leading run of `/* ... */` comments is skipped before
117 // producing the next real token.
118 loop {
119 let Some(c) = self.consume() else {
120 // §5.3: end of stream → <EOF-token>
121 self.eof_emitted = true;
122 self.state = State::Eof;
123 return Token::Eof;
124 };
125
126 // §4.3.2 comment dispatch: `/` followed by `*`.
127 if c == '/' && self.peek(0) == Some('*') {
128 self.consume(); // consume `*`
129 self.consume_comments_body();
130 // Loop back to consume the next token (which may itself
131 // be preceded by another comment).
132 continue;
133 }
134
135 return match c {
136 // §4.3.1: whitespace → <whitespace-token>
137 ' ' | '\t' | '\n' | '\r' | '\u{000C}' => {
138 // Consume all consecutive whitespace.
139 while let Some(next) = self.peek(0) {
140 if matches!(next, ' ' | '\t' | '\n' | '\r' | '\u{000C}') {
141 self.consume();
142 } else {
143 break;
144 }
145 }
146 Token::Whitespace
147 }
148 // §4.3.1: `"` or `'` → consume a string token (§4.3.5)
149 '"' | '\'' => self.consume_a_string_token(c),
150 // §4.3.1: `#` → consume an ident-like token (§4.3.4) for hash
151 '#' => self.consume_a_hash_token(),
152 // §4.3.1: `(` → <(-token>
153 '(' => Token::OpenParen,
154 // §4.3.1: `)` → <)-token>
155 ')' => Token::CloseParen,
156 // §4.3.1: `+` or `-` → if would-start-a-number, consume a
157 // numeric token (§4.3.3); else <delim-token>.
158 // §4.3.1: `-` additionally: if next two are `-` and `>`
159 // (i.e. `-->`), consume them and return <CDC-token>.
160 // §4.3.1: `.` → if would-start-a-number, consume a numeric
161 // token; else <delim-token>.
162 '-' => {
163 if self.starts_with_number(c) {
164 self.reconsume();
165 self.consume_a_numeric_token()
166 } else if self.peek(0) == Some('-') && self.peek(1) == Some('>') {
167 // §4.3.1: `-->` → <CDC-token>
168 self.consume(); // consume second `-`
169 self.consume(); // consume `>`
170 Token::Cdc
171 } else if self.would_start_ident_sequence_at(0) {
172 // §4.3.1: `-` starting an ident sequence → reconsume
173 // and consume an ident-like token.
174 self.reconsume();
175 self.consume_an_ident_like_token()
176 } else {
177 Token::Delim(c)
178 }
179 }
180 '+' | '.' => {
181 if self.starts_with_number(c) {
182 self.reconsume();
183 self.consume_a_numeric_token()
184 } else {
185 Token::Delim(c)
186 }
187 }
188 // §4.3.1: `<` → if next three are `!`, `-`, `-` (i.e.
189 // `<!--`), consume them and return <CDO-token>. Otherwise
190 // emit <delim-token>.
191 '<' => {
192 if self.peek(0) == Some('!')
193 && self.peek(1) == Some('-')
194 && self.peek(2) == Some('-')
195 {
196 // Consume `!`, `-`, `-`.
197 self.consume();
198 self.consume();
199 self.consume();
200 Token::Cdo
201 } else {
202 Token::Delim(c)
203 }
204 }
205 // §4.3.1: `@` → if next would-start-an-ident-sequence,
206 // consume an ident-like token (§4.3.4) for at-keyword.
207 '@' => self.consume_an_at_keyword_token(),
208 // §4.3.1: `[` → <[-token>
209 '[' => Token::OpenBracket,
210 // §4.3.1: `\` → if would-start-an-escape (§4.3.8), reconsume
211 // and consume an ident-like token (§4.3.4); else parse error,
212 // <delim-token>.
213 '\\' => self.consume_ident_or_delim(),
214 // §4.3.1: `]` → <]-token>
215 ']' => Token::CloseBracket,
216 // §4.3.1: `{` → <{-token>
217 '{' => Token::OpenBrace,
218 // §4.3.1: `}` → <}-token>
219 '}' => Token::CloseBrace,
220 // §4.3.1: digit → reconsume and consume a numeric token (§4.3.3)
221 '0'..='9' => {
222 self.reconsume();
223 self.consume_a_numeric_token()
224 }
225 // §4.3.1: `U+` or `u+` → if would-start-a-unicode-range (§4.3.11),
226 // consume a unicode-range token (§4.3.14).
227 'u' | 'U' => self.consume_u_or_unicode_range(),
228 // §4.3.1: ident-start code point → reconsume and consume an
229 // ident-like token (§4.3.4)
230 _ if is_ident_start_code_point(c) => {
231 self.reconsume();
232 self.consume_an_ident_like_token()
233 }
234 // §4.3.1: `:` → <colon-token>
235 ':' => Token::Colon,
236 // §4.3.1: `;` → <semicolon-token>
237 ';' => Token::Semicolon,
238 // §4.3.1: `,` → <comma-token>
239 ',' => Token::Comma,
240 // §4.3.1: anything else → <delim-token>
241 _ => Token::Delim(c),
242 };
243 }
244 }
245
246 /// §4.3.2 Consume comments — body only.
247 ///
248 /// Precondition: the opening `/*` has already been consumed. Consume
249 /// code points until the closing `*/` or EOF. Per §4.3.2, an
250 /// unterminated comment (EOF before `*/`) is a parse error; we
251 /// silently consume to EOF in that case (no error reporting yet).
252 fn consume_comments_body(&mut self) {
253 loop {
254 match self.consume() {
255 None => return, // EOF in comment: parse error (unreported)
256 Some('*') => {
257 if self.peek(0) == Some('/') {
258 self.consume(); // consume `/`
259 return;
260 }
261 }
262 Some(_) => {}
263 }
264 }
265 }
266
267 // ── Sub-algorithm stubs (implemented in subsequent commits) ───────
268
269 /// §4.3.5 (L1081-1128) Consume a string token.
270 ///
271 /// `quote` is the ending code point (the opening quote `"` or `'`).
272 /// Returns a `string-token` (normal or EOF-terminated) or a
273 /// `bad-string-token` (unescaped newline).
274 ///
275 /// Per §4.3.5:
276 /// - ending code point (L1097-1099) → return string-token.
277 /// - EOF (L1101-1104) → parse error, return string-token.
278 /// - newline (L1106-1110) → parse error, reconsume, return bad-string.
279 /// - `\` (L1112-1124):
280 /// - next EOF → do nothing (loop; next consume hits EOF → string).
281 /// - next newline → consume it (line continuation).
282 /// - otherwise (valid escape) → consume an escaped code point, append.
283 /// - anything else (L1126-1128) → append.
284 fn consume_a_string_token(&mut self, quote: char) -> Token {
285 let mut value = String::new();
286 loop {
287 match self.consume() {
288 None => {
289 // §4.3.5 L1101-1104: EOF → parse error, return string-token.
290 return Token::String(value);
291 }
292 Some(c) if c == quote => {
293 // §4.3.5 L1097-1099: ending code point → return string-token.
294 return Token::String(value);
295 }
296 Some('\n') => {
297 // §4.3.5 L1106-1110: newline → parse error, reconsume, return bad-string.
298 self.reconsume();
299 return Token::BadString;
300 }
301 Some('\\') => {
302 // §4.3.5 L1112-1124: escape handling.
303 match self.peek(0) {
304 None => {
305 // §4.3.5 L1114-1115: next is EOF, do nothing.
306 // Loop continues; next consume() returns None → String.
307 }
308 Some('\n') => {
309 // §4.3.5 L1117-1119: next is newline, consume it (line continuation).
310 self.consume();
311 }
312 Some(_) => {
313 // §4.3.5 L1121-1124: valid escape, consume escaped code point.
314 let escaped = self.consume_an_escaped_code_point();
315 value.push(escaped);
316 }
317 }
318 }
319 Some(c) => {
320 // §4.3.5 L1126-1128: anything else → append.
321 value.push(c);
322 }
323 }
324 }
325 }
326
327 /// §4.3.1 (L801-826) `#` branch: Consume a hash token.
328 ///
329 /// Precondition: `#` has been consumed. Per §4.3.1 L801-826:
330 /// - If the next code point is an ident code point OR the next two
331 /// are a valid escape, then create a `hash-token` (type "id" if
332 /// would-start-ident-sequence, else "unrestricted"), consume an
333 /// ident sequence for the value, and return it.
334 /// - Otherwise, return a `delim-token` with value `#`.
335 fn consume_a_hash_token(&mut self) -> Token {
336 // §4.3.1 L803-805: next is ident code point OR next two are valid escape.
337 let next_is_ident = self.peek(0).is_some_and(is_ident_code_point);
338 let next_is_valid_escape = self.is_valid_escape_at(0);
339 if !next_is_ident && !next_is_valid_escape {
340 // §4.3.1 L824-826: return delim-token with value `#`.
341 return Token::Delim('#');
342 }
343 let hash_type = if self.would_start_ident_sequence_at(0) {
344 HashType::Id
345 } else {
346 HashType::Unrestricted
347 };
348 let name = self.consume_an_ident_sequence();
349 Token::Hash(name, hash_type)
350 }
351
352 /// §4.3.3 (L1011-1042) Consume a numeric token.
353 ///
354 /// Returns a `number-token`, `percentage-token`, or `dimension-token`.
355 /// Per §4.3.3:
356 /// 1. Consume a number (§4.3.13).
357 /// 2. If next 3 would start an ident sequence → dimension-token (consume
358 /// an ident sequence for the unit).
359 /// 3. Else if next is `%` → percentage-token (consume `%`).
360 /// 4. Else → number-token.
361 fn consume_a_numeric_token(&mut self) -> Token {
362 let (value, is_integer) = self.consume_a_number();
363 let numeric = Numeric { value, is_integer };
364 // §4.3.3 L1019-1029: dimension
365 if self.would_start_ident_sequence_at(0) {
366 let unit = self.consume_an_ident_sequence();
367 return Token::Dimension(numeric, unit);
368 }
369 // §4.3.3 L1032-1036: percentage
370 if self.peek(0) == Some('%') {
371 self.consume();
372 return Token::Percentage(numeric);
373 }
374 // §4.3.3 L1040-1042: number
375 Token::Number(numeric)
376 }
377
378 /// §4.3.13 (L1415-1483) Consume a number.
379 ///
380 /// Returns `(value, is_integer)` where `is_integer` is true when the
381 /// source representation had no fractional part and no exponent (type
382 /// "integer"), false when it had either (type "number"). The sign is
383 /// included in `value`.
384 ///
385 /// Per §4.3.13:
386 /// 1. type = "integer"; number_part = ""; exponent_part = "".
387 /// 2. If next is `+`/`-`, consume, append to number_part.
388 /// 3. While next is digit, consume, append to number_part.
389 /// 4. If next 2 are `.` + digit: consume `.`, consume digits; type = "number".
390 /// 5. If next 2/3 are `e`/`E` + optional `+`/`-` + digit: consume `e`/`E`,
391 /// optional sign, digits into exponent_part; type = "number".
392 /// 6. value = parse number_part; if exponent_part non-empty, value *= 10^exp.
393 /// 7. Return value, type, sign.
394 fn consume_a_number(&mut self) -> (f64, bool) {
395 let mut type_is_number = false;
396 let mut number_part = String::new();
397 let mut exponent_part = String::new();
398
399 // §4.3.13 L1436-1440: sign
400 if self.peek(0) == Some('+') || self.peek(0) == Some('-') {
401 number_part.push(self.consume().unwrap());
402 }
403
404 // §4.3.13 L1442-1444: integer digits
405 while let Some(d) = self.peek(0) {
406 if is_digit(d) {
407 number_part.push(d);
408 self.consume();
409 } else {
410 break;
411 }
412 }
413
414 // §4.3.13 L1446-1454: fraction part
415 if self.peek(0) == Some('.') && matches!(self.peek(1), Some(d) if is_digit(d)) {
416 number_part.push(self.consume().unwrap()); // consume `.`
417 type_is_number = true;
418 while let Some(d) = self.peek(0) {
419 if is_digit(d) {
420 number_part.push(d);
421 self.consume();
422 } else {
423 break;
424 }
425 }
426 }
427
428 // §4.3.13 L1457-1469: exponent part
429 if matches!(self.peek(0), Some('e') | Some('E')) {
430 let p1 = self.peek(1);
431 let p2 = self.peek(2);
432 let valid_exp = match p1 {
433 Some('+') | Some('-') => matches!(p2, Some(d) if is_digit(d)),
434 Some(d) if is_digit(d) => true,
435 _ => false,
436 };
437 if valid_exp {
438 self.consume(); // consume `e`/`E`
439 if self.peek(0) == Some('+') || self.peek(0) == Some('-') {
440 exponent_part.push(self.consume().unwrap());
441 }
442 while let Some(d) = self.peek(0) {
443 if is_digit(d) {
444 exponent_part.push(d);
445 self.consume();
446 } else {
447 break;
448 }
449 }
450 type_is_number = true;
451 }
452 }
453
454 // §4.3.13 L1472-1480: compute value
455 let mut value: f64 = number_part.parse().unwrap_or(0.0);
456 if !exponent_part.is_empty() {
457 let exp: i32 = exponent_part.parse().unwrap_or(0);
458 value *= 10.0f64.powi(exp);
459 }
460
461 (value, !type_is_number)
462 }
463
464 /// §4.3.4 (L1045-1078) Consume an ident-like token.
465 ///
466 /// Precondition: the current position is at the start of an ident
467 /// sequence (the caller has not consumed anything; or has reconsumed
468 /// the first code point). Returns an `ident-token`, `function-token`,
469 /// `url-token`, or `bad-url-token`.
470 ///
471 /// Per §4.3.4:
472 /// 1. Consume an ident sequence (§4.3.12), yielding `name`.
473 /// 2. If `name` is ASCII-case-insensitive "url" and next is `(`:
474 /// consume `(`, collapse leading whitespace pairs, then if the
475 /// next 1-2 code points are a quote (or whitespace+quote) return
476 /// a `function-token`; otherwise consume a url token (§4.3.6).
477 /// 3. Else if next is `(`: consume it, return `function-token`.
478 /// 4. Else return `ident-token`.
479 fn consume_an_ident_like_token(&mut self) -> Token {
480 let name = self.consume_an_ident_sequence();
481 // §4.3.4 L1053-1066: url( special case
482 if name.eq_ignore_ascii_case("url") && self.peek(0) == Some('(') {
483 self.consume(); // consume `(`
484 // §4.3.4 L1056-1057: while next two are whitespace, consume one
485 while self.peek(0).is_some_and(is_whitespace) && self.peek(1).is_some_and(is_whitespace)
486 {
487 self.consume();
488 }
489 // §4.3.4 L1058-1063: if next 1-2 are " / ' / ws+" / ws+' → Function
490 let p0 = self.peek(0);
491 let p1 = self.peek(1);
492 let is_quote_case = matches!(p0, Some('"') | Some('\''))
493 || (p0.is_some_and(is_whitespace) && matches!(p1, Some('"') | Some('\'')));
494 if is_quote_case {
495 return Token::Function(name);
496 }
497 // §4.3.4 L1064-1066: otherwise consume a url token
498 return self.consume_a_url_token();
499 }
500 // §4.3.4 L1068-1073: ordinary function
501 if self.peek(0) == Some('(') {
502 self.consume();
503 return Token::Function(name);
504 }
505 // §4.3.4 L1075-1078: ident
506 Token::Ident(name)
507 }
508
509 /// §4.3.4 (at-keyword branch) Consume an at-keyword token.
510 ///
511 /// Precondition: `@` has been consumed. Per §4.3.1, if the next code
512 /// points would start an ident sequence, consume it and return an
513 /// `<at-keyword-token>`; otherwise return a `<delim-token>` with `@`.
514 fn consume_an_at_keyword_token(&mut self) -> Token {
515 if self.would_start_ident_sequence_at(0) {
516 let name = self.consume_an_ident_sequence();
517 Token::AtKeyword(name)
518 } else {
519 Token::Delim('@')
520 }
521 }
522
523 /// §4.3.1 (backslash branch): if would-start-an-escape, consume an
524 /// ident-like token; else emit `<delim-token>`.
525 fn consume_ident_or_delim(&mut self) -> Token {
526 if self.is_valid_escape_next() {
527 self.reconsume();
528 self.consume_an_ident_like_token()
529 } else {
530 Token::Delim('\\')
531 }
532 }
533
534 /// §4.3.6 (L1132-1203) Consume a url token.
535 ///
536 /// Precondition: `url(` has been consumed by the caller (§4.3.4
537 /// L1053-1066). This algorithm consumes the "unquoted" url body and
538 /// returns a `url-token` or `bad-url-token`. A quoted value like
539 /// `url("foo")` is handled by §4.3.4 as a `function-token` and never
540 /// reaches here.
541 ///
542 /// Per §4.3.6:
543 /// - `)` → return url-token.
544 /// - EOF → parse error, return url-token.
545 /// - whitespace → consume trailing whitespace; if next is `)` or EOF
546 /// return url-token, else bad-url.
547 /// - `"` / `'` / `(` / non-printable → parse error, bad-url.
548 /// - `\` → if valid escape, consume escaped and append; else bad-url.
549 /// - anything else → append.
550 fn consume_a_url_token(&mut self) -> Token {
551 let mut value = String::new();
552 // §4.3.6 L1151: consume as much whitespace as possible
553 while self.peek(0).is_some_and(is_whitespace) {
554 self.consume();
555 }
556 loop {
557 match self.consume() {
558 // §4.3.6 L1157-1159: `)` → return url-token
559 Some(')') => return Token::Url(value),
560 // §4.3.6 L1161-1164: EOF → parse error, return url-token
561 None => return Token::Url(value),
562 // §4.3.6 L1166-1175: whitespace → consume trailing ws, then check
563 Some(c) if is_whitespace(c) => {
564 while self.peek(0).is_some_and(is_whitespace) {
565 self.consume();
566 }
567 match self.peek(0) {
568 Some(')') => {
569 self.consume();
570 return Token::Url(value);
571 }
572 None => return Token::Url(value), // parse error
573 _ => {
574 self.consume_the_remnants_of_a_bad_url();
575 return Token::BadUrl;
576 }
577 }
578 }
579 // §4.3.6 L1177-1185: " / ' / ( / non-printable → bad url
580 Some(c) if c == '"' || c == '\'' || c == '(' || is_non_printable(c) => {
581 self.consume_the_remnants_of_a_bad_url();
582 return Token::BadUrl;
583 }
584 // §4.3.6 L1187-1197: `\`
585 Some('\\') => {
586 if self.is_valid_escape_next() {
587 let escaped = self.consume_an_escaped_code_point();
588 value.push(escaped);
589 } else {
590 self.consume_the_remnants_of_a_bad_url();
591 return Token::BadUrl;
592 }
593 }
594 // §4.3.6 L1199-1202: anything else → append
595 Some(c) => value.push(c),
596 }
597 }
598 }
599
600 /// §4.3.15 (L1551-1577) Consume the remnants of a bad url.
601 ///
602 /// Consumes code points until `)` or EOF, allowing an escaped `)`
603 /// (`\)`) to be consumed without ending the bad-url. Per §4.3.15:
604 /// - `)` or EOF → return.
605 /// - valid escape (stream starts with `\` + non-newline non-EOF) →
606 /// consume an escaped code point.
607 /// - anything else → do nothing.
608 fn consume_the_remnants_of_a_bad_url(&mut self) {
609 loop {
610 match self.consume() {
611 // §4.3.15 L1563-1566: `)` or EOF → return
612 Some(')') | None => return,
613 // §4.3.15 L1568-1572: valid escape → consume escaped code point
614 Some('\\') if self.is_valid_escape_next() => {
615 let _ = self.consume_an_escaped_code_point();
616 }
617 // §4.3.15 L1574-1576: anything else → do nothing
618 Some(_) => {}
619 }
620 }
621 }
622
623 /// §4.3.1 (L960-972) U/u branch: if `unicode_ranges_allowed` is true
624 /// and the stream would start a unicode-range (§4.3.11), consume a
625 /// unicode-range token (§4.3.14); otherwise consume an ident-like token.
626 ///
627 /// Precondition: `u`/`U` has been consumed, `pos` points just past it.
628 /// This method reconsumes back to `u` before dispatching.
629 fn consume_u_or_unicode_range(&mut self) -> Token {
630 self.reconsume(); // pos back to `u`/`U`
631 // §4.3.1 L963-967: unicode_ranges_allowed && would-start-unicode-range
632 if self.unicode_ranges_allowed && self.would_start_unicode_range_at(0) {
633 return self.consume_a_unicode_range_token();
634 }
635 // §4.3.1 L969-972: otherwise ident-like
636 self.consume_an_ident_like_token()
637 }
638
639 /// §4.3.11 (L1356-1378) Check if three code points would start a
640 /// unicode-range, examining code points starting at `offset` from the
641 /// current position.
642 ///
643 /// Per §4.3.11: true iff first is `U`/`u`, second is `+`, third is `?`
644 /// or a hex digit.
645 fn would_start_unicode_range_at(&self, offset: usize) -> bool {
646 let first = match self.peek(offset) {
647 Some(c) => c,
648 None => return false,
649 };
650 if !matches!(first, 'U' | 'u') {
651 return false;
652 }
653 if self.peek(offset + 1) != Some('+') {
654 return false;
655 }
656 match self.peek(offset + 2) {
657 Some('?') => true,
658 Some(c) if is_hex_digit(c) => true,
659 _ => false,
660 }
661 }
662
663 /// §4.3.14 (L1487-1548) Consume a unicode-range token.
664 ///
665 /// Precondition: the stream starts with `u`/`U` + `+` + (`?` or hex
666 /// digit) (i.e. [`would_start_unicode_range_at`](Self::would_start_unicode_range_at)
667 /// returns true at the current position).
668 ///
669 /// Returns `UnicodeRange(Some(start), Some(end))`.
670 ///
671 /// Per §4.3.14:
672 /// 1. Consume and discard `u`/`U` and `+`.
673 /// 2. Consume up to 6 hex digits; if fewer than 6, consume `?` up to a
674 /// total of 6 → `first_segment`.
675 /// 3. If `first_segment` contains `?`: start = `?`→`0`, end = `?`→`F`.
676 /// 4. Else start = hex value of `first_segment`.
677 /// 5. If next two are `-` + hex digit: consume `-`, consume up to 6 hex
678 /// → end.
679 /// 6. Else end = start.
680 fn consume_a_unicode_range_token(&mut self) -> Token {
681 // §4.3.14 step 1: consume and discard `u`/`U` + `+`
682 self.consume(); // `u`/`U`
683 self.consume(); // `+`
684
685 // §4.3.14 step 2: consume up to 6 hex digits
686 let mut first_segment = String::new();
687 while first_segment.len() < 6 {
688 match self.peek(0) {
689 Some(c) if is_hex_digit(c) => {
690 first_segment.push(c);
691 self.consume();
692 }
693 _ => break,
694 }
695 }
696 // then consume `?` up to a total of 6
697 while first_segment.len() < 6 {
698 match self.peek(0) {
699 Some('?') => {
700 first_segment.push('?');
701 self.consume();
702 }
703 _ => break,
704 }
705 }
706
707 // §4.3.14 step 3: if first_segment contains `?`
708 if first_segment.contains('?') {
709 let start_str: String = first_segment
710 .chars()
711 .map(|c| if c == '?' { '0' } else { c })
712 .collect();
713 let start = u32::from_str_radix(&start_str, 16).unwrap_or(0);
714 let end_str: String = first_segment
715 .chars()
716 .map(|c| if c == '?' { 'F' } else { c })
717 .collect();
718 let end = u32::from_str_radix(&end_str, 16).unwrap_or(0);
719 return Token::UnicodeRange(Some(start), Some(end));
720 }
721
722 // §4.3.14 step 4: first_segment as hex → start
723 let start = u32::from_str_radix(&first_segment, 16).unwrap_or(0);
724
725 // §4.3.14 step 5: if next two are `-` + hex digit
726 if self.peek(0) == Some('-') && self.peek(1).is_some_and(is_hex_digit) {
727 self.consume(); // consume `-`
728 let mut end_segment = String::new();
729 while end_segment.len() < 6 {
730 match self.peek(0) {
731 Some(c) if is_hex_digit(c) => {
732 end_segment.push(c);
733 self.consume();
734 }
735 _ => break,
736 }
737 }
738 let end = u32::from_str_radix(&end_segment, 16).unwrap_or(0);
739 return Token::UnicodeRange(Some(start), Some(end));
740 }
741
742 // §4.3.14 step 6: otherwise start == end
743 Token::UnicodeRange(Some(start), Some(start))
744 }
745
746 // ── §4.3.12 Consume an ident sequence ────────────────────────────
747
748 /// §4.3.12 Consume an ident sequence.
749 ///
750 /// Repeatedly consume code points as long as they form an ident
751 /// sequence: ident code points, valid escapes (consumed via
752 /// §4.3.7), or `\`-`-` (for `--`-prefixed names). Returns the
753 /// decoded name with escapes resolved.
754 ///
755 /// Per §4.3.12, the loop terminates when:
756 /// - EOF is reached, or
757 /// - the next code point is not an ident code point, not a valid
758 /// escape, and not part of a `\`-`-` continuation.
759 fn consume_an_ident_sequence(&mut self) -> String {
760 let mut result = String::new();
761 loop {
762 let Some(c) = self.consume() else {
763 return result;
764 };
765 if is_ident_code_point(c) {
766 result.push(c);
767 } else if c == '\\' && self.is_valid_escape_next() {
768 // §4.3.12: valid escape → consume the escaped code point
769 // and append it.
770 let escaped = self.consume_an_escaped_code_point();
771 result.push(escaped);
772 } else {
773 // Not part of the ident sequence: reconsume and stop.
774 self.reconsume();
775 return result;
776 }
777 }
778 }
779
780 // ── §4.3.7 Consume an escaped code point ────────────────────────
781
782 /// §4.3.7 Consume an escaped code point.
783 ///
784 /// Precondition: the `\` has been consumed. Returns the decoded code
785 /// point.
786 ///
787 /// Per §4.3.7 (L1207-1240):
788 /// - EOF (L1233-1236) → parse error, return U+FFFD REPLACEMENT CHARACTER.
789 /// - newline → parse error, reconsume (invalid escape; the caller
790 /// should have checked §4.3.8 first, but we handle defensively).
791 /// - hex digit(s) (L1219-1231) → consume 1-6 hex digits, optionally
792 /// followed by whitespace, and return the code point with that
793 /// value. U+0000 NULL or out-of-range → replacement character U+FFFD.
794 /// - anything else (L1238-1240) → return the consumed code point.
795 fn consume_an_escaped_code_point(&mut self) -> char {
796 let Some(c) = self.consume() else {
797 // §4.3.7 L1233-1236: EOF → parse error, return U+FFFD.
798 return '\u{FFFD}';
799 };
800 if is_hex_digit(c) {
801 // §4.3.7 step 4: consume up to 5 more hex digits (total ≤ 6).
802 let mut hex = String::new();
803 hex.push(c);
804 while hex.len() < 6 {
805 match self.peek(0) {
806 Some(next) if is_hex_digit(next) => {
807 hex.push(next);
808 self.consume();
809 }
810 _ => break,
811 }
812 }
813 // §4.3.7 step 4: consume one trailing whitespace.
814 if self.peek(0) == Some(' ') || self.peek(0) == Some('\t') || self.peek(0) == Some('\n')
815 {
816 self.consume();
817 }
818 // Parse the hex value.
819 let value = u32::from_str_radix(&hex, 16).unwrap_or(0);
820 // §4.3.7 step 4: 0 or > 0x10FFFF → U+FFFD. Also surrogates
821 // (0xD800–0xDFFF) → U+FFFD.
822 if value == 0 || value > 0x10FFFF || (0xD800..=0xDFFF).contains(&value) {
823 '\u{FFFD}'
824 } else {
825 char::from_u32(value).unwrap_or('\u{FFFD}')
826 }
827 } else if c == '\n' {
828 // §4.3.7 step 3: newline after `\` is a parse error. This
829 // should not happen if the caller checked §4.3.8, but handle
830 // defensively by returning `\` and reconsuming the newline.
831 self.reconsume();
832 '\\'
833 } else {
834 // §4.3.7 step 5: any other code point → return it.
835 c
836 }
837 }
838
839 // ── §4.3.8 / §4.3.9 Predicates ──────────────────────────────────
840
841 /// §4.3.8 Check if two code points are a valid escape.
842 ///
843 /// Precondition: the `\` has been consumed. Returns true if the next
844 /// code point forms a valid escape with the consumed `\` (i.e. the
845 /// next code point is not a newline and not EOF).
846 fn is_valid_escape_next(&self) -> bool {
847 match self.peek(0) {
848 None => false,
849 Some('\n') => false,
850 Some(_) => true,
851 }
852 }
853
854 /// §4.3.9 Check if three code points would start an ident sequence,
855 /// examining code points starting at `offset` from the current
856 /// position.
857 ///
858 /// Per §4.3.9:
859 /// - U+002D HYPHEN-MINUS (`-`):
860 /// - If the next is also `-`, true (start of `--...`).
861 /// - If the next is an ident-start code point (or a valid escape),
862 /// true.
863 /// - Else false.
864 /// - U+005C REVERSE SOLIDUS (`\`): true if valid escape.
865 /// - ident-start code point: true.
866 /// - Else false.
867 fn would_start_ident_sequence_at(&self, offset: usize) -> bool {
868 let first = match self.peek(offset) {
869 Some(c) => c,
870 None => return false,
871 };
872 match first {
873 '-' => {
874 // §4.3.9: if next is `-`, true.
875 match self.peek(offset + 1) {
876 Some('-') => true,
877 Some(next) if is_ident_start_code_point(next) => true,
878 Some('\\') => {
879 // §4.3.9: if `\` and the following forms a valid escape.
880 self.is_valid_escape_at(offset + 1)
881 }
882 _ => false,
883 }
884 }
885 '\\' => self.is_valid_escape_at(offset),
886 _ if is_ident_start_code_point(first) => true,
887 _ => false,
888 }
889 }
890
891 /// §4.3.8 Check if the code point at `offset` starts a valid escape
892 /// (i.e. `offset` points at `\` and `offset+1` is not newline/EOF).
893 fn is_valid_escape_at(&self, offset: usize) -> bool {
894 if self.peek(offset) != Some('\\') {
895 return false;
896 }
897 match self.peek(offset + 1) {
898 None => false,
899 Some('\n') => false,
900 Some(_) => true,
901 }
902 }
903
904 // ── Predicates (stubs returning false; implemented in C-4/C-6) ──
905
906 /// §4.3.10 (L1307-1352) Check if three code points would start a number.
907 ///
908 /// `first` is the code point already consumed (the caller passes it
909 /// so the predicate can examine the next two without reconsume). The
910 /// three code points examined are `first`, `peek(0)`, `peek(1)`.
911 ///
912 /// Per §4.3.10:
913 /// - `+`/`-`: second is digit → true; second is `.` and third is digit
914 /// → true; else false.
915 /// - `.`: second is digit → true; else false.
916 /// - digit: true.
917 /// - anything else: false.
918 fn starts_with_number(&self, first: char) -> bool {
919 match first {
920 '+' | '-' => match self.peek(0) {
921 Some(d) if is_digit(d) => true,
922 Some('.') => matches!(self.peek(1), Some(d) if is_digit(d)),
923 _ => false,
924 },
925 '.' => matches!(self.peek(0), Some(d) if is_digit(d)),
926 d if is_digit(d) => true,
927 _ => false,
928 }
929 }
930}
931
932// ── Free functions (§4.2 code point classes) ──────────────────────────
933
934/// §4.2 Whether `c` is an ident-start code point.
935///
936/// An ident-start code point is an ASCII letter, a non-ASCII code point
937/// (U+0080 or higher), or U+005F LOW LINE (`_`).
938fn is_ident_start_code_point(c: char) -> bool {
939 matches!(c, 'A'..='Z' | 'a'..='z' | '_' | '\u{0080}'..=char::MAX)
940}
941
942/// §4.2 Whether `c` is an ident code point (allowed after the first).
943///
944/// An ident code point is an ident-start code point, or an ASCII digit,
945/// or U+002D HYPHEN-MINUS (`-`).
946fn is_ident_code_point(c: char) -> bool {
947 matches!(c, 'A'..='Z' | 'a'..='z' | '0'..='9' | '_' | '-' | '\u{0080}'..=char::MAX)
948}
949
950/// §4.2 Whether `c` is a digit (ASCII 0-9).
951fn is_digit(c: char) -> bool {
952 c.is_ascii_digit()
953}
954
955/// §4.2 Whether `c` is a hex digit (ASCII 0-9, A-F, a-f).
956fn is_hex_digit(c: char) -> bool {
957 c.is_ascii_hexdigit()
958}
959
960/// §4.2 Whether `c` is a whitespace code point.
961///
962/// Per §4.2: U+0009 TAB, U+000A LF, U+000C FF, U+000D CR, U+0020 SPACE.
963/// (After §5.3 preprocessing, U+000C FF and lone U+000D CR have been
964/// normalized to U+000A LF, but the helper follows the full §4.2
965/// definition for correctness when called on arbitrary input.)
966fn is_whitespace(c: char) -> bool {
967 matches!(c, '\t' | '\n' | '\u{000C}' | '\r' | ' ')
968}
969
970/// §4.2 Whether `c` is a non-printable code point.
971///
972/// Per §4.2: U+0000-U+0008, U+000B, U+000E-U+001F, U+007F-U+009F.
973fn is_non_printable(c: char) -> bool {
974 matches!(
975 c,
976 '\u{0000}'..='\u{0008}' | '\u{000B}' | '\u{000E}'..='\u{001F}' | '\u{007F}'..='\u{009F}'
977 )
978}
979
980/// §5.3 input preprocessing.
981///
982/// Per the "input stream" definition in §5.3:
983/// - U+000D CR followed by U+000A LF → single U+000A LF
984/// - lone U+000D CR → U+000A LF
985/// - U+000C FF → U+000A LF
986///
987/// The tokenizer operates on this normalized stream.
988fn preprocess_input(s: &str) -> Vec<char> {
989 let mut out = Vec::with_capacity(s.len());
990 let mut chars = s.chars().peekable();
991 while let Some(c) = chars.next() {
992 match c {
993 '\r' => {
994 if chars.peek() == Some(&'\n') {
995 chars.next();
996 }
997 out.push('\n');
998 }
999 '\u{000C}' => out.push('\n'),
1000 _ => out.push(c),
1001 }
1002 }
1003 out
1004}
1005
1006impl Tokenizer for CssTokenizer {
1007 fn next_token(&mut self) -> Option<Token> {
1008 if self.eof_emitted {
1009 return None;
1010 }
1011 Some(self.consume_a_token())
1012 }
1013
1014 fn state(&self) -> State {
1015 self.state
1016 }
1017
1018 fn set_state(&mut self, state: State) {
1019 self.state = state;
1020 if state == State::Data {
1021 // Resetting to Data after EOF re-enables the iterator; useful
1022 // for `reset()` callers that want to re-tokenize.
1023 self.eof_emitted = false;
1024 }
1025 }
1026
1027 fn reset(&mut self) {
1028 self.pos = 0;
1029 self.state = State::Data;
1030 self.eof_emitted = false;
1031 }
1032
1033 fn set_unicode_ranges_allowed(&mut self, allowed: bool) {
1034 self.unicode_ranges_allowed = allowed;
1035 }
1036
1037 fn position(&self) -> usize {
1038 self.pos
1039 }
1040}
1041
1042// ── Test helpers exposed for unit tests in this file ──────────────────
1043
1044#[cfg(test)]
1045impl CssTokenizer {
1046 /// Tokenize the entire input and return all tokens except the
1047 /// trailing `<EOF-token>`. Test-only convenience.
1048 fn collect(input: &str) -> Vec<Token> {
1049 let mut tz = Self::new(input);
1050 let mut out = Vec::new();
1051 while let Some(t) = tz.next_token() {
1052 if matches!(t, Token::Eof) {
1053 break;
1054 }
1055 out.push(t);
1056 }
1057 out
1058 }
1059}
1060
1061#[cfg(test)]
1062mod tests {
1063 use super::*;
1064
1065 #[test]
1066 fn empty_input_emits_eof_only() {
1067 let mut tz = CssTokenizer::new("");
1068 assert!(matches!(tz.next_token(), Some(Token::Eof)));
1069 assert!(tz.next_token().is_none());
1070 }
1071
1072 #[test]
1073 fn whitespace_collapses_runs() {
1074 let tokens = CssTokenizer::collect(" \t\n ");
1075 assert_eq!(tokens.len(), 1, "whitespace run → single token");
1076 assert!(matches!(tokens[0], Token::Whitespace));
1077 }
1078
1079 #[test]
1080 fn simple_punctuation_tokens() {
1081 let tokens = CssTokenizer::collect(":;,.{}[]()");
1082 assert_eq!(tokens.len(), 10);
1083 assert!(matches!(tokens[0], Token::Colon));
1084 assert!(matches!(tokens[1], Token::Semicolon));
1085 assert!(matches!(tokens[2], Token::Comma));
1086 assert!(matches!(tokens[3], Token::Delim('.')));
1087 assert!(matches!(tokens[4], Token::OpenBrace));
1088 assert!(matches!(tokens[5], Token::CloseBrace));
1089 assert!(matches!(tokens[6], Token::OpenBracket));
1090 assert!(matches!(tokens[7], Token::CloseBracket));
1091 assert!(matches!(tokens[8], Token::OpenParen));
1092 assert!(matches!(tokens[9], Token::CloseParen));
1093 }
1094
1095 #[test]
1096 fn close_paren_is_emitted() {
1097 let tokens = CssTokenizer::collect(")");
1098 assert_eq!(tokens.len(), 1);
1099 assert!(matches!(tokens[0], Token::CloseParen));
1100 }
1101
1102 #[test]
1103 fn unknown_delim_chars() {
1104 let tokens = CssTokenizer::collect("^~`>");
1105 assert_eq!(tokens.len(), 4);
1106 assert!(matches!(tokens[0], Token::Delim('^')));
1107 assert!(matches!(tokens[1], Token::Delim('~')));
1108 assert!(matches!(tokens[2], Token::Delim('`')));
1109 assert!(matches!(tokens[3], Token::Delim('>')));
1110 }
1111
1112 #[test]
1113 fn preprocessing_normalizes_cr_lf_ff() {
1114 // CR LF → single LF
1115 let tz = CssTokenizer::new("a\r\nb");
1116 assert_eq!(tz.input, vec!['a', '\n', 'b']);
1117 // Lone CR → LF
1118 let tz = CssTokenizer::new("a\rb");
1119 assert_eq!(tz.input, vec!['a', '\n', 'b']);
1120 // FF → LF
1121 let tz = CssTokenizer::new("a\u{000C}b");
1122 assert_eq!(tz.input, vec!['a', '\n', 'b']);
1123 }
1124
1125 #[test]
1126 fn eof_emitted_once() {
1127 // C-1: use a non-ident-start, non-digit char to avoid the
1128 // ident-like / numeric branches (which are stubbed with todo!()
1129 // until C-2 / C-4).
1130 let mut tz = CssTokenizer::new(",");
1131 let t1 = tz.next_token();
1132 let t2 = tz.next_token();
1133 let t3 = tz.next_token();
1134 assert!(matches!(t1, Some(Token::Comma)));
1135 assert!(matches!(t2, Some(Token::Eof)));
1136 assert!(t3.is_none());
1137 }
1138
1139 #[test]
1140 fn block_comment_is_skipped() {
1141 // §4.3.2: `/* ... */` is consumed and no token is emitted.
1142 let tokens = CssTokenizer::collect("/* hello */:/*x*/;");
1143 assert_eq!(tokens.len(), 2);
1144 assert!(matches!(tokens[0], Token::Colon));
1145 assert!(matches!(tokens[1], Token::Semicolon));
1146 }
1147
1148 #[test]
1149 fn unterminated_comment_consumes_to_eof() {
1150 // §4.3.2: EOF before `*/` is a parse error; we consume to EOF.
1151 let tokens = CssTokenizer::collect("/* unfinished");
1152 assert_eq!(tokens.len(), 0, "unterminated comment → no tokens");
1153 }
1154
1155 #[test]
1156 fn comment_between_tokens() {
1157 // C-2 not yet: `a`/`b` would be ident tokens (todo!()). Verify
1158 // comment is consumed without affecting surrounding simple tokens
1159 // by using only punctuation.
1160 let tokens = CssTokenizer::collect(":/*c*/;");
1161 assert_eq!(tokens.len(), 2);
1162 assert!(matches!(tokens[0], Token::Colon));
1163 assert!(matches!(tokens[1], Token::Semicolon));
1164 }
1165
1166 #[test]
1167 fn cdo_token_emitted() {
1168 // §4.3.1: `<!--` → <CDO-token>
1169 let tokens = CssTokenizer::collect("<!--");
1170 assert_eq!(tokens.len(), 1);
1171 assert!(matches!(tokens[0], Token::Cdo));
1172 }
1173
1174 #[test]
1175 fn cdc_token_emitted() {
1176 // §4.3.1: `-->` → <CDC-token>
1177 let tokens = CssTokenizer::collect("-->");
1178 assert_eq!(tokens.len(), 1);
1179 assert!(matches!(tokens[0], Token::Cdc));
1180 }
1181
1182 #[test]
1183 fn cdo_cdc_wrapping_stylesheet() {
1184 // Legacy CSS 1/2.1 stylesheet wrapping.
1185 // `<!-- : ; -->`
1186 // tokens: Cdo, WS, Colon, WS, Semicolon, WS, Cdc = 7
1187 let tokens = CssTokenizer::collect("<!-- : ; -->");
1188 assert_eq!(tokens.len(), 7);
1189 assert!(matches!(tokens[0], Token::Cdo));
1190 assert!(matches!(tokens[1], Token::Whitespace));
1191 assert!(matches!(tokens[2], Token::Colon));
1192 assert!(matches!(tokens[3], Token::Whitespace));
1193 assert!(matches!(tokens[4], Token::Semicolon));
1194 assert!(matches!(tokens[5], Token::Whitespace));
1195 assert!(matches!(tokens[6], Token::Cdc));
1196 }
1197
1198 #[test]
1199 fn less_than_alone_is_delim() {
1200 // §4.3.1: `<` not followed by `!-` → <delim-token>
1201 let tokens = CssTokenizer::collect("< ");
1202 assert_eq!(tokens.len(), 2);
1203 assert!(matches!(tokens[0], Token::Delim('<')));
1204 assert!(matches!(tokens[1], Token::Whitespace));
1205 }
1206
1207 #[test]
1208 fn hyphen_alone_is_delim() {
1209 // §4.3.1: `-` not followed by number/ident/`->` → <delim-token>
1210 let tokens = CssTokenizer::collect("- ");
1211 assert_eq!(tokens.len(), 2);
1212 assert!(matches!(tokens[0], Token::Delim('-')));
1213 assert!(matches!(tokens[1], Token::Whitespace));
1214 }
1215
1216 // ── C-2 tests: ident / function / at-keyword / hash ──────────────
1217
1218 #[test]
1219 fn simple_ident() {
1220 let tokens = CssTokenizer::collect("color");
1221 assert_eq!(tokens.len(), 1);
1222 assert!(matches!(&tokens[0], Token::Ident(s) if s == "color"));
1223 }
1224
1225 #[test]
1226 fn ident_with_hyphen_prefix() {
1227 let tokens = CssTokenizer::collect("-webkit-flex");
1228 assert_eq!(tokens.len(), 1);
1229 assert!(matches!(&tokens[0], Token::Ident(s) if s == "-webkit-flex"));
1230 }
1231
1232 #[test]
1233 fn ident_with_double_hyphen() {
1234 // §4.3.9: `--` starts an ident sequence (custom properties).
1235 let tokens = CssTokenizer::collect("--my-var");
1236 assert_eq!(tokens.len(), 1);
1237 assert!(matches!(&tokens[0], Token::Ident(s) if s == "--my-var"));
1238 }
1239
1240 #[test]
1241 fn ident_with_underscore() {
1242 let tokens = CssTokenizer::collect("_private");
1243 assert_eq!(tokens.len(), 1);
1244 assert!(matches!(&tokens[0], Token::Ident(s) if s == "_private"));
1245 }
1246
1247 #[test]
1248 fn ident_with_escape() {
1249 // §4.3.7: `\26` → '&' (U+0026)
1250 let tokens = CssTokenizer::collect("color\\26 B");
1251 assert_eq!(tokens.len(), 1);
1252 assert!(matches!(&tokens[0], Token::Ident(s) if s == "color&B"));
1253 }
1254
1255 #[test]
1256 fn function_token() {
1257 let tokens = CssTokenizer::collect("translate(");
1258 assert_eq!(tokens.len(), 1);
1259 assert!(matches!(&tokens[0], Token::Function(s) if s == "translate"));
1260 }
1261
1262 #[test]
1263 fn at_keyword_token() {
1264 let tokens = CssTokenizer::collect("@media");
1265 assert_eq!(tokens.len(), 1);
1266 assert!(matches!(&tokens[0], Token::AtKeyword(s) if s == "media"));
1267 }
1268
1269 #[test]
1270 fn at_keyword_with_hyphen() {
1271 let tokens = CssTokenizer::collect("@-webkit-keyframes");
1272 assert_eq!(tokens.len(), 1);
1273 assert!(matches!(&tokens[0], Token::AtKeyword(s) if s == "-webkit-keyframes"));
1274 }
1275
1276 #[test]
1277 fn at_sign_alone_is_delim() {
1278 // §4.3.1: `@` not followed by ident-start → <delim-token>
1279 let tokens = CssTokenizer::collect("@ ");
1280 assert_eq!(tokens.len(), 2);
1281 assert!(matches!(tokens[0], Token::Delim('@')));
1282 assert!(matches!(tokens[1], Token::Whitespace));
1283 }
1284
1285 #[test]
1286 fn hash_id_type() {
1287 // §4.3.1 / §4.3.4: `#main` → <hash-token> with type "id"
1288 let tokens = CssTokenizer::collect("#main");
1289 assert_eq!(tokens.len(), 1);
1290 match &tokens[0] {
1291 Token::Hash(s, t) => {
1292 assert_eq!(s, "main");
1293 assert_eq!(*t, crate::types::HashType::Id);
1294 }
1295 _ => panic!("expected Hash, got {:?}", tokens[0]),
1296 }
1297 }
1298
1299 #[test]
1300 fn hash_unrestricted_type() {
1301 // §4.3.1 L801-826 + §4.3.4: `#123` → next `1` is an ident code
1302 // point (digits are ident code points per §4.2), so a hash-token
1303 // is created. `123` does not would-start-an-ident-sequence (digit
1304 // is not ident-start), so type is "unrestricted". The value is
1305 // "123" (consume_an_ident_sequence consumes all ident code points).
1306 let tokens = CssTokenizer::collect("#123");
1307 assert_eq!(tokens.len(), 1);
1308 match &tokens[0] {
1309 Token::Hash(s, t) => {
1310 assert_eq!(s, "123");
1311 assert_eq!(*t, crate::types::HashType::Unrestricted);
1312 }
1313 _ => panic!("expected Hash, got {:?}", tokens[0]),
1314 }
1315 }
1316
1317 #[test]
1318 fn hash_hex_color_is_unrestricted() {
1319 // §4.3.4: `#ff00aa` → starts with `f` which IS ident-start, so
1320 // this is actually type "id" (the hex digits form a valid ident).
1321 let tokens = CssTokenizer::collect("#ff00aa");
1322 assert_eq!(tokens.len(), 1);
1323 match &tokens[0] {
1324 Token::Hash(s, t) => {
1325 assert_eq!(s, "ff00aa");
1326 assert_eq!(*t, crate::types::HashType::Id);
1327 }
1328 _ => panic!("expected Hash, got {:?}", tokens[0]),
1329 }
1330 }
1331
1332 #[test]
1333 fn backslash_escape_in_ident() {
1334 // §4.3.7: `\` followed by non-hex → literal char. Use `z` (not
1335 // `b`, which is a hex digit and would decode to U+000B).
1336 let tokens = CssTokenizer::collect("a\\z");
1337 assert_eq!(tokens.len(), 1);
1338 assert!(matches!(&tokens[0], Token::Ident(s) if s == "az"));
1339 }
1340
1341 #[test]
1342 fn backslash_alone_is_delim() {
1343 // §4.3.1: `\` not followed by valid escape (EOF or newline) →
1344 // <delim-token>
1345 let tokens = CssTokenizer::collect("\\");
1346 assert_eq!(tokens.len(), 1);
1347 assert!(matches!(tokens[0], Token::Delim('\\')));
1348 }
1349
1350 #[test]
1351 fn backslash_newline_is_delim() {
1352 // §4.3.8: `\` followed by newline is not a valid escape.
1353 let tokens = CssTokenizer::collect("\\\n");
1354 assert_eq!(tokens.len(), 2);
1355 assert!(matches!(tokens[0], Token::Delim('\\')));
1356 assert!(matches!(tokens[1], Token::Whitespace));
1357 }
1358
1359 #[test]
1360 fn declaration_with_ident_and_value() {
1361 let tokens = CssTokenizer::collect("color: red");
1362 assert_eq!(tokens.len(), 4);
1363 assert!(matches!(&tokens[0], Token::Ident(s) if s == "color"));
1364 assert!(matches!(tokens[1], Token::Colon));
1365 assert!(matches!(tokens[2], Token::Whitespace));
1366 assert!(matches!(&tokens[3], Token::Ident(s) if s == "red"));
1367 }
1368
1369 #[test]
1370 fn unicode_escape_in_ident() {
1371 // §4.3.7 L1223-1225: `\000026` → '&' (U+0026); after 6 hex digits,
1372 // if next is whitespace, consume it. So `\000026 B` → the space is
1373 // consumed as trailing whitespace, `B` joins the ident sequence →
1374 // single token Ident("&B").
1375 let tokens = CssTokenizer::collect("\\000026 B");
1376 assert_eq!(tokens.len(), 1);
1377 assert!(matches!(&tokens[0], Token::Ident(s) if s == "&B"));
1378 }
1379
1380 #[test]
1381 fn null_escape_becomes_replacement_char() {
1382 // §4.3.7: `\0` → U+FFFD (NULL is not allowed)
1383 let tokens = CssTokenizer::collect("\\0");
1384 assert_eq!(tokens.len(), 1);
1385 assert!(matches!(&tokens[0], Token::Ident(s) if s == "\u{FFFD}"));
1386 }
1387
1388 #[test]
1389 fn surrogate_escape_becomes_replacement_char() {
1390 // §4.3.7: `\D800` → U+FFFD (surrogates not allowed)
1391 let tokens = CssTokenizer::collect("\\D800");
1392 assert_eq!(tokens.len(), 1);
1393 assert!(matches!(&tokens[0], Token::Ident(s) if s == "\u{FFFD}"));
1394 }
1395
1396 // ── C-2 Bug 1 回归测试: # Delim 回退 (§4.3.1 L824-826) ────────────
1397
1398 #[test]
1399 fn hash_alone_is_delim() {
1400 // §4.3.1 L824-826: `#` followed by EOF → next is not ident code
1401 // point, not valid escape → return delim-token with value `#`.
1402 let tokens = CssTokenizer::collect("#");
1403 assert_eq!(tokens.len(), 1);
1404 assert!(matches!(tokens[0], Token::Delim('#')));
1405 }
1406
1407 #[test]
1408 fn hash_followed_by_space_is_delim() {
1409 // §4.3.1 L824-826: `#` followed by space (not ident code point,
1410 // not valid escape) → delim-token with value `#`.
1411 let tokens = CssTokenizer::collect("# ");
1412 assert_eq!(tokens.len(), 2);
1413 assert!(matches!(tokens[0], Token::Delim('#')));
1414 assert!(matches!(tokens[1], Token::Whitespace));
1415 }
1416
1417 #[test]
1418 fn hash_followed_by_at_is_delim() {
1419 // §4.3.1 L824-826: `#` followed by `@` (not ident code point,
1420 // not valid escape) → delim-token with value `#`.
1421 let tokens = CssTokenizer::collect("#@");
1422 assert_eq!(tokens.len(), 2);
1423 assert!(matches!(tokens[0], Token::Delim('#')));
1424 assert!(matches!(tokens[1], Token::Delim('@')));
1425 }
1426
1427 // ── C-3 tests: §4.3.5 string + bad-string ────────────────────────
1428
1429 #[test]
1430 fn string_double_quoted() {
1431 // §4.3.5: "hello" → String("hello")
1432 let tokens = CssTokenizer::collect("\"hello\"");
1433 assert_eq!(tokens.len(), 1);
1434 assert!(matches!(&tokens[0], Token::String(s) if s == "hello"));
1435 }
1436
1437 #[test]
1438 fn string_single_quoted() {
1439 // §4.3.5: 'hello' → String("hello")
1440 let tokens = CssTokenizer::collect("'hello'");
1441 assert_eq!(tokens.len(), 1);
1442 assert!(matches!(&tokens[0], Token::String(s) if s == "hello"));
1443 }
1444
1445 #[test]
1446 fn string_with_escape() {
1447 // §4.3.5 + §4.3.7: "a\z" → String("az") (z is non-hex → literal)
1448 let tokens = CssTokenizer::collect("\"a\\z\"");
1449 assert_eq!(tokens.len(), 1);
1450 assert!(matches!(&tokens[0], Token::String(s) if s == "az"));
1451 }
1452
1453 #[test]
1454 fn string_unterminated_eof() {
1455 // §4.3.5 L1101-1104: EOF before closing quote → parse error,
1456 // return string-token with value accumulated so far.
1457 let tokens = CssTokenizer::collect("\"hello");
1458 assert_eq!(tokens.len(), 1);
1459 assert!(matches!(&tokens[0], Token::String(s) if s == "hello"));
1460 }
1461
1462 #[test]
1463 fn string_unescaped_newline_is_bad_string() {
1464 // §4.3.5 L1106-1110: unescaped newline → parse error, reconsume,
1465 // return bad-string. The reconsumed newline is then tokenized as
1466 // whitespace by the next consume_a_token call.
1467 let tokens = CssTokenizer::collect("\"\n");
1468 assert_eq!(tokens.len(), 2);
1469 assert!(matches!(tokens[0], Token::BadString));
1470 assert!(matches!(tokens[1], Token::Whitespace));
1471 }
1472
1473 #[test]
1474 fn string_line_continuation() {
1475 // §4.3.5 L1117-1119: backslash followed by newline → line
1476 // continuation (newline consumed, no char appended).
1477 let tokens = CssTokenizer::collect("\"a\\\nb\"");
1478 assert_eq!(tokens.len(), 1);
1479 assert!(matches!(&tokens[0], Token::String(s) if s == "ab"));
1480 }
1481
1482 #[test]
1483 fn string_hex_escape() {
1484 // §4.3.5 + §4.3.7: "\26" → String("&") (U+0026)
1485 let tokens = CssTokenizer::collect("\"\\26\"");
1486 assert_eq!(tokens.len(), 1);
1487 assert!(matches!(&tokens[0], Token::String(s) if s == "&"));
1488 }
1489
1490 // ── C-4 tests: §4.3.3/§4.3.10/§4.3.13 number/percentage/dimension ─
1491
1492 #[test]
1493 fn number_integer() {
1494 let tokens = CssTokenizer::collect("42");
1495 assert_eq!(tokens.len(), 1);
1496 assert_eq!(
1497 tokens[0],
1498 Token::Number(Numeric {
1499 value: 42.0,
1500 is_integer: true
1501 })
1502 );
1503 }
1504
1505 #[test]
1506 fn number_decimal() {
1507 let tokens = CssTokenizer::collect("3.5");
1508 assert_eq!(tokens.len(), 1);
1509 assert_eq!(
1510 tokens[0],
1511 Token::Number(Numeric {
1512 value: 3.5,
1513 is_integer: false
1514 })
1515 );
1516 }
1517
1518 #[test]
1519 fn number_signed() {
1520 let tokens = CssTokenizer::collect("-5");
1521 assert_eq!(tokens.len(), 1);
1522 assert_eq!(
1523 tokens[0],
1524 Token::Number(Numeric {
1525 value: -5.0,
1526 is_integer: true
1527 })
1528 );
1529 }
1530
1531 #[test]
1532 fn number_exponent() {
1533 // §4.3.13: 1e3 → 1.0 * 10^3 = 1000.0, type "number"
1534 let tokens = CssTokenizer::collect("1e3");
1535 assert_eq!(tokens.len(), 1);
1536 assert_eq!(
1537 tokens[0],
1538 Token::Number(Numeric {
1539 value: 1000.0,
1540 is_integer: false
1541 })
1542 );
1543 }
1544
1545 #[test]
1546 fn number_decimal_exponent() {
1547 // §4.3.13: 1.5e2 → 1.5 * 10^2 = 150.0, type "number"
1548 let tokens = CssTokenizer::collect("1.5e2");
1549 assert_eq!(tokens.len(), 1);
1550 assert_eq!(
1551 tokens[0],
1552 Token::Number(Numeric {
1553 value: 150.0,
1554 is_integer: false
1555 })
1556 );
1557 }
1558
1559 #[test]
1560 fn percentage_token() {
1561 let tokens = CssTokenizer::collect("50%");
1562 assert_eq!(tokens.len(), 1);
1563 assert_eq!(
1564 tokens[0],
1565 Token::Percentage(Numeric {
1566 value: 50.0,
1567 is_integer: true
1568 })
1569 );
1570 }
1571
1572 #[test]
1573 fn dimension_px() {
1574 let tokens = CssTokenizer::collect("10px");
1575 assert_eq!(tokens.len(), 1);
1576 assert_eq!(
1577 tokens[0],
1578 Token::Dimension(
1579 Numeric {
1580 value: 10.0,
1581 is_integer: true
1582 },
1583 "px".to_string(),
1584 )
1585 );
1586 }
1587
1588 #[test]
1589 fn dimension_em() {
1590 let tokens = CssTokenizer::collect("1.5em");
1591 assert_eq!(tokens.len(), 1);
1592 assert_eq!(
1593 tokens[0],
1594 Token::Dimension(
1595 Numeric {
1596 value: 1.5,
1597 is_integer: false
1598 },
1599 "em".to_string(),
1600 )
1601 );
1602 }
1603
1604 #[test]
1605 fn dimension_signed() {
1606 let tokens = CssTokenizer::collect("-30deg");
1607 assert_eq!(tokens.len(), 1);
1608 assert_eq!(
1609 tokens[0],
1610 Token::Dimension(
1611 Numeric {
1612 value: -30.0,
1613 is_integer: true
1614 },
1615 "deg".to_string(),
1616 )
1617 );
1618 }
1619
1620 #[test]
1621 fn plus_sign_number() {
1622 let tokens = CssTokenizer::collect("+5");
1623 assert_eq!(tokens.len(), 1);
1624 assert_eq!(
1625 tokens[0],
1626 Token::Number(Numeric {
1627 value: 5.0,
1628 is_integer: true
1629 })
1630 );
1631 }
1632
1633 #[test]
1634 fn dot_starts_number() {
1635 let tokens = CssTokenizer::collect(".5");
1636 assert_eq!(tokens.len(), 1);
1637 assert_eq!(
1638 tokens[0],
1639 Token::Number(Numeric {
1640 value: 0.5,
1641 is_integer: false
1642 })
1643 );
1644 }
1645
1646 // ── C-5 tests: §4.3.4 url( special case + §4.3.6 Url + §4.3.15 BadUrl ─
1647
1648 #[test]
1649 fn url_unquoted_simple() {
1650 // §4.3.6: url(foo) → Url("foo")
1651 let tokens = CssTokenizer::collect("url(foo)");
1652 assert_eq!(tokens.len(), 1);
1653 assert_eq!(tokens[0], Token::Url("foo".to_string()));
1654 }
1655
1656 #[test]
1657 fn url_unquoted_with_spaces() {
1658 // §4.3.6: leading/trailing whitespace consumed; url( foo ) → Url("foo")
1659 let tokens = CssTokenizer::collect("url( foo )");
1660 assert_eq!(tokens.len(), 1);
1661 assert_eq!(tokens[0], Token::Url("foo".to_string()));
1662 }
1663
1664 #[test]
1665 fn url_empty() {
1666 // §4.3.6: url() → Url("")
1667 let tokens = CssTokenizer::collect("url()");
1668 assert_eq!(tokens.len(), 1);
1669 assert_eq!(tokens[0], Token::Url(String::new()));
1670 }
1671
1672 #[test]
1673 fn url_eof_unterminated() {
1674 // §4.3.6 L1161-1164: EOF → parse error, return url-token with value so far
1675 let tokens = CssTokenizer::collect("url(foo");
1676 assert_eq!(tokens.len(), 1);
1677 assert_eq!(tokens[0], Token::Url("foo".to_string()));
1678 }
1679
1680 #[test]
1681 fn url_quoted_is_function() {
1682 // §4.3.4 L1058-1063: url("foo") → Function("url"); the body
1683 // `"foo")` remains in the stream → String("foo") + CloseParen.
1684 let tokens = CssTokenizer::collect("url(\"foo\")");
1685 assert_eq!(tokens.len(), 3);
1686 assert_eq!(tokens[0], Token::Function("url".to_string()));
1687 assert_eq!(tokens[1], Token::String("foo".to_string()));
1688 assert_eq!(tokens[2], Token::CloseParen);
1689 }
1690
1691 #[test]
1692 fn url_single_quoted_is_function() {
1693 // §4.3.4 L1058-1063: url('foo') → Function("url") + String + CloseParen
1694 let tokens = CssTokenizer::collect("url('foo')");
1695 assert_eq!(tokens.len(), 3);
1696 assert_eq!(tokens[0], Token::Function("url".to_string()));
1697 assert_eq!(tokens[1], Token::String("foo".to_string()));
1698 assert_eq!(tokens[2], Token::CloseParen);
1699 }
1700
1701 #[test]
1702 fn url_ws_then_quote_is_function() {
1703 // §4.3.4 L1058-1063: url( "foo") → whitespace+quote → Function.
1704 // L1056-1057 only collapses when *two* consecutive whitespace
1705 // follow `(`; here only one space precedes `"`, so it is NOT
1706 // consumed and becomes a Whitespace token.
1707 let tokens = CssTokenizer::collect("url( \"foo\")");
1708 assert_eq!(tokens.len(), 4);
1709 assert_eq!(tokens[0], Token::Function("url".to_string()));
1710 assert_eq!(tokens[1], Token::Whitespace);
1711 assert_eq!(tokens[2], Token::String("foo".to_string()));
1712 assert_eq!(tokens[3], Token::CloseParen);
1713 }
1714
1715 #[test]
1716 fn url_with_escape() {
1717 // §4.3.6 + §4.3.7: url(foo\29 bar) → \29 consumes trailing space,
1718 // decodes to `)`, then `bar` appended → Url("foo)bar")
1719 let tokens = CssTokenizer::collect("url(foo\\29 bar)");
1720 assert_eq!(tokens.len(), 1);
1721 assert_eq!(tokens[0], Token::Url("foo)bar".to_string()));
1722 }
1723
1724 #[test]
1725 fn url_bad_paren_in_unquoted() {
1726 // §4.3.6 L1177-1185: `(` in unquoted url → BadUrl
1727 let tokens = CssTokenizer::collect("url(foo(bar)");
1728 assert_eq!(tokens.len(), 1);
1729 assert_eq!(tokens[0], Token::BadUrl);
1730 }
1731
1732 #[test]
1733 fn url_bad_quote_in_unquoted() {
1734 // §4.3.6 L1177-1185: `"` in unquoted url → BadUrl
1735 let tokens = CssTokenizer::collect("url(foo\"bar)");
1736 assert_eq!(tokens.len(), 1);
1737 assert_eq!(tokens[0], Token::BadUrl);
1738 }
1739
1740 // ── C-6 tests: §4.3.1 unicode_ranges_allowed + §4.3.11 + §4.3.14 ──
1741
1742 #[test]
1743 fn unicode_range_disabled_by_default() {
1744 // §4.3.1 L782-783: default unicode_ranges_allowed=false.
1745 // `U+1234` → `U` ident-start → Ident("U") (`+` stops ident seq);
1746 // then `+1234` → starts_with_number (`+`+digit) → Number(1234).
1747 let tokens = CssTokenizer::collect("U+1234");
1748 assert_eq!(tokens.len(), 2);
1749 assert_eq!(tokens[0], Token::Ident("U".to_string()));
1750 assert_eq!(
1751 tokens[1],
1752 Token::Number(Numeric {
1753 value: 1234.0,
1754 is_integer: true
1755 })
1756 );
1757 }
1758
1759 #[test]
1760 fn unicode_range_simple() {
1761 // §4.3.14: U+1234 → UnicodeRange(0x1234, 0x1234)
1762 let mut tz = CssTokenizer::new("U+1234");
1763 tz.set_unicode_ranges_allowed(true);
1764 let t = tz.next_token().unwrap();
1765 assert_eq!(t, Token::UnicodeRange(Some(0x1234), Some(0x1234)));
1766 }
1767
1768 #[test]
1769 fn unicode_range_question_marks() {
1770 // §4.3.14 step 3: U+12?? → start=0x1200 (?→0), end=0x12FF (?→F)
1771 let mut tz = CssTokenizer::new("U+12??");
1772 tz.set_unicode_ranges_allowed(true);
1773 let t = tz.next_token().unwrap();
1774 assert_eq!(t, Token::UnicodeRange(Some(0x1200), Some(0x12FF)));
1775 }
1776
1777 #[test]
1778 fn unicode_range_range() {
1779 // §4.3.14 step 5: U+12-34FF → start=0x12, end=0x34FF
1780 let mut tz = CssTokenizer::new("U+12-34FF");
1781 tz.set_unicode_ranges_allowed(true);
1782 let t = tz.next_token().unwrap();
1783 assert_eq!(t, Token::UnicodeRange(Some(0x12), Some(0x34FF)));
1784 }
1785
1786 #[test]
1787 fn unicode_range_max_hex() {
1788 // §4.3.14: U+10FFFF → UnicodeRange(0x10FFFF, 0x10FFFF)
1789 let mut tz = CssTokenizer::new("U+10FFFF");
1790 tz.set_unicode_ranges_allowed(true);
1791 let t = tz.next_token().unwrap();
1792 assert_eq!(t, Token::UnicodeRange(Some(0x10FFFF), Some(0x10FFFF)));
1793 }
1794
1795 #[test]
1796 fn unicode_range_lowercase_u() {
1797 // §4.3.11: lowercase u also triggers; u+abc → UnicodeRange(0xABC, 0xABC)
1798 let mut tz = CssTokenizer::new("u+abc");
1799 tz.set_unicode_ranges_allowed(true);
1800 let t = tz.next_token().unwrap();
1801 assert_eq!(t, Token::UnicodeRange(Some(0xABC), Some(0xABC)));
1802 }
1803}