muskitty_css_tokenizer/impls.rs
1//! CSS Syntax tokenizer implementation.
2//!
3//! Implements the "Tokenizer Algorithms" of CSS Syntax Module Level 3 §4.3.
4//! The tokenizer is recursive-descent: [`CssTokenizer::next_token`] calls
5//! [`consume_a_token`](CssTokenizer::consume_a_token) (§4.3.1), which
6//! dispatches to sub-algorithms based on the current input code point.
7//!
8//! # Coverage
9//!
10//! All §4.3 sub-algorithms are implemented:
11//! - §4.3.1 Consume a token (full dispatch, incl. `unicode_ranges_allowed`)
12//! - §4.3.2 Consume comments
13//! - §4.3.3 Consume a numeric token
14//! - §4.3.4 Consume an ident-like token (incl. `url(` special case)
15//! - §4.3.5 Consume a string token
16//! - §4.3.6 Consume a url token
17//! - §4.3.7 Consume an escaped code point
18//! - §4.3.8 Check if two code points are a valid escape
19//! - §4.3.9 Check if three code points would start an ident sequence
20//! - §4.3.10 Check if three code points would start a number
21//! - §4.3.11 Check if three code points would start a unicode-range
22//! - §4.3.12 Consume an ident sequence
23//! - §4.3.13 Consume a number
24//! - §4.3.14 Consume a unicode-range token
25//! - §4.3.15 Consume the remnants of a bad url
26
27use crate::trait_def::Tokenizer;
28use crate::types::{HashType, Numeric, State, Token};
29
30/// The CSS tokenizer (§4.3).
31///
32/// Holds the preprocessed input stream (§5.3 normalization applied at
33/// construction), the current position, and EOF tracking. Sub-algorithms
34/// are implemented as methods on this struct so they share the input
35/// stream and position.
36pub struct CssTokenizer {
37 /// Input code points after §5.3 preprocessing (CR/LF/FF normalized to LF).
38 input: Vec<char>,
39 /// Current position in `input` (0-based index of the next code point
40 /// to consume). `pos == input.len()` means the stream is exhausted.
41 pos: usize,
42 /// Top-level state (§4.3.1 dispatch vs. EOF).
43 state: State,
44 /// Whether `<EOF-token>` has been emitted. After this is `true`,
45 /// [`next_token`](Tokenizer::next_token) returns `None`.
46 eof_emitted: bool,
47 /// §4.3.1 L782-783: `unicode_ranges_allowed` flag, default `false`.
48 /// When `true`, the `U+`/`u+` branch of §4.3.1 produces a
49 /// `<unicode-range-token>` (§4.3.14); otherwise `U`/`u` is tokenized
50 /// as an ident-like token.
51 unicode_ranges_allowed: bool,
52}
53
54impl CssTokenizer {
55 /// Construct a new tokenizer over `input`.
56 ///
57 /// Applies §5.3 input preprocessing: per the "input stream"
58 /// definition, U+000D CR followed by U+000A LF is collapsed to a
59 /// single LF, lone CR is replaced with LF, and U+000C FF is replaced
60 /// with LF. The tokenizer operates on this normalized stream.
61 pub fn new(input: &str) -> Self {
62 let input = preprocess_input(input);
63 Self {
64 input,
65 pos: 0,
66 state: State::Data,
67 eof_emitted: false,
68 unicode_ranges_allowed: false,
69 }
70 }
71
72 /// Peek at the code point at `offset` from the current position
73 /// without consuming it. Returns `None` if past end of input.
74 ///
75 /// Used by §4.3.8 / §4.3.9 / §4.3.10 / §4.3.11 "check if N code
76 /// points..." predicates, which must look ahead without advancing.
77 fn peek(&self, offset: usize) -> Option<char> {
78 self.input.get(self.pos + offset).copied()
79 }
80
81 /// Consume and return the next code point, advancing `pos`. Returns
82 /// `None` if at end of input.
83 ///
84 /// Implements the "consume the next input code point" primitive used
85 /// throughout §4.3.
86 fn consume(&mut self) -> Option<char> {
87 let c = self.input.get(self.pos).copied();
88 if c.is_some() {
89 self.pos += 1;
90 }
91 c
92 }
93
94 /// Re-consume the current code point: step `pos` back by one.
95 ///
96 /// Per §4.3.1 "reconsume the current input code point": the next
97 /// call to [`consume`](Self::consume) will return the same code point
98 /// that was just consumed. No-op if `pos == 0`.
99 fn reconsume(&mut self) {
100 if self.pos > 0 {
101 self.pos -= 1;
102 }
103 }
104
105 /// §4.3.1 Consume a token.
106 ///
107 /// This is the main entry point of the tokenizer. Dispatches based on
108 /// the current input code point to one of the sub-algorithms (§4.3.2
109 /// through §4.3.6) or emits a simple token directly. The optional
110 /// `unicode_ranges_allowed` flag (§4.3.1 L782-783, default `false`)
111 /// gates the `U+`/`u+` unicode-range branch.
112 fn consume_a_token(&mut self) -> Token {
113 // §4.3.2: If the next two input code points are U+002F SOLIDUS
114 // (`/`) and U+002A ASTERISK (`*`), consume comments and
115 // recursively consume the next token. We implement this as a
116 // loop: any leading run of `/* ... */` comments is skipped before
117 // producing the next real token.
118 loop {
119 let Some(c) = self.consume() else {
120 // §5.3: end of stream → <EOF-token>
121 self.eof_emitted = true;
122 self.state = State::Eof;
123 return Token::Eof;
124 };
125
126 // §4.3.2 comment dispatch: `/` followed by `*`.
127 if c == '/' && self.peek(0) == Some('*') {
128 self.consume(); // consume `*`
129 self.consume_comments_body();
130 // Loop back to consume the next token (which may itself
131 // be preceded by another comment).
132 continue;
133 }
134
135 return match c {
136 // §4.3.1: whitespace → <whitespace-token>
137 ' ' | '\t' | '\n' | '\r' | '\u{000C}' => {
138 // Consume all consecutive whitespace.
139 while let Some(next) = self.peek(0) {
140 if matches!(next, ' ' | '\t' | '\n' | '\r' | '\u{000C}') {
141 self.consume();
142 } else {
143 break;
144 }
145 }
146 Token::Whitespace
147 }
148 // §4.3.1: `"` or `'` → consume a string token (§4.3.5)
149 '"' | '\'' => self.consume_a_string_token(c),
150 // §4.3.1: `#` → consume an ident-like token (§4.3.4) for hash
151 '#' => self.consume_a_hash_token(),
152 // §4.3.1: `(` → <(-token>
153 '(' => Token::OpenParen,
154 // §4.3.1: `)` → <)-token>
155 ')' => Token::CloseParen,
156 // §4.3.1: `+` or `-` → if would-start-a-number, consume a
157 // numeric token (§4.3.3); else <delim-token>.
158 // §4.3.1: `-` additionally: if next two are `-` and `>`
159 // (i.e. `-->`), consume them and return <CDC-token>.
160 // §4.3.1: `.` → if would-start-a-number, consume a numeric
161 // token; else <delim-token>.
162 '-' => {
163 if self.starts_with_number(c) {
164 self.reconsume();
165 self.consume_a_numeric_token()
166 } else if self.peek(0) == Some('-') && self.peek(1) == Some('>') {
167 // §4.3.1: `-->` → <CDC-token>
168 self.consume(); // consume second `-`
169 self.consume(); // consume `>`
170 Token::Cdc
171 } else if self.would_start_ident_sequence_at(0) {
172 // §4.3.1: `-` starting an ident sequence → reconsume
173 // and consume an ident-like token.
174 self.reconsume();
175 self.consume_an_ident_like_token()
176 } else {
177 Token::Delim(c)
178 }
179 }
180 '+' | '.' => {
181 if self.starts_with_number(c) {
182 self.reconsume();
183 self.consume_a_numeric_token()
184 } else {
185 Token::Delim(c)
186 }
187 }
188 // §4.3.1: `<` → if next three are `!`, `-`, `-` (i.e.
189 // `<!--`), consume them and return <CDO-token>. Otherwise
190 // emit <delim-token>.
191 '<' => {
192 if self.peek(0) == Some('!')
193 && self.peek(1) == Some('-')
194 && self.peek(2) == Some('-')
195 {
196 // Consume `!`, `-`, `-`.
197 self.consume();
198 self.consume();
199 self.consume();
200 Token::Cdo
201 } else {
202 Token::Delim(c)
203 }
204 }
205 // §4.3.1: `@` → if next would-start-an-ident-sequence,
206 // consume an ident-like token (§4.3.4) for at-keyword.
207 '@' => self.consume_an_at_keyword_token(),
208 // §4.3.1: `[` → <[-token>
209 '[' => Token::OpenBracket,
210 // §4.3.1: `\` → if would-start-an-escape (§4.3.8), reconsume
211 // and consume an ident-like token (§4.3.4); else parse error,
212 // <delim-token>.
213 '\\' => self.consume_ident_or_delim(),
214 // §4.3.1: `]` → <]-token>
215 ']' => Token::CloseBracket,
216 // §4.3.1: `{` → <{-token>
217 '{' => Token::OpenBrace,
218 // §4.3.1: `}` → <}-token>
219 '}' => Token::CloseBrace,
220 // §4.3.1: digit → reconsume and consume a numeric token (§4.3.3)
221 '0'..='9' => {
222 self.reconsume();
223 self.consume_a_numeric_token()
224 }
225 // §4.3.1: `U+` or `u+` → if would-start-a-unicode-range (§4.3.11),
226 // consume a unicode-range token (§4.3.14).
227 'u' | 'U' => self.consume_u_or_unicode_range(),
228 // §4.3.1: ident-start code point → reconsume and consume an
229 // ident-like token (§4.3.4)
230 _ if is_ident_start_code_point(c) => {
231 self.reconsume();
232 self.consume_an_ident_like_token()
233 }
234 // §4.3.1: `:` → <colon-token>
235 ':' => Token::Colon,
236 // §4.3.1: `;` → <semicolon-token>
237 ';' => Token::Semicolon,
238 // §4.3.1: `,` → <comma-token>
239 ',' => Token::Comma,
240 // §4.3.1: anything else → <delim-token>
241 _ => Token::Delim(c),
242 };
243 }
244 }
245
246 /// §4.3.2 Consume comments — body only.
247 ///
248 /// Precondition: the opening `/*` has already been consumed. Consume
249 /// code points until the closing `*/` or EOF. Per §4.3.2, an
250 /// unterminated comment (EOF before `*/`) is a parse error; we
251 /// silently consume to EOF in that case (no error reporting yet).
252 fn consume_comments_body(&mut self) {
253 loop {
254 match self.consume() {
255 None => return, // EOF in comment: parse error (unreported)
256 Some('*') => {
257 if self.peek(0) == Some('/') {
258 self.consume(); // consume `/`
259 return;
260 }
261 }
262 Some(_) => {}
263 }
264 }
265 }
266
267 // ── Sub-algorithm stubs (implemented in subsequent commits) ───────
268
269 /// §4.3.5 (L1081-1128) Consume a string token.
270 ///
271 /// `quote` is the ending code point (the opening quote `"` or `'`).
272 /// Returns a `string-token` (normal or EOF-terminated) or a
273 /// `bad-string-token` (unescaped newline).
274 ///
275 /// Per §4.3.5:
276 /// - ending code point (L1097-1099) → return string-token.
277 /// - EOF (L1101-1104) → parse error, return string-token.
278 /// - newline (L1106-1110) → parse error, reconsume, return bad-string.
279 /// - `\` (L1112-1124):
280 /// - next EOF → do nothing (loop; next consume hits EOF → string).
281 /// - next newline → consume it (line continuation).
282 /// - otherwise (valid escape) → consume an escaped code point, append.
283 /// - anything else (L1126-1128) → append.
284 fn consume_a_string_token(&mut self, quote: char) -> Token {
285 let mut value = String::new();
286 loop {
287 match self.consume() {
288 None => {
289 // §4.3.5 L1101-1104: EOF → parse error, return string-token.
290 return Token::String(value);
291 }
292 Some(c) if c == quote => {
293 // §4.3.5 L1097-1099: ending code point → return string-token.
294 return Token::String(value);
295 }
296 Some('\n') => {
297 // §4.3.5 L1106-1110: newline → parse error, reconsume, return bad-string.
298 self.reconsume();
299 return Token::BadString;
300 }
301 Some('\\') => {
302 // §4.3.5 L1112-1124: escape handling.
303 match self.peek(0) {
304 None => {
305 // §4.3.5 L1114-1115: next is EOF, do nothing.
306 // Loop continues; next consume() returns None → String.
307 }
308 Some('\n') => {
309 // §4.3.5 L1117-1119: next is newline, consume it (line continuation).
310 self.consume();
311 }
312 Some(_) => {
313 // §4.3.5 L1121-1124: valid escape, consume escaped code point.
314 let escaped = self.consume_an_escaped_code_point();
315 value.push(escaped);
316 }
317 }
318 }
319 Some(c) => {
320 // §4.3.5 L1126-1128: anything else → append.
321 value.push(c);
322 }
323 }
324 }
325 }
326
327 /// §4.3.1 (L801-826) `#` branch: Consume a hash token.
328 ///
329 /// Precondition: `#` has been consumed. Per §4.3.1 L801-826:
330 /// - If the next code point is an ident code point OR the next two
331 /// are a valid escape, then create a `hash-token` (type "id" if
332 /// would-start-ident-sequence, else "unrestricted"), consume an
333 /// ident sequence for the value, and return it.
334 /// - Otherwise, return a `delim-token` with value `#`.
335 fn consume_a_hash_token(&mut self) -> Token {
336 // §4.3.1 L803-805: next is ident code point OR next two are valid escape.
337 let next_is_ident = self.peek(0).is_some_and(is_ident_code_point);
338 let next_is_valid_escape = self.is_valid_escape_at(0);
339 if !next_is_ident && !next_is_valid_escape {
340 // §4.3.1 L824-826: return delim-token with value `#`.
341 return Token::Delim('#');
342 }
343 let hash_type = if self.would_start_ident_sequence_at(0) {
344 HashType::Id
345 } else {
346 HashType::Unrestricted
347 };
348 let name = self.consume_an_ident_sequence();
349 Token::Hash(name, hash_type)
350 }
351
352 /// §4.3.3 (L1011-1042) Consume a numeric token.
353 ///
354 /// Returns a `number-token`, `percentage-token`, or `dimension-token`.
355 /// Per §4.3.3:
356 /// 1. Consume a number (§4.3.13).
357 /// 2. If next 3 would start an ident sequence → dimension-token (consume
358 /// an ident sequence for the unit).
359 /// 3. Else if next is `%` → percentage-token (consume `%`).
360 /// 4. Else → number-token.
361 fn consume_a_numeric_token(&mut self) -> Token {
362 let (value, is_integer) = self.consume_a_number();
363 let numeric = Numeric { value, is_integer };
364 // §4.3.3 L1019-1029: dimension
365 if self.would_start_ident_sequence_at(0) {
366 let unit = self.consume_an_ident_sequence();
367 return Token::Dimension(numeric, unit);
368 }
369 // §4.3.3 L1032-1036: percentage
370 if self.peek(0) == Some('%') {
371 self.consume();
372 return Token::Percentage(numeric);
373 }
374 // §4.3.3 L1040-1042: number
375 Token::Number(numeric)
376 }
377
378 /// §4.3.13 (L1415-1483) Consume a number.
379 ///
380 /// Returns `(value, is_integer)` where `is_integer` is true when the
381 /// source representation had no fractional part and no exponent (type
382 /// "integer"), false when it had either (type "number"). The sign is
383 /// included in `value`.
384 ///
385 /// Per §4.3.13:
386 /// 1. type = "integer"; number_part = ""; exponent_part = "".
387 /// 2. If next is `+`/`-`, consume, append to number_part.
388 /// 3. While next is digit, consume, append to number_part.
389 /// 4. If next 2 are `.` + digit: consume `.`, consume digits; type = "number".
390 /// 5. If next 2/3 are `e`/`E` + optional `+`/`-` + digit: consume `e`/`E`,
391 /// optional sign, digits into exponent_part; type = "number".
392 /// 6. value = parse number_part; if exponent_part non-empty, value *= 10^exp.
393 /// 7. Return value, type, sign.
394 fn consume_a_number(&mut self) -> (f64, bool) {
395 let mut type_is_number = false;
396 let mut number_part = String::new();
397 let mut exponent_part = String::new();
398
399 // §4.3.13 L1436-1440: sign
400 if self.peek(0) == Some('+') || self.peek(0) == Some('-') {
401 number_part.push(self.consume().unwrap());
402 }
403
404 // §4.3.13 L1442-1444: integer digits
405 while let Some(d) = self.peek(0) {
406 if is_digit(d) {
407 number_part.push(d);
408 self.consume();
409 } else {
410 break;
411 }
412 }
413
414 // §4.3.13 L1446-1454: fraction part
415 if self.peek(0) == Some('.') && matches!(self.peek(1), Some(d) if is_digit(d)) {
416 number_part.push(self.consume().unwrap()); // consume `.`
417 type_is_number = true;
418 while let Some(d) = self.peek(0) {
419 if is_digit(d) {
420 number_part.push(d);
421 self.consume();
422 } else {
423 break;
424 }
425 }
426 }
427
428 // §4.3.13 L1457-1469: exponent part
429 if matches!(self.peek(0), Some('e') | Some('E')) {
430 let p1 = self.peek(1);
431 let p2 = self.peek(2);
432 let valid_exp = match p1 {
433 Some('+') | Some('-') => matches!(p2, Some(d) if is_digit(d)),
434 Some(d) if is_digit(d) => true,
435 _ => false,
436 };
437 if valid_exp {
438 self.consume(); // consume `e`/`E`
439 if self.peek(0) == Some('+') || self.peek(0) == Some('-') {
440 exponent_part.push(self.consume().unwrap());
441 }
442 while let Some(d) = self.peek(0) {
443 if is_digit(d) {
444 exponent_part.push(d);
445 self.consume();
446 } else {
447 break;
448 }
449 }
450 type_is_number = true;
451 }
452 }
453
454 // §4.3.13 L1472-1480: compute value
455 let mut value: f64 = number_part.parse().unwrap_or(0.0);
456 if !exponent_part.is_empty() {
457 let exp: i32 = exponent_part.parse().unwrap_or(0);
458 value *= 10.0f64.powi(exp);
459 }
460
461 (value, !type_is_number)
462 }
463
464 /// §4.3.4 (L1045-1078) Consume an ident-like token.
465 ///
466 /// Precondition: the current position is at the start of an ident
467 /// sequence (the caller has not consumed anything; or has reconsumed
468 /// the first code point). Returns an `ident-token`, `function-token`,
469 /// `url-token`, or `bad-url-token`.
470 ///
471 /// Per §4.3.4:
472 /// 1. Consume an ident sequence (§4.3.12), yielding `name`.
473 /// 2. If `name` is ASCII-case-insensitive "url" and next is `(`:
474 /// consume `(`, collapse leading whitespace pairs, then if the
475 /// next 1-2 code points are a quote (or whitespace+quote) return
476 /// a `function-token`; otherwise consume a url token (§4.3.6).
477 /// 3. Else if next is `(`: consume it, return `function-token`.
478 /// 4. Else return `ident-token`.
479 fn consume_an_ident_like_token(&mut self) -> Token {
480 let name = self.consume_an_ident_sequence();
481 // §4.3.4 L1053-1066: url( special case
482 if name.eq_ignore_ascii_case("url") && self.peek(0) == Some('(') {
483 self.consume(); // consume `(`
484 // §4.3.4 L1056-1057: while next two are whitespace, consume one
485 while self.peek(0).is_some_and(is_whitespace) && self.peek(1).is_some_and(is_whitespace)
486 {
487 self.consume();
488 }
489 // §4.3.4 L1058-1063: if next 1-2 are " / ' / ws+" / ws+' → Function
490 let p0 = self.peek(0);
491 let p1 = self.peek(1);
492 let is_quote_case = matches!(p0, Some('"') | Some('\''))
493 || (p0.is_some_and(is_whitespace) && matches!(p1, Some('"') | Some('\'')));
494 if is_quote_case {
495 return Token::Function(name);
496 }
497 // §4.3.4 L1064-1066: otherwise consume a url token
498 return self.consume_a_url_token();
499 }
500 // §4.3.4 L1068-1073: ordinary function
501 if self.peek(0) == Some('(') {
502 self.consume();
503 return Token::Function(name);
504 }
505 // §4.3.4 L1075-1078: ident
506 Token::Ident(name)
507 }
508
509 /// §4.3.4 (at-keyword branch) Consume an at-keyword token.
510 ///
511 /// Precondition: `@` has been consumed. Per §4.3.1, if the next code
512 /// points would start an ident sequence, consume it and return an
513 /// `<at-keyword-token>`; otherwise return a `<delim-token>` with `@`.
514 fn consume_an_at_keyword_token(&mut self) -> Token {
515 if self.would_start_ident_sequence_at(0) {
516 let name = self.consume_an_ident_sequence();
517 Token::AtKeyword(name)
518 } else {
519 Token::Delim('@')
520 }
521 }
522
523 /// §4.3.1 (backslash branch): if would-start-an-escape, consume an
524 /// ident-like token; else emit `<delim-token>`.
525 fn consume_ident_or_delim(&mut self) -> Token {
526 if self.is_valid_escape_next() {
527 self.reconsume();
528 self.consume_an_ident_like_token()
529 } else {
530 Token::Delim('\\')
531 }
532 }
533
534 /// §4.3.6 (L1132-1203) Consume a url token.
535 ///
536 /// Precondition: `url(` has been consumed by the caller (§4.3.4
537 /// L1053-1066). This algorithm consumes the "unquoted" url body and
538 /// returns a `url-token` or `bad-url-token`. A quoted value like
539 /// `url("foo")` is handled by §4.3.4 as a `function-token` and never
540 /// reaches here.
541 ///
542 /// Per §4.3.6:
543 /// - `)` → return url-token.
544 /// - EOF → parse error, return url-token.
545 /// - whitespace → consume trailing whitespace; if next is `)` or EOF
546 /// return url-token, else bad-url.
547 /// - `"` / `'` / `(` / non-printable → parse error, bad-url.
548 /// - `\` → if valid escape, consume escaped and append; else bad-url.
549 /// - anything else → append.
550 fn consume_a_url_token(&mut self) -> Token {
551 let mut value = String::new();
552 // §4.3.6 L1151: consume as much whitespace as possible
553 while self.peek(0).is_some_and(is_whitespace) {
554 self.consume();
555 }
556 loop {
557 match self.consume() {
558 // §4.3.6 L1157-1159: `)` → return url-token
559 Some(')') => return Token::Url(value),
560 // §4.3.6 L1161-1164: EOF → parse error, return url-token
561 None => return Token::Url(value),
562 // §4.3.6 L1166-1175: whitespace → consume trailing ws, then check
563 Some(c) if is_whitespace(c) => {
564 while self.peek(0).is_some_and(is_whitespace) {
565 self.consume();
566 }
567 match self.peek(0) {
568 Some(')') => {
569 self.consume();
570 return Token::Url(value);
571 }
572 None => return Token::Url(value), // parse error
573 _ => {
574 self.consume_the_remnants_of_a_bad_url();
575 return Token::BadUrl;
576 }
577 }
578 }
579 // §4.3.6 L1177-1185: " / ' / ( / non-printable → bad url
580 Some(c) if c == '"' || c == '\'' || c == '(' || is_non_printable(c) => {
581 self.consume_the_remnants_of_a_bad_url();
582 return Token::BadUrl;
583 }
584 // §4.3.6 L1187-1197: `\`
585 Some('\\') => {
586 if self.is_valid_escape_next() {
587 let escaped = self.consume_an_escaped_code_point();
588 value.push(escaped);
589 } else {
590 self.consume_the_remnants_of_a_bad_url();
591 return Token::BadUrl;
592 }
593 }
594 // §4.3.6 L1199-1202: anything else → append
595 Some(c) => value.push(c),
596 }
597 }
598 }
599
600 /// §4.3.15 (L1551-1577) Consume the remnants of a bad url.
601 ///
602 /// Consumes code points until `)` or EOF, allowing an escaped `)`
603 /// (`\)`) to be consumed without ending the bad-url. Per §4.3.15:
604 /// - `)` or EOF → return.
605 /// - valid escape (stream starts with `\` + non-newline non-EOF) →
606 /// consume an escaped code point.
607 /// - anything else → do nothing.
608 fn consume_the_remnants_of_a_bad_url(&mut self) {
609 loop {
610 match self.consume() {
611 // §4.3.15 L1563-1566: `)` or EOF → return
612 Some(')') | None => return,
613 // §4.3.15 L1568-1572: valid escape → consume escaped code point
614 Some('\\') if self.is_valid_escape_next() => {
615 let _ = self.consume_an_escaped_code_point();
616 }
617 // §4.3.15 L1574-1576: anything else → do nothing
618 Some(_) => {}
619 }
620 }
621 }
622
623 /// §4.3.1 (L960-972) U/u branch: if `unicode_ranges_allowed` is true
624 /// and the stream would start a unicode-range (§4.3.11), consume a
625 /// unicode-range token (§4.3.14); otherwise consume an ident-like token.
626 ///
627 /// Precondition: `u`/`U` has been consumed, `pos` points just past it.
628 /// This method reconsumes back to `u` before dispatching.
629 fn consume_u_or_unicode_range(&mut self) -> Token {
630 self.reconsume(); // pos back to `u`/`U`
631 // §4.3.1 L963-967: unicode_ranges_allowed && would-start-unicode-range
632 if self.unicode_ranges_allowed && self.would_start_unicode_range_at(0) {
633 return self.consume_a_unicode_range_token();
634 }
635 // §4.3.1 L969-972: otherwise ident-like
636 self.consume_an_ident_like_token()
637 }
638
639 /// §4.3.11 (L1356-1378) Check if three code points would start a
640 /// unicode-range, examining code points starting at `offset` from the
641 /// current position.
642 ///
643 /// Per §4.3.11: true iff first is `U`/`u`, second is `+`, third is `?`
644 /// or a hex digit.
645 fn would_start_unicode_range_at(&self, offset: usize) -> bool {
646 let first = match self.peek(offset) {
647 Some(c) => c,
648 None => return false,
649 };
650 if !matches!(first, 'U' | 'u') {
651 return false;
652 }
653 if self.peek(offset + 1) != Some('+') {
654 return false;
655 }
656 match self.peek(offset + 2) {
657 Some('?') => true,
658 Some(c) if is_hex_digit(c) => true,
659 _ => false,
660 }
661 }
662
663 /// §4.3.14 (L1487-1548) Consume a unicode-range token.
664 ///
665 /// Precondition: the stream starts with `u`/`U` + `+` + (`?` or hex
666 /// digit) (i.e. [`would_start_unicode_range_at`](Self::would_start_unicode_range_at)
667 /// returns true at the current position).
668 ///
669 /// Returns `UnicodeRange(Some(start), Some(end))`.
670 ///
671 /// Per §4.3.14:
672 /// 1. Consume and discard `u`/`U` and `+`.
673 /// 2. Consume up to 6 hex digits; if fewer than 6, consume `?` up to a
674 /// total of 6 → `first_segment`.
675 /// 3. If `first_segment` contains `?`: start = `?`→`0`, end = `?`→`F`.
676 /// 4. Else start = hex value of `first_segment`.
677 /// 5. If next two are `-` + hex digit: consume `-`, consume up to 6 hex
678 /// → end.
679 /// 6. Else end = start.
680 fn consume_a_unicode_range_token(&mut self) -> Token {
681 // §4.3.14 step 1: consume and discard `u`/`U` + `+`
682 self.consume(); // `u`/`U`
683 self.consume(); // `+`
684
685 // §4.3.14 step 2: consume up to 6 hex digits
686 let mut first_segment = String::new();
687 while first_segment.len() < 6 {
688 match self.peek(0) {
689 Some(c) if is_hex_digit(c) => {
690 first_segment.push(c);
691 self.consume();
692 }
693 _ => break,
694 }
695 }
696 // then consume `?` up to a total of 6
697 while first_segment.len() < 6 {
698 match self.peek(0) {
699 Some('?') => {
700 first_segment.push('?');
701 self.consume();
702 }
703 _ => break,
704 }
705 }
706
707 // §4.3.14 step 3: if first_segment contains `?`
708 if first_segment.contains('?') {
709 let start_str: String = first_segment
710 .chars()
711 .map(|c| if c == '?' { '0' } else { c })
712 .collect();
713 let start = u32::from_str_radix(&start_str, 16).unwrap_or(0);
714 let end_str: String = first_segment
715 .chars()
716 .map(|c| if c == '?' { 'F' } else { c })
717 .collect();
718 let end = u32::from_str_radix(&end_str, 16).unwrap_or(0);
719 return Token::UnicodeRange(Some(start), Some(end));
720 }
721
722 // §4.3.14 step 4: first_segment as hex → start
723 let start = u32::from_str_radix(&first_segment, 16).unwrap_or(0);
724
725 // §4.3.14 step 5: if next two are `-` + hex digit
726 if self.peek(0) == Some('-') && self.peek(1).is_some_and(is_hex_digit) {
727 self.consume(); // consume `-`
728 let mut end_segment = String::new();
729 while end_segment.len() < 6 {
730 match self.peek(0) {
731 Some(c) if is_hex_digit(c) => {
732 end_segment.push(c);
733 self.consume();
734 }
735 _ => break,
736 }
737 }
738 let end = u32::from_str_radix(&end_segment, 16).unwrap_or(0);
739 return Token::UnicodeRange(Some(start), Some(end));
740 }
741
742 // §4.3.14 step 6: otherwise start == end
743 Token::UnicodeRange(Some(start), Some(start))
744 }
745
746 // ── §4.3.12 Consume an ident sequence ────────────────────────────
747
748 /// §4.3.12 Consume an ident sequence.
749 ///
750 /// Repeatedly consume code points as long as they form an ident
751 /// sequence: ident code points, valid escapes (consumed via
752 /// §4.3.7), or `\`-`-` (for `--`-prefixed names). Returns the
753 /// decoded name with escapes resolved.
754 ///
755 /// Per §4.3.12, the loop terminates when:
756 /// - EOF is reached, or
757 /// - the next code point is not an ident code point, not a valid
758 /// escape, and not part of a `\`-`-` continuation.
759 fn consume_an_ident_sequence(&mut self) -> String {
760 let mut result = String::new();
761 loop {
762 let Some(c) = self.consume() else {
763 return result;
764 };
765 if is_ident_code_point(c) {
766 result.push(c);
767 } else if c == '\\' && self.is_valid_escape_next() {
768 // §4.3.12: valid escape → consume the escaped code point
769 // and append it.
770 let escaped = self.consume_an_escaped_code_point();
771 result.push(escaped);
772 } else {
773 // Not part of the ident sequence: reconsume and stop.
774 self.reconsume();
775 return result;
776 }
777 }
778 }
779
780 // ── §4.3.7 Consume an escaped code point ────────────────────────
781
782 /// §4.3.7 Consume an escaped code point.
783 ///
784 /// Precondition: the `\` has been consumed. Returns the decoded code
785 /// point.
786 ///
787 /// Per §4.3.7 (L1207-1240):
788 /// - EOF (L1233-1236) → parse error, return U+FFFD REPLACEMENT CHARACTER.
789 /// - newline → parse error, reconsume (invalid escape; the caller
790 /// should have checked §4.3.8 first, but we handle defensively).
791 /// - hex digit(s) (L1219-1231) → consume 1-6 hex digits, optionally
792 /// followed by whitespace, and return the code point with that
793 /// value. U+0000 NULL or out-of-range → replacement character U+FFFD.
794 /// - anything else (L1238-1240) → return the consumed code point.
795 fn consume_an_escaped_code_point(&mut self) -> char {
796 let Some(c) = self.consume() else {
797 // §4.3.7 L1233-1236: EOF → parse error, return U+FFFD.
798 return '\u{FFFD}';
799 };
800 if is_hex_digit(c) {
801 // §4.3.7 step 4: consume up to 5 more hex digits (total ≤ 6).
802 let mut hex = String::new();
803 hex.push(c);
804 while hex.len() < 6 {
805 match self.peek(0) {
806 Some(next) if is_hex_digit(next) => {
807 hex.push(next);
808 self.consume();
809 }
810 _ => break,
811 }
812 }
813 // §4.3.7 step 4: consume one trailing whitespace.
814 if self.peek(0) == Some(' ') || self.peek(0) == Some('\t') || self.peek(0) == Some('\n')
815 {
816 self.consume();
817 }
818 // Parse the hex value.
819 let value = u32::from_str_radix(&hex, 16).unwrap_or(0);
820 // §4.3.7 step 4: 0 or > 0x10FFFF → U+FFFD. Also surrogates
821 // (0xD800–0xDFFF) → U+FFFD.
822 if value == 0 || value > 0x10FFFF || (0xD800..=0xDFFF).contains(&value) {
823 '\u{FFFD}'
824 } else {
825 char::from_u32(value).unwrap_or('\u{FFFD}')
826 }
827 } else if c == '\n' {
828 // §4.3.7 step 3: newline after `\` is a parse error. This
829 // should not happen if the caller checked §4.3.8, but handle
830 // defensively by returning `\` and reconsuming the newline.
831 self.reconsume();
832 '\\'
833 } else {
834 // §4.3.7 step 5: any other code point → return it.
835 c
836 }
837 }
838
839 // ── §4.3.8 / §4.3.9 Predicates ──────────────────────────────────
840
841 /// §4.3.8 Check if two code points are a valid escape.
842 ///
843 /// Precondition: the `\` has been consumed. Returns true if the next
844 /// code point forms a valid escape with the consumed `\` (i.e. the
845 /// next code point is not a newline and not EOF).
846 fn is_valid_escape_next(&self) -> bool {
847 match self.peek(0) {
848 None => false,
849 Some('\n') => false,
850 Some(_) => true,
851 }
852 }
853
854 /// §4.3.9 Check if three code points would start an ident sequence,
855 /// examining code points starting at `offset` from the current
856 /// position.
857 ///
858 /// Per §4.3.9:
859 /// - U+002D HYPHEN-MINUS (`-`):
860 /// - If the next is also `-`, true (start of `--...`).
861 /// - If the next is an ident-start code point (or a valid escape),
862 /// true.
863 /// - Else false.
864 /// - U+005C REVERSE SOLIDUS (`\`): true if valid escape.
865 /// - ident-start code point: true.
866 /// - Else false.
867 fn would_start_ident_sequence_at(&self, offset: usize) -> bool {
868 let first = match self.peek(offset) {
869 Some(c) => c,
870 None => return false,
871 };
872 match first {
873 '-' => {
874 // §4.3.9: if next is `-`, true.
875 match self.peek(offset + 1) {
876 Some('-') => true,
877 Some(next) if is_ident_start_code_point(next) => true,
878 Some('\\') => {
879 // §4.3.9: if `\` and the following forms a valid escape.
880 self.is_valid_escape_at(offset + 1)
881 }
882 _ => false,
883 }
884 }
885 '\\' => self.is_valid_escape_at(offset),
886 _ if is_ident_start_code_point(first) => true,
887 _ => false,
888 }
889 }
890
891 /// §4.3.8 Check if the code point at `offset` starts a valid escape
892 /// (i.e. `offset` points at `\` and `offset+1` is not newline/EOF).
893 fn is_valid_escape_at(&self, offset: usize) -> bool {
894 if self.peek(offset) != Some('\\') {
895 return false;
896 }
897 match self.peek(offset + 1) {
898 None => false,
899 Some('\n') => false,
900 Some(_) => true,
901 }
902 }
903
904 // ── Predicates (stubs returning false; implemented in C-4/C-6) ──
905
906 /// §4.3.10 (L1307-1352) Check if three code points would start a number.
907 ///
908 /// `first` is the code point already consumed (the caller passes it
909 /// so the predicate can examine the next two without reconsume). The
910 /// three code points examined are `first`, `peek(0)`, `peek(1)`.
911 ///
912 /// Per §4.3.10:
913 /// - `+`/`-`: second is digit → true; second is `.` and third is digit
914 /// → true; else false.
915 /// - `.`: second is digit → true; else false.
916 /// - digit: true.
917 /// - anything else: false.
918 fn starts_with_number(&self, first: char) -> bool {
919 match first {
920 '+' | '-' => match self.peek(0) {
921 Some(d) if is_digit(d) => true,
922 Some('.') => matches!(self.peek(1), Some(d) if is_digit(d)),
923 _ => false,
924 },
925 '.' => matches!(self.peek(0), Some(d) if is_digit(d)),
926 d if is_digit(d) => true,
927 _ => false,
928 }
929 }
930}
931
932// ── Free functions (§4.2 code point classes) ──────────────────────────
933
934/// §4.2 Whether `c` is an ident-start code point.
935///
936/// An ident-start code point is an ASCII letter, a non-ASCII code point
937/// (U+0080 or higher), or U+005F LOW LINE (`_`).
938fn is_ident_start_code_point(c: char) -> bool {
939 matches!(c, 'A'..='Z' | 'a'..='z' | '_' | '\u{0080}'..=char::MAX)
940}
941
942/// §4.2 Whether `c` is an ident code point (allowed after the first).
943///
944/// An ident code point is an ident-start code point, or an ASCII digit,
945/// or U+002D HYPHEN-MINUS (`-`).
946fn is_ident_code_point(c: char) -> bool {
947 matches!(c, 'A'..='Z' | 'a'..='z' | '0'..='9' | '_' | '-' | '\u{0080}'..=char::MAX)
948}
949
950/// §4.2 Whether `c` is a digit (ASCII 0-9).
951fn is_digit(c: char) -> bool {
952 c.is_ascii_digit()
953}
954
955/// §4.2 Whether `c` is a hex digit (ASCII 0-9, A-F, a-f).
956fn is_hex_digit(c: char) -> bool {
957 c.is_ascii_hexdigit()
958}
959
960/// §4.2 Whether `c` is a whitespace code point.
961///
962/// Per §4.2: U+0009 TAB, U+000A LF, U+000C FF, U+000D CR, U+0020 SPACE.
963/// (After §5.3 preprocessing, U+000C FF and lone U+000D CR have been
964/// normalized to U+000A LF, but the helper follows the full §4.2
965/// definition for correctness when called on arbitrary input.)
966fn is_whitespace(c: char) -> bool {
967 matches!(c, '\t' | '\n' | '\u{000C}' | '\r' | ' ')
968}
969
970/// §4.2 Whether `c` is a non-printable code point.
971///
972/// Per §4.2: U+0000-U+0008, U+000B, U+000E-U+001F, U+007F-U+009F.
973fn is_non_printable(c: char) -> bool {
974 matches!(
975 c,
976 '\u{0000}'..='\u{0008}' | '\u{000B}' | '\u{000E}'..='\u{001F}' | '\u{007F}'..='\u{009F}'
977 )
978}
979
980/// §5.3 input preprocessing.
981///
982/// Per the "input stream" definition in §5.3:
983/// - U+000D CR followed by U+000A LF → single U+000A LF
984/// - lone U+000D CR → U+000A LF
985/// - U+000C FF → U+000A LF
986///
987/// The tokenizer operates on this normalized stream.
988fn preprocess_input(s: &str) -> Vec<char> {
989 let mut out = Vec::with_capacity(s.len());
990 let mut chars = s.chars().peekable();
991 while let Some(c) = chars.next() {
992 match c {
993 '\r' => {
994 if chars.peek() == Some(&'\n') {
995 chars.next();
996 }
997 out.push('\n');
998 }
999 '\u{000C}' => out.push('\n'),
1000 _ => out.push(c),
1001 }
1002 }
1003 out
1004}
1005
1006impl Tokenizer for CssTokenizer {
1007 fn next_token(&mut self) -> Option<Token> {
1008 if self.eof_emitted {
1009 return None;
1010 }
1011 Some(self.consume_a_token())
1012 }
1013
1014 fn state(&self) -> State {
1015 self.state
1016 }
1017
1018 fn set_state(&mut self, state: State) {
1019 self.state = state;
1020 if state == State::Data {
1021 // Resetting to Data after EOF re-enables the iterator; useful
1022 // for `reset()` callers that want to re-tokenize.
1023 self.eof_emitted = false;
1024 }
1025 }
1026
1027 fn reset(&mut self) {
1028 self.pos = 0;
1029 self.state = State::Data;
1030 self.eof_emitted = false;
1031 }
1032
1033 fn set_unicode_ranges_allowed(&mut self, allowed: bool) {
1034 self.unicode_ranges_allowed = allowed;
1035 }
1036}
1037
1038// ── Test helpers exposed for unit tests in this file ──────────────────
1039
1040#[cfg(test)]
1041impl CssTokenizer {
1042 /// Tokenize the entire input and return all tokens except the
1043 /// trailing `<EOF-token>`. Test-only convenience.
1044 fn collect(input: &str) -> Vec<Token> {
1045 let mut tz = Self::new(input);
1046 let mut out = Vec::new();
1047 while let Some(t) = tz.next_token() {
1048 if matches!(t, Token::Eof) {
1049 break;
1050 }
1051 out.push(t);
1052 }
1053 out
1054 }
1055}
1056
1057#[cfg(test)]
1058mod tests {
1059 use super::*;
1060
1061 #[test]
1062 fn empty_input_emits_eof_only() {
1063 let mut tz = CssTokenizer::new("");
1064 assert!(matches!(tz.next_token(), Some(Token::Eof)));
1065 assert!(tz.next_token().is_none());
1066 }
1067
1068 #[test]
1069 fn whitespace_collapses_runs() {
1070 let tokens = CssTokenizer::collect(" \t\n ");
1071 assert_eq!(tokens.len(), 1, "whitespace run → single token");
1072 assert!(matches!(tokens[0], Token::Whitespace));
1073 }
1074
1075 #[test]
1076 fn simple_punctuation_tokens() {
1077 let tokens = CssTokenizer::collect(":;,.{}[]()");
1078 assert_eq!(tokens.len(), 10);
1079 assert!(matches!(tokens[0], Token::Colon));
1080 assert!(matches!(tokens[1], Token::Semicolon));
1081 assert!(matches!(tokens[2], Token::Comma));
1082 assert!(matches!(tokens[3], Token::Delim('.')));
1083 assert!(matches!(tokens[4], Token::OpenBrace));
1084 assert!(matches!(tokens[5], Token::CloseBrace));
1085 assert!(matches!(tokens[6], Token::OpenBracket));
1086 assert!(matches!(tokens[7], Token::CloseBracket));
1087 assert!(matches!(tokens[8], Token::OpenParen));
1088 assert!(matches!(tokens[9], Token::CloseParen));
1089 }
1090
1091 #[test]
1092 fn close_paren_is_emitted() {
1093 let tokens = CssTokenizer::collect(")");
1094 assert_eq!(tokens.len(), 1);
1095 assert!(matches!(tokens[0], Token::CloseParen));
1096 }
1097
1098 #[test]
1099 fn unknown_delim_chars() {
1100 let tokens = CssTokenizer::collect("^~`>");
1101 assert_eq!(tokens.len(), 4);
1102 assert!(matches!(tokens[0], Token::Delim('^')));
1103 assert!(matches!(tokens[1], Token::Delim('~')));
1104 assert!(matches!(tokens[2], Token::Delim('`')));
1105 assert!(matches!(tokens[3], Token::Delim('>')));
1106 }
1107
1108 #[test]
1109 fn preprocessing_normalizes_cr_lf_ff() {
1110 // CR LF → single LF
1111 let tz = CssTokenizer::new("a\r\nb");
1112 assert_eq!(tz.input, vec!['a', '\n', 'b']);
1113 // Lone CR → LF
1114 let tz = CssTokenizer::new("a\rb");
1115 assert_eq!(tz.input, vec!['a', '\n', 'b']);
1116 // FF → LF
1117 let tz = CssTokenizer::new("a\u{000C}b");
1118 assert_eq!(tz.input, vec!['a', '\n', 'b']);
1119 }
1120
1121 #[test]
1122 fn eof_emitted_once() {
1123 // C-1: use a non-ident-start, non-digit char to avoid the
1124 // ident-like / numeric branches (which are stubbed with todo!()
1125 // until C-2 / C-4).
1126 let mut tz = CssTokenizer::new(",");
1127 let t1 = tz.next_token();
1128 let t2 = tz.next_token();
1129 let t3 = tz.next_token();
1130 assert!(matches!(t1, Some(Token::Comma)));
1131 assert!(matches!(t2, Some(Token::Eof)));
1132 assert!(t3.is_none());
1133 }
1134
1135 #[test]
1136 fn block_comment_is_skipped() {
1137 // §4.3.2: `/* ... */` is consumed and no token is emitted.
1138 let tokens = CssTokenizer::collect("/* hello */:/*x*/;");
1139 assert_eq!(tokens.len(), 2);
1140 assert!(matches!(tokens[0], Token::Colon));
1141 assert!(matches!(tokens[1], Token::Semicolon));
1142 }
1143
1144 #[test]
1145 fn unterminated_comment_consumes_to_eof() {
1146 // §4.3.2: EOF before `*/` is a parse error; we consume to EOF.
1147 let tokens = CssTokenizer::collect("/* unfinished");
1148 assert_eq!(tokens.len(), 0, "unterminated comment → no tokens");
1149 }
1150
1151 #[test]
1152 fn comment_between_tokens() {
1153 // C-2 not yet: `a`/`b` would be ident tokens (todo!()). Verify
1154 // comment is consumed without affecting surrounding simple tokens
1155 // by using only punctuation.
1156 let tokens = CssTokenizer::collect(":/*c*/;");
1157 assert_eq!(tokens.len(), 2);
1158 assert!(matches!(tokens[0], Token::Colon));
1159 assert!(matches!(tokens[1], Token::Semicolon));
1160 }
1161
1162 #[test]
1163 fn cdo_token_emitted() {
1164 // §4.3.1: `<!--` → <CDO-token>
1165 let tokens = CssTokenizer::collect("<!--");
1166 assert_eq!(tokens.len(), 1);
1167 assert!(matches!(tokens[0], Token::Cdo));
1168 }
1169
1170 #[test]
1171 fn cdc_token_emitted() {
1172 // §4.3.1: `-->` → <CDC-token>
1173 let tokens = CssTokenizer::collect("-->");
1174 assert_eq!(tokens.len(), 1);
1175 assert!(matches!(tokens[0], Token::Cdc));
1176 }
1177
1178 #[test]
1179 fn cdo_cdc_wrapping_stylesheet() {
1180 // Legacy CSS 1/2.1 stylesheet wrapping.
1181 // `<!-- : ; -->`
1182 // tokens: Cdo, WS, Colon, WS, Semicolon, WS, Cdc = 7
1183 let tokens = CssTokenizer::collect("<!-- : ; -->");
1184 assert_eq!(tokens.len(), 7);
1185 assert!(matches!(tokens[0], Token::Cdo));
1186 assert!(matches!(tokens[1], Token::Whitespace));
1187 assert!(matches!(tokens[2], Token::Colon));
1188 assert!(matches!(tokens[3], Token::Whitespace));
1189 assert!(matches!(tokens[4], Token::Semicolon));
1190 assert!(matches!(tokens[5], Token::Whitespace));
1191 assert!(matches!(tokens[6], Token::Cdc));
1192 }
1193
1194 #[test]
1195 fn less_than_alone_is_delim() {
1196 // §4.3.1: `<` not followed by `!-` → <delim-token>
1197 let tokens = CssTokenizer::collect("< ");
1198 assert_eq!(tokens.len(), 2);
1199 assert!(matches!(tokens[0], Token::Delim('<')));
1200 assert!(matches!(tokens[1], Token::Whitespace));
1201 }
1202
1203 #[test]
1204 fn hyphen_alone_is_delim() {
1205 // §4.3.1: `-` not followed by number/ident/`->` → <delim-token>
1206 let tokens = CssTokenizer::collect("- ");
1207 assert_eq!(tokens.len(), 2);
1208 assert!(matches!(tokens[0], Token::Delim('-')));
1209 assert!(matches!(tokens[1], Token::Whitespace));
1210 }
1211
1212 // ── C-2 tests: ident / function / at-keyword / hash ──────────────
1213
1214 #[test]
1215 fn simple_ident() {
1216 let tokens = CssTokenizer::collect("color");
1217 assert_eq!(tokens.len(), 1);
1218 assert!(matches!(&tokens[0], Token::Ident(s) if s == "color"));
1219 }
1220
1221 #[test]
1222 fn ident_with_hyphen_prefix() {
1223 let tokens = CssTokenizer::collect("-webkit-flex");
1224 assert_eq!(tokens.len(), 1);
1225 assert!(matches!(&tokens[0], Token::Ident(s) if s == "-webkit-flex"));
1226 }
1227
1228 #[test]
1229 fn ident_with_double_hyphen() {
1230 // §4.3.9: `--` starts an ident sequence (custom properties).
1231 let tokens = CssTokenizer::collect("--my-var");
1232 assert_eq!(tokens.len(), 1);
1233 assert!(matches!(&tokens[0], Token::Ident(s) if s == "--my-var"));
1234 }
1235
1236 #[test]
1237 fn ident_with_underscore() {
1238 let tokens = CssTokenizer::collect("_private");
1239 assert_eq!(tokens.len(), 1);
1240 assert!(matches!(&tokens[0], Token::Ident(s) if s == "_private"));
1241 }
1242
1243 #[test]
1244 fn ident_with_escape() {
1245 // §4.3.7: `\26` → '&' (U+0026)
1246 let tokens = CssTokenizer::collect("color\\26 B");
1247 assert_eq!(tokens.len(), 1);
1248 assert!(matches!(&tokens[0], Token::Ident(s) if s == "color&B"));
1249 }
1250
1251 #[test]
1252 fn function_token() {
1253 let tokens = CssTokenizer::collect("translate(");
1254 assert_eq!(tokens.len(), 1);
1255 assert!(matches!(&tokens[0], Token::Function(s) if s == "translate"));
1256 }
1257
1258 #[test]
1259 fn at_keyword_token() {
1260 let tokens = CssTokenizer::collect("@media");
1261 assert_eq!(tokens.len(), 1);
1262 assert!(matches!(&tokens[0], Token::AtKeyword(s) if s == "media"));
1263 }
1264
1265 #[test]
1266 fn at_keyword_with_hyphen() {
1267 let tokens = CssTokenizer::collect("@-webkit-keyframes");
1268 assert_eq!(tokens.len(), 1);
1269 assert!(matches!(&tokens[0], Token::AtKeyword(s) if s == "-webkit-keyframes"));
1270 }
1271
1272 #[test]
1273 fn at_sign_alone_is_delim() {
1274 // §4.3.1: `@` not followed by ident-start → <delim-token>
1275 let tokens = CssTokenizer::collect("@ ");
1276 assert_eq!(tokens.len(), 2);
1277 assert!(matches!(tokens[0], Token::Delim('@')));
1278 assert!(matches!(tokens[1], Token::Whitespace));
1279 }
1280
1281 #[test]
1282 fn hash_id_type() {
1283 // §4.3.1 / §4.3.4: `#main` → <hash-token> with type "id"
1284 let tokens = CssTokenizer::collect("#main");
1285 assert_eq!(tokens.len(), 1);
1286 match &tokens[0] {
1287 Token::Hash(s, t) => {
1288 assert_eq!(s, "main");
1289 assert_eq!(*t, crate::types::HashType::Id);
1290 }
1291 _ => panic!("expected Hash, got {:?}", tokens[0]),
1292 }
1293 }
1294
1295 #[test]
1296 fn hash_unrestricted_type() {
1297 // §4.3.1 L801-826 + §4.3.4: `#123` → next `1` is an ident code
1298 // point (digits are ident code points per §4.2), so a hash-token
1299 // is created. `123` does not would-start-an-ident-sequence (digit
1300 // is not ident-start), so type is "unrestricted". The value is
1301 // "123" (consume_an_ident_sequence consumes all ident code points).
1302 let tokens = CssTokenizer::collect("#123");
1303 assert_eq!(tokens.len(), 1);
1304 match &tokens[0] {
1305 Token::Hash(s, t) => {
1306 assert_eq!(s, "123");
1307 assert_eq!(*t, crate::types::HashType::Unrestricted);
1308 }
1309 _ => panic!("expected Hash, got {:?}", tokens[0]),
1310 }
1311 }
1312
1313 #[test]
1314 fn hash_hex_color_is_unrestricted() {
1315 // §4.3.4: `#ff00aa` → starts with `f` which IS ident-start, so
1316 // this is actually type "id" (the hex digits form a valid ident).
1317 let tokens = CssTokenizer::collect("#ff00aa");
1318 assert_eq!(tokens.len(), 1);
1319 match &tokens[0] {
1320 Token::Hash(s, t) => {
1321 assert_eq!(s, "ff00aa");
1322 assert_eq!(*t, crate::types::HashType::Id);
1323 }
1324 _ => panic!("expected Hash, got {:?}", tokens[0]),
1325 }
1326 }
1327
1328 #[test]
1329 fn backslash_escape_in_ident() {
1330 // §4.3.7: `\` followed by non-hex → literal char. Use `z` (not
1331 // `b`, which is a hex digit and would decode to U+000B).
1332 let tokens = CssTokenizer::collect("a\\z");
1333 assert_eq!(tokens.len(), 1);
1334 assert!(matches!(&tokens[0], Token::Ident(s) if s == "az"));
1335 }
1336
1337 #[test]
1338 fn backslash_alone_is_delim() {
1339 // §4.3.1: `\` not followed by valid escape (EOF or newline) →
1340 // <delim-token>
1341 let tokens = CssTokenizer::collect("\\");
1342 assert_eq!(tokens.len(), 1);
1343 assert!(matches!(tokens[0], Token::Delim('\\')));
1344 }
1345
1346 #[test]
1347 fn backslash_newline_is_delim() {
1348 // §4.3.8: `\` followed by newline is not a valid escape.
1349 let tokens = CssTokenizer::collect("\\\n");
1350 assert_eq!(tokens.len(), 2);
1351 assert!(matches!(tokens[0], Token::Delim('\\')));
1352 assert!(matches!(tokens[1], Token::Whitespace));
1353 }
1354
1355 #[test]
1356 fn declaration_with_ident_and_value() {
1357 let tokens = CssTokenizer::collect("color: red");
1358 assert_eq!(tokens.len(), 4);
1359 assert!(matches!(&tokens[0], Token::Ident(s) if s == "color"));
1360 assert!(matches!(tokens[1], Token::Colon));
1361 assert!(matches!(tokens[2], Token::Whitespace));
1362 assert!(matches!(&tokens[3], Token::Ident(s) if s == "red"));
1363 }
1364
1365 #[test]
1366 fn unicode_escape_in_ident() {
1367 // §4.3.7 L1223-1225: `\000026` → '&' (U+0026); after 6 hex digits,
1368 // if next is whitespace, consume it. So `\000026 B` → the space is
1369 // consumed as trailing whitespace, `B` joins the ident sequence →
1370 // single token Ident("&B").
1371 let tokens = CssTokenizer::collect("\\000026 B");
1372 assert_eq!(tokens.len(), 1);
1373 assert!(matches!(&tokens[0], Token::Ident(s) if s == "&B"));
1374 }
1375
1376 #[test]
1377 fn null_escape_becomes_replacement_char() {
1378 // §4.3.7: `\0` → U+FFFD (NULL is not allowed)
1379 let tokens = CssTokenizer::collect("\\0");
1380 assert_eq!(tokens.len(), 1);
1381 assert!(matches!(&tokens[0], Token::Ident(s) if s == "\u{FFFD}"));
1382 }
1383
1384 #[test]
1385 fn surrogate_escape_becomes_replacement_char() {
1386 // §4.3.7: `\D800` → U+FFFD (surrogates not allowed)
1387 let tokens = CssTokenizer::collect("\\D800");
1388 assert_eq!(tokens.len(), 1);
1389 assert!(matches!(&tokens[0], Token::Ident(s) if s == "\u{FFFD}"));
1390 }
1391
1392 // ── C-2 Bug 1 回归测试: # Delim 回退 (§4.3.1 L824-826) ────────────
1393
1394 #[test]
1395 fn hash_alone_is_delim() {
1396 // §4.3.1 L824-826: `#` followed by EOF → next is not ident code
1397 // point, not valid escape → return delim-token with value `#`.
1398 let tokens = CssTokenizer::collect("#");
1399 assert_eq!(tokens.len(), 1);
1400 assert!(matches!(tokens[0], Token::Delim('#')));
1401 }
1402
1403 #[test]
1404 fn hash_followed_by_space_is_delim() {
1405 // §4.3.1 L824-826: `#` followed by space (not ident code point,
1406 // not valid escape) → delim-token with value `#`.
1407 let tokens = CssTokenizer::collect("# ");
1408 assert_eq!(tokens.len(), 2);
1409 assert!(matches!(tokens[0], Token::Delim('#')));
1410 assert!(matches!(tokens[1], Token::Whitespace));
1411 }
1412
1413 #[test]
1414 fn hash_followed_by_at_is_delim() {
1415 // §4.3.1 L824-826: `#` followed by `@` (not ident code point,
1416 // not valid escape) → delim-token with value `#`.
1417 let tokens = CssTokenizer::collect("#@");
1418 assert_eq!(tokens.len(), 2);
1419 assert!(matches!(tokens[0], Token::Delim('#')));
1420 assert!(matches!(tokens[1], Token::Delim('@')));
1421 }
1422
1423 // ── C-3 tests: §4.3.5 string + bad-string ────────────────────────
1424
1425 #[test]
1426 fn string_double_quoted() {
1427 // §4.3.5: "hello" → String("hello")
1428 let tokens = CssTokenizer::collect("\"hello\"");
1429 assert_eq!(tokens.len(), 1);
1430 assert!(matches!(&tokens[0], Token::String(s) if s == "hello"));
1431 }
1432
1433 #[test]
1434 fn string_single_quoted() {
1435 // §4.3.5: 'hello' → String("hello")
1436 let tokens = CssTokenizer::collect("'hello'");
1437 assert_eq!(tokens.len(), 1);
1438 assert!(matches!(&tokens[0], Token::String(s) if s == "hello"));
1439 }
1440
1441 #[test]
1442 fn string_with_escape() {
1443 // §4.3.5 + §4.3.7: "a\z" → String("az") (z is non-hex → literal)
1444 let tokens = CssTokenizer::collect("\"a\\z\"");
1445 assert_eq!(tokens.len(), 1);
1446 assert!(matches!(&tokens[0], Token::String(s) if s == "az"));
1447 }
1448
1449 #[test]
1450 fn string_unterminated_eof() {
1451 // §4.3.5 L1101-1104: EOF before closing quote → parse error,
1452 // return string-token with value accumulated so far.
1453 let tokens = CssTokenizer::collect("\"hello");
1454 assert_eq!(tokens.len(), 1);
1455 assert!(matches!(&tokens[0], Token::String(s) if s == "hello"));
1456 }
1457
1458 #[test]
1459 fn string_unescaped_newline_is_bad_string() {
1460 // §4.3.5 L1106-1110: unescaped newline → parse error, reconsume,
1461 // return bad-string. The reconsumed newline is then tokenized as
1462 // whitespace by the next consume_a_token call.
1463 let tokens = CssTokenizer::collect("\"\n");
1464 assert_eq!(tokens.len(), 2);
1465 assert!(matches!(tokens[0], Token::BadString));
1466 assert!(matches!(tokens[1], Token::Whitespace));
1467 }
1468
1469 #[test]
1470 fn string_line_continuation() {
1471 // §4.3.5 L1117-1119: backslash followed by newline → line
1472 // continuation (newline consumed, no char appended).
1473 let tokens = CssTokenizer::collect("\"a\\\nb\"");
1474 assert_eq!(tokens.len(), 1);
1475 assert!(matches!(&tokens[0], Token::String(s) if s == "ab"));
1476 }
1477
1478 #[test]
1479 fn string_hex_escape() {
1480 // §4.3.5 + §4.3.7: "\26" → String("&") (U+0026)
1481 let tokens = CssTokenizer::collect("\"\\26\"");
1482 assert_eq!(tokens.len(), 1);
1483 assert!(matches!(&tokens[0], Token::String(s) if s == "&"));
1484 }
1485
1486 // ── C-4 tests: §4.3.3/§4.3.10/§4.3.13 number/percentage/dimension ─
1487
1488 #[test]
1489 fn number_integer() {
1490 let tokens = CssTokenizer::collect("42");
1491 assert_eq!(tokens.len(), 1);
1492 assert_eq!(
1493 tokens[0],
1494 Token::Number(Numeric {
1495 value: 42.0,
1496 is_integer: true
1497 })
1498 );
1499 }
1500
1501 #[test]
1502 fn number_decimal() {
1503 let tokens = CssTokenizer::collect("3.5");
1504 assert_eq!(tokens.len(), 1);
1505 assert_eq!(
1506 tokens[0],
1507 Token::Number(Numeric {
1508 value: 3.5,
1509 is_integer: false
1510 })
1511 );
1512 }
1513
1514 #[test]
1515 fn number_signed() {
1516 let tokens = CssTokenizer::collect("-5");
1517 assert_eq!(tokens.len(), 1);
1518 assert_eq!(
1519 tokens[0],
1520 Token::Number(Numeric {
1521 value: -5.0,
1522 is_integer: true
1523 })
1524 );
1525 }
1526
1527 #[test]
1528 fn number_exponent() {
1529 // §4.3.13: 1e3 → 1.0 * 10^3 = 1000.0, type "number"
1530 let tokens = CssTokenizer::collect("1e3");
1531 assert_eq!(tokens.len(), 1);
1532 assert_eq!(
1533 tokens[0],
1534 Token::Number(Numeric {
1535 value: 1000.0,
1536 is_integer: false
1537 })
1538 );
1539 }
1540
1541 #[test]
1542 fn number_decimal_exponent() {
1543 // §4.3.13: 1.5e2 → 1.5 * 10^2 = 150.0, type "number"
1544 let tokens = CssTokenizer::collect("1.5e2");
1545 assert_eq!(tokens.len(), 1);
1546 assert_eq!(
1547 tokens[0],
1548 Token::Number(Numeric {
1549 value: 150.0,
1550 is_integer: false
1551 })
1552 );
1553 }
1554
1555 #[test]
1556 fn percentage_token() {
1557 let tokens = CssTokenizer::collect("50%");
1558 assert_eq!(tokens.len(), 1);
1559 assert_eq!(
1560 tokens[0],
1561 Token::Percentage(Numeric {
1562 value: 50.0,
1563 is_integer: true
1564 })
1565 );
1566 }
1567
1568 #[test]
1569 fn dimension_px() {
1570 let tokens = CssTokenizer::collect("10px");
1571 assert_eq!(tokens.len(), 1);
1572 assert_eq!(
1573 tokens[0],
1574 Token::Dimension(
1575 Numeric {
1576 value: 10.0,
1577 is_integer: true
1578 },
1579 "px".to_string(),
1580 )
1581 );
1582 }
1583
1584 #[test]
1585 fn dimension_em() {
1586 let tokens = CssTokenizer::collect("1.5em");
1587 assert_eq!(tokens.len(), 1);
1588 assert_eq!(
1589 tokens[0],
1590 Token::Dimension(
1591 Numeric {
1592 value: 1.5,
1593 is_integer: false
1594 },
1595 "em".to_string(),
1596 )
1597 );
1598 }
1599
1600 #[test]
1601 fn dimension_signed() {
1602 let tokens = CssTokenizer::collect("-30deg");
1603 assert_eq!(tokens.len(), 1);
1604 assert_eq!(
1605 tokens[0],
1606 Token::Dimension(
1607 Numeric {
1608 value: -30.0,
1609 is_integer: true
1610 },
1611 "deg".to_string(),
1612 )
1613 );
1614 }
1615
1616 #[test]
1617 fn plus_sign_number() {
1618 let tokens = CssTokenizer::collect("+5");
1619 assert_eq!(tokens.len(), 1);
1620 assert_eq!(
1621 tokens[0],
1622 Token::Number(Numeric {
1623 value: 5.0,
1624 is_integer: true
1625 })
1626 );
1627 }
1628
1629 #[test]
1630 fn dot_starts_number() {
1631 let tokens = CssTokenizer::collect(".5");
1632 assert_eq!(tokens.len(), 1);
1633 assert_eq!(
1634 tokens[0],
1635 Token::Number(Numeric {
1636 value: 0.5,
1637 is_integer: false
1638 })
1639 );
1640 }
1641
1642 // ── C-5 tests: §4.3.4 url( special case + §4.3.6 Url + §4.3.15 BadUrl ─
1643
1644 #[test]
1645 fn url_unquoted_simple() {
1646 // §4.3.6: url(foo) → Url("foo")
1647 let tokens = CssTokenizer::collect("url(foo)");
1648 assert_eq!(tokens.len(), 1);
1649 assert_eq!(tokens[0], Token::Url("foo".to_string()));
1650 }
1651
1652 #[test]
1653 fn url_unquoted_with_spaces() {
1654 // §4.3.6: leading/trailing whitespace consumed; url( foo ) → Url("foo")
1655 let tokens = CssTokenizer::collect("url( foo )");
1656 assert_eq!(tokens.len(), 1);
1657 assert_eq!(tokens[0], Token::Url("foo".to_string()));
1658 }
1659
1660 #[test]
1661 fn url_empty() {
1662 // §4.3.6: url() → Url("")
1663 let tokens = CssTokenizer::collect("url()");
1664 assert_eq!(tokens.len(), 1);
1665 assert_eq!(tokens[0], Token::Url(String::new()));
1666 }
1667
1668 #[test]
1669 fn url_eof_unterminated() {
1670 // §4.3.6 L1161-1164: EOF → parse error, return url-token with value so far
1671 let tokens = CssTokenizer::collect("url(foo");
1672 assert_eq!(tokens.len(), 1);
1673 assert_eq!(tokens[0], Token::Url("foo".to_string()));
1674 }
1675
1676 #[test]
1677 fn url_quoted_is_function() {
1678 // §4.3.4 L1058-1063: url("foo") → Function("url"); the body
1679 // `"foo")` remains in the stream → String("foo") + CloseParen.
1680 let tokens = CssTokenizer::collect("url(\"foo\")");
1681 assert_eq!(tokens.len(), 3);
1682 assert_eq!(tokens[0], Token::Function("url".to_string()));
1683 assert_eq!(tokens[1], Token::String("foo".to_string()));
1684 assert_eq!(tokens[2], Token::CloseParen);
1685 }
1686
1687 #[test]
1688 fn url_single_quoted_is_function() {
1689 // §4.3.4 L1058-1063: url('foo') → Function("url") + String + CloseParen
1690 let tokens = CssTokenizer::collect("url('foo')");
1691 assert_eq!(tokens.len(), 3);
1692 assert_eq!(tokens[0], Token::Function("url".to_string()));
1693 assert_eq!(tokens[1], Token::String("foo".to_string()));
1694 assert_eq!(tokens[2], Token::CloseParen);
1695 }
1696
1697 #[test]
1698 fn url_ws_then_quote_is_function() {
1699 // §4.3.4 L1058-1063: url( "foo") → whitespace+quote → Function.
1700 // L1056-1057 only collapses when *two* consecutive whitespace
1701 // follow `(`; here only one space precedes `"`, so it is NOT
1702 // consumed and becomes a Whitespace token.
1703 let tokens = CssTokenizer::collect("url( \"foo\")");
1704 assert_eq!(tokens.len(), 4);
1705 assert_eq!(tokens[0], Token::Function("url".to_string()));
1706 assert_eq!(tokens[1], Token::Whitespace);
1707 assert_eq!(tokens[2], Token::String("foo".to_string()));
1708 assert_eq!(tokens[3], Token::CloseParen);
1709 }
1710
1711 #[test]
1712 fn url_with_escape() {
1713 // §4.3.6 + §4.3.7: url(foo\29 bar) → \29 consumes trailing space,
1714 // decodes to `)`, then `bar` appended → Url("foo)bar")
1715 let tokens = CssTokenizer::collect("url(foo\\29 bar)");
1716 assert_eq!(tokens.len(), 1);
1717 assert_eq!(tokens[0], Token::Url("foo)bar".to_string()));
1718 }
1719
1720 #[test]
1721 fn url_bad_paren_in_unquoted() {
1722 // §4.3.6 L1177-1185: `(` in unquoted url → BadUrl
1723 let tokens = CssTokenizer::collect("url(foo(bar)");
1724 assert_eq!(tokens.len(), 1);
1725 assert_eq!(tokens[0], Token::BadUrl);
1726 }
1727
1728 #[test]
1729 fn url_bad_quote_in_unquoted() {
1730 // §4.3.6 L1177-1185: `"` in unquoted url → BadUrl
1731 let tokens = CssTokenizer::collect("url(foo\"bar)");
1732 assert_eq!(tokens.len(), 1);
1733 assert_eq!(tokens[0], Token::BadUrl);
1734 }
1735
1736 // ── C-6 tests: §4.3.1 unicode_ranges_allowed + §4.3.11 + §4.3.14 ──
1737
1738 #[test]
1739 fn unicode_range_disabled_by_default() {
1740 // §4.3.1 L782-783: default unicode_ranges_allowed=false.
1741 // `U+1234` → `U` ident-start → Ident("U") (`+` stops ident seq);
1742 // then `+1234` → starts_with_number (`+`+digit) → Number(1234).
1743 let tokens = CssTokenizer::collect("U+1234");
1744 assert_eq!(tokens.len(), 2);
1745 assert_eq!(tokens[0], Token::Ident("U".to_string()));
1746 assert_eq!(
1747 tokens[1],
1748 Token::Number(Numeric {
1749 value: 1234.0,
1750 is_integer: true
1751 })
1752 );
1753 }
1754
1755 #[test]
1756 fn unicode_range_simple() {
1757 // §4.3.14: U+1234 → UnicodeRange(0x1234, 0x1234)
1758 let mut tz = CssTokenizer::new("U+1234");
1759 tz.set_unicode_ranges_allowed(true);
1760 let t = tz.next_token().unwrap();
1761 assert_eq!(t, Token::UnicodeRange(Some(0x1234), Some(0x1234)));
1762 }
1763
1764 #[test]
1765 fn unicode_range_question_marks() {
1766 // §4.3.14 step 3: U+12?? → start=0x1200 (?→0), end=0x12FF (?→F)
1767 let mut tz = CssTokenizer::new("U+12??");
1768 tz.set_unicode_ranges_allowed(true);
1769 let t = tz.next_token().unwrap();
1770 assert_eq!(t, Token::UnicodeRange(Some(0x1200), Some(0x12FF)));
1771 }
1772
1773 #[test]
1774 fn unicode_range_range() {
1775 // §4.3.14 step 5: U+12-34FF → start=0x12, end=0x34FF
1776 let mut tz = CssTokenizer::new("U+12-34FF");
1777 tz.set_unicode_ranges_allowed(true);
1778 let t = tz.next_token().unwrap();
1779 assert_eq!(t, Token::UnicodeRange(Some(0x12), Some(0x34FF)));
1780 }
1781
1782 #[test]
1783 fn unicode_range_max_hex() {
1784 // §4.3.14: U+10FFFF → UnicodeRange(0x10FFFF, 0x10FFFF)
1785 let mut tz = CssTokenizer::new("U+10FFFF");
1786 tz.set_unicode_ranges_allowed(true);
1787 let t = tz.next_token().unwrap();
1788 assert_eq!(t, Token::UnicodeRange(Some(0x10FFFF), Some(0x10FFFF)));
1789 }
1790
1791 #[test]
1792 fn unicode_range_lowercase_u() {
1793 // §4.3.11: lowercase u also triggers; u+abc → UnicodeRange(0xABC, 0xABC)
1794 let mut tz = CssTokenizer::new("u+abc");
1795 tz.set_unicode_ranges_allowed(true);
1796 let t = tz.next_token().unwrap();
1797 assert_eq!(t, Token::UnicodeRange(Some(0xABC), Some(0xABC)));
1798 }
1799}