1use std::borrow::Cow;
13
14#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)]
16pub struct Span {
17 pub start: usize,
18 pub end: usize,
19}
20
21impl Span {
22 pub fn new(start: usize, end: usize) -> Span {
23 Span { start, end }
24 }
25
26 pub fn text<'s>(&self, src: &'s str) -> &'s str {
28 &src[self.start..self.end]
29 }
30}
31
32#[derive(Clone, Copy, Debug, PartialEq)]
34pub struct Number<'a> {
35 pub value: f64,
36 pub integer: bool,
38 pub repr: &'a str,
40}
41
42impl Number<'_> {
43 pub fn sign(&self) -> Option<char> {
45 match self.repr.as_bytes().first() {
46 Some(b'+') => Some('+'),
47 Some(b'-') => Some('-'),
48 _ => None,
49 }
50 }
51
52 pub fn int(&self) -> i32 {
55 self.value as i32
56 }
57}
58
59#[derive(Clone, Debug, PartialEq)]
62pub enum Token<'a> {
63 Ident(Cow<'a, str>),
64 Function(Cow<'a, str>),
66 AtKeyword(Cow<'a, str>),
68 Hash {
71 value: Cow<'a, str>,
72 id: bool,
73 },
74 String(Cow<'a, str>),
75 BadString,
76 Url(Cow<'a, str>),
78 BadUrl,
79 Delim(char),
80 Number(Number<'a>),
81 Percentage(Number<'a>),
82 Dimension {
83 number: Number<'a>,
84 unit: Cow<'a, str>,
85 },
86 UnicodeRange {
89 start: u32,
90 end: u32,
91 },
92 Whitespace,
93 Cdo,
94 Cdc,
95 Colon,
96 Semicolon,
97 Comma,
98 LeftBracket,
99 RightBracket,
100 LeftParen,
101 RightParen,
102 LeftBrace,
103 RightBrace,
104}
105
106#[derive(Clone, Debug)]
117pub struct Tokenizer<'a> {
118 src: &'a str,
119 pos: usize,
120 unicode_ranges: bool,
121}
122
123const NAME: [bool; 256] = {
125 let mut t = [false; 256];
126 let mut b = 0;
127 while b < 128 {
128 t[b] = matches!(b as u8, b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9' | b'-' | b'_');
129 b += 1;
130 }
131 t
132};
133
134fn is_newline(b: u8) -> bool {
136 matches!(b, b'\n' | b'\r' | b'\x0c')
137}
138
139fn is_whitespace(b: u8) -> bool {
140 matches!(b, b' ' | b'\t' | b'\n' | b'\r' | b'\x0c')
141}
142
143fn is_non_printable(b: u8) -> bool {
144 matches!(b, 0x01..=0x08 | 0x0b | 0x0e..=0x1f | 0x7f)
145}
146
147pub(crate) fn is_non_ascii_ident(c: char) -> bool {
150 matches!(c as u32,
151 0xb7 | 0xc0..=0xd6 | 0xd8..=0xf6 | 0xf8..=0x37d | 0x37f..=0x1fff
152 | 0x200c | 0x200d | 0x203f | 0x2040 | 0x2070..=0x218f
153 | 0x2c00..=0x2fef | 0x3001..=0xd7ff | 0xf900..=0xfdcf
154 | 0xfdf0..=0xfffd | 0x10000..)
155}
156
157impl<'a> Tokenizer<'a> {
158 pub fn new(src: &'a str) -> Tokenizer<'a> {
159 Tokenizer {
160 src,
161 pos: 0,
162 unicode_ranges: false,
163 }
164 }
165
166 pub fn with_unicode_ranges(src: &'a str) -> Tokenizer<'a> {
169 Tokenizer {
170 unicode_ranges: true,
171 ..Tokenizer::new(src)
172 }
173 }
174
175 pub fn position(&self) -> usize {
177 self.pos
178 }
179
180 pub fn reset(&mut self, position: usize) {
182 self.pos = position;
183 }
184
185 pub fn source(&self) -> &'a str {
186 self.src
187 }
188
189 #[inline]
190 fn at(&self, i: usize) -> Option<u8> {
191 self.src.as_bytes().get(i).copied()
192 }
193
194 fn char_at(&self, i: usize) -> Option<char> {
195 self.src.get(i..)?.chars().next()
196 }
197
198 fn is_ident_start(&self, i: usize) -> bool {
199 match self.at(i) {
200 Some(b'a'..=b'z' | b'A'..=b'Z' | b'_' | 0) => true,
201 Some(0x80..) => self.char_at(i).is_some_and(is_non_ascii_ident),
202 _ => false,
203 }
204 }
205
206 fn is_ident_char(&self, i: usize) -> bool {
207 matches!(self.at(i), Some(b'0'..=b'9' | b'-')) || self.is_ident_start(i)
208 }
209
210 fn is_escape(&self, i: usize) -> bool {
212 self.at(i) == Some(b'\\') && !self.at(i + 1).is_some_and(is_newline)
213 }
214
215 fn starts_ident(&self, i: usize) -> bool {
216 match self.at(i) {
217 Some(b'-') => {
218 self.at(i + 1) == Some(b'-') || self.is_ident_start(i + 1) || self.is_escape(i + 1)
219 }
220 Some(b'\\') => self.is_escape(i),
221 _ => self.is_ident_start(i),
222 }
223 }
224
225 fn starts_number(&self, i: usize) -> bool {
226 let digit = |j| self.at(j).is_some_and(|b: u8| b.is_ascii_digit());
227 match self.at(i) {
228 Some(b'+' | b'-') => digit(i + 1) || (self.at(i + 1) == Some(b'.') && digit(i + 2)),
229 Some(b'.') => digit(i + 1),
230 Some(b'0'..=b'9') => true,
231 _ => false,
232 }
233 }
234
235 fn starts_unicode_range(&self, i: usize) -> bool {
236 matches!(self.at(i), Some(b'u' | b'U'))
237 && self.at(i + 1) == Some(b'+')
238 && self
239 .at(i + 2)
240 .is_some_and(|b| b == b'?' || b.is_ascii_hexdigit())
241 }
242
243 fn skip_newline(&mut self) {
245 if self.at(self.pos) == Some(b'\r') && self.at(self.pos + 1) == Some(b'\n') {
246 self.pos += 2;
247 } else {
248 self.pos += 1;
249 }
250 }
251
252 fn skip_whitespace(&mut self) {
253 let bytes = self.src.as_bytes();
254 while bytes.get(self.pos).is_some_and(|&b| is_whitespace(b)) {
255 self.pos += 1;
256 }
257 }
258
259 fn skip_comments(&mut self) {
260 while self.src.as_bytes()[self.pos..].starts_with(b"/*") {
261 self.pos = match self.src[self.pos + 2..].find("*/") {
262 Some(end) => self.pos + 2 + end + 2,
263 None => self.src.len(),
264 };
265 }
266 }
267
268 fn escape(&mut self) -> char {
270 let Some(b) = self.at(self.pos) else {
271 return '\u{fffd}';
272 };
273 if b.is_ascii_hexdigit() {
274 let start = self.pos;
275 while self.pos - start < 6 && self.at(self.pos).is_some_and(|b| b.is_ascii_hexdigit()) {
276 self.pos += 1;
277 }
278 let n = u32::from_str_radix(&self.src[start..self.pos], 16).unwrap();
279 if self.at(self.pos).is_some_and(is_whitespace) {
280 self.skip_newline();
281 }
282 return match char::from_u32(n) {
283 Some(c) if n != 0 => c,
284 _ => '\u{fffd}',
285 };
286 }
287 if b == 0 {
288 self.pos += 1;
289 return '\u{fffd}';
290 }
291 let c = self.char_at(self.pos).unwrap();
292 self.pos += c.len_utf8();
293 c
294 }
295
296 fn ident_sequence(&mut self) -> Cow<'a, str> {
298 let start = self.pos;
299 let bytes = self.src.as_bytes();
300 loop {
301 while bytes.get(self.pos).is_some_and(|&b| NAME[b as usize]) {
303 self.pos += 1;
304 }
305 match self.at(self.pos) {
306 Some(0x80..) if self.is_ident_start(self.pos) => {
307 self.pos += self.char_at(self.pos).unwrap().len_utf8();
308 }
309 Some(b'\\' | 0) => break,
310 _ => return Cow::Borrowed(&self.src[start..self.pos]),
311 }
312 }
313 let mut out = self.src[start..self.pos].to_string();
314 loop {
315 if self.is_escape(self.pos) {
316 self.pos += 1;
317 out.push(self.escape());
318 } else if self.at(self.pos) == Some(0) {
319 self.pos += 1;
320 out.push('\u{fffd}');
321 } else if self.is_ident_char(self.pos) {
322 let c = self.char_at(self.pos).unwrap();
323 self.pos += c.len_utf8();
324 out.push(c);
325 } else {
326 return Cow::Owned(out);
327 }
328 }
329 }
330
331 fn number(&mut self) -> Number<'a> {
332 let start = self.pos;
333 let digits = |t: &mut Self| {
334 while t.at(t.pos).is_some_and(|b| b.is_ascii_digit()) {
335 t.pos += 1;
336 }
337 };
338 let digit = |t: &Self, i| t.at(i).is_some_and(|b: u8| b.is_ascii_digit());
339 let mut integer = true;
340 if matches!(self.at(self.pos), Some(b'+' | b'-')) {
341 self.pos += 1;
342 }
343 digits(self);
344 if self.at(self.pos) == Some(b'.') && digit(self, self.pos + 1) {
345 self.pos += 1;
346 digits(self);
347 integer = false;
348 }
349 if matches!(self.at(self.pos), Some(b'e' | b'E')) {
350 let sign = matches!(self.at(self.pos + 1), Some(b'+' | b'-')) as usize;
351 if digit(self, self.pos + 1 + sign) {
352 self.pos += 1 + sign;
353 digits(self);
354 integer = false;
355 }
356 }
357 let repr = &self.src[start..self.pos];
358 Number {
359 value: repr.parse().unwrap_or(0.0),
360 integer,
361 repr,
362 }
363 }
364
365 fn numeric(&mut self) -> Token<'a> {
366 let number = self.number();
367 if self.starts_ident(self.pos) {
368 let unit = self.ident_sequence();
369 Token::Dimension { number, unit }
370 } else if self.at(self.pos) == Some(b'%') {
371 self.pos += 1;
372 Token::Percentage(number)
373 } else {
374 Token::Number(number)
375 }
376 }
377
378 fn ident_like(&mut self) -> Token<'a> {
379 let name = self.ident_sequence();
380 if self.at(self.pos) != Some(b'(') {
381 return Token::Ident(name);
382 }
383 self.pos += 1;
384 if !name.eq_ignore_ascii_case("url") {
385 return Token::Function(name);
386 }
387 let open = self.pos;
388 while self.at(self.pos).is_some_and(is_whitespace)
389 && self.at(self.pos + 1).is_some_and(is_whitespace)
390 {
391 self.pos += 1;
392 }
393 let quote = |b: Option<u8>| matches!(b, Some(b'"' | b'\''));
394 let next = self.at(self.pos);
395 if quote(next) || (next.is_some_and(is_whitespace) && quote(self.at(self.pos + 1))) {
396 self.pos = open;
400 return Token::Function(name);
401 }
402 self.url()
403 }
404
405 fn url(&mut self) -> Token<'a> {
407 self.skip_whitespace();
408 let start = self.pos;
409 let mut owned: Option<String> = None;
410 while let Some(b) = self.at(self.pos) {
411 match b {
412 b')' => break,
413 _ if is_whitespace(b) => {
414 let end = self.pos;
415 self.skip_whitespace();
416 if matches!(self.at(self.pos), None | Some(b')')) {
417 let value = finish(owned, &self.src[start..end]);
418 self.pos = (self.pos + 1).min(self.src.len());
419 return Token::Url(value);
420 }
421 self.bad_url_remnants();
422 return Token::BadUrl;
423 }
424 b'"' | b'\'' | b'(' => {
425 self.bad_url_remnants();
426 return Token::BadUrl;
427 }
428 _ if is_non_printable(b) => {
429 self.bad_url_remnants();
430 return Token::BadUrl;
431 }
432 b'\\' => {
433 if !self.is_escape(self.pos) {
434 self.bad_url_remnants();
435 return Token::BadUrl;
436 }
437 let s = owned.get_or_insert_with(|| self.src[start..self.pos].to_string());
438 self.pos += 1;
439 s.push(self.escape());
440 }
441 0 => {
442 owned
443 .get_or_insert_with(|| self.src[start..self.pos].to_string())
444 .push('\u{fffd}');
445 self.pos += 1;
446 }
447 _ => {
448 let len = self.char_at(self.pos).unwrap().len_utf8();
449 if let Some(s) = &mut owned {
450 s.push_str(&self.src[self.pos..self.pos + len]);
451 }
452 self.pos += len;
453 }
454 }
455 }
456 let value = finish(owned, &self.src[start..self.pos]);
457 self.pos = (self.pos + 1).min(self.src.len());
458 Token::Url(value)
459 }
460
461 fn bad_url_remnants(&mut self) {
462 while let Some(b) = self.at(self.pos) {
463 if b == b')' {
464 self.pos += 1;
465 return;
466 }
467 if self.is_escape(self.pos) {
468 self.pos += 1;
469 self.escape();
470 } else {
471 self.pos += self.char_at(self.pos).unwrap().len_utf8();
472 }
473 }
474 }
475
476 fn string(&mut self, quote: u8) -> Token<'a> {
478 let start = self.pos;
479 let mut owned: Option<String> = None;
480 loop {
481 let run = self.pos;
483 while self
484 .at(self.pos)
485 .is_some_and(|b| b != quote && !matches!(b, b'\\' | b'\n' | b'\r' | b'\x0c' | 0))
486 {
487 self.pos += 1;
488 }
489 if let Some(s) = &mut owned {
490 s.push_str(&self.src[run..self.pos]);
491 }
492 match self.at(self.pos) {
493 None => return Token::String(finish(owned, &self.src[start..self.pos])),
494 Some(b) if b == quote => {
495 let value = finish(owned, &self.src[start..self.pos]);
496 self.pos += 1;
497 return Token::String(value);
498 }
499 Some(b'\n' | b'\r' | b'\x0c') => return Token::BadString,
500 Some(0) => {
501 owned
502 .get_or_insert_with(|| self.src[start..self.pos].to_string())
503 .push('\u{fffd}');
504 self.pos += 1;
505 }
506 _ => {
507 let s = owned.get_or_insert_with(|| self.src[start..self.pos].to_string());
508 self.pos += 1;
509 match self.at(self.pos) {
510 None => {}
511 Some(b) if is_newline(b) => self.skip_newline(),
512 Some(_) => {
513 let c = self.escape();
514 s.push(c);
515 }
516 }
517 }
518 }
519 }
520 }
521
522 fn unicode_range(&mut self) -> Token<'a> {
524 self.pos += 2;
525 let hex = |t: &mut Self, max: usize| {
526 let start = t.pos;
527 while t.pos - start < max && t.at(t.pos).is_some_and(|b| b.is_ascii_hexdigit()) {
528 t.pos += 1;
529 }
530 start
531 };
532 let start = hex(self, 6);
533 let digits = self.pos - start;
534 while self.pos - start < 6 && self.at(self.pos) == Some(b'?') {
535 self.pos += 1;
536 }
537 let first = &self.src[start..self.pos];
538 if self.pos - start > digits {
539 let low = u32::from_str_radix(&first.replace('?', "0"), 16).unwrap();
540 let high = u32::from_str_radix(&first.replace('?', "F"), 16).unwrap();
541 return Token::UnicodeRange {
542 start: low,
543 end: high,
544 };
545 }
546 let low = u32::from_str_radix(first, 16).unwrap();
547 if self.at(self.pos) == Some(b'-')
548 && self.at(self.pos + 1).is_some_and(|b| b.is_ascii_hexdigit())
549 {
550 self.pos += 1;
551 let s = hex(self, 6);
552 let high = u32::from_str_radix(&self.src[s..self.pos], 16).unwrap();
553 return Token::UnicodeRange {
554 start: low,
555 end: high,
556 };
557 }
558 Token::UnicodeRange {
559 start: low,
560 end: low,
561 }
562 }
563
564 pub fn next_token(&mut self) -> Option<(Token<'a>, Span)> {
566 self.skip_comments();
567 let start = self.pos;
568 let b = self.at(start)?;
569 let simple = |t: &mut Self, token| {
570 t.pos += 1;
571 token
572 };
573 let token = match b {
574 b'a'..=b't' | b'v'..=b'z' | b'A'..=b'T' | b'V'..=b'Z' | b'_' => self.ident_like(),
576 b' ' | b'\t' | b'\n' | b'\r' | b'\x0c' => {
577 self.skip_whitespace();
578 Token::Whitespace
579 }
580 b'"' | b'\'' => {
581 self.pos += 1;
582 self.string(b)
583 }
584 b'#' => {
585 if self.is_ident_char(start + 1) || self.is_escape(start + 1) {
586 let id = self.starts_ident(start + 1);
587 self.pos += 1;
588 Token::Hash {
589 value: self.ident_sequence(),
590 id,
591 }
592 } else {
593 simple(self, Token::Delim('#'))
594 }
595 }
596 b'(' => simple(self, Token::LeftParen),
597 b')' => simple(self, Token::RightParen),
598 b',' => simple(self, Token::Comma),
599 b':' => simple(self, Token::Colon),
600 b';' => simple(self, Token::Semicolon),
601 b'[' => simple(self, Token::LeftBracket),
602 b']' => simple(self, Token::RightBracket),
603 b'{' => simple(self, Token::LeftBrace),
604 b'}' => simple(self, Token::RightBrace),
605 b'+' | b'.' if self.starts_number(start) => self.numeric(),
606 b'-' if self.starts_number(start) => self.numeric(),
607 b'-' if self.src.as_bytes()[start + 1..].starts_with(b"->") => {
608 self.pos += 3;
609 Token::Cdc
610 }
611 b'-' if self.starts_ident(start) => self.ident_like(),
612 b'0'..=b'9' => self.numeric(),
613 b'<' if self.src.as_bytes()[start + 1..].starts_with(b"!--") => {
614 self.pos += 4;
615 Token::Cdo
616 }
617 b'@' if self.starts_ident(start + 1) => {
618 self.pos += 1;
619 Token::AtKeyword(self.ident_sequence())
620 }
621 b'\\' if self.is_escape(start) => self.ident_like(),
622 b'u' | b'U' if self.unicode_ranges && self.starts_unicode_range(start) => {
623 self.unicode_range()
624 }
625 _ if self.is_ident_start(start) => self.ident_like(),
626 0..0x80 => simple(self, Token::Delim(b as char)),
627 _ => {
628 let c = self.char_at(start).unwrap();
629 self.pos += c.len_utf8();
630 Token::Delim(c)
631 }
632 };
633 Some((token, Span::new(start, self.pos)))
634 }
635}
636
637fn finish<'a>(owned: Option<String>, borrowed: &'a str) -> Cow<'a, str> {
638 match owned {
639 Some(s) => Cow::Owned(s),
640 None => Cow::Borrowed(borrowed),
641 }
642}
643
644impl<'a> Iterator for Tokenizer<'a> {
645 type Item = (Token<'a>, Span);
646
647 fn next(&mut self) -> Option<(Token<'a>, Span)> {
648 self.next_token()
649 }
650}
651
652#[cfg(test)]
653mod tests {
654 use super::*;
655
656 fn tokens(css: &str) -> Vec<Token<'_>> {
657 Tokenizer::new(css).map(|(t, _)| t).collect()
658 }
659
660 #[test]
661 fn values_borrow_unless_an_escape_changes_them() {
662 let t = tokens("abc a\\62 c 'x\\'y' url(q\\)r)");
663 assert!(matches!(&t[0], Token::Ident(Cow::Borrowed("abc"))));
664 assert!(matches!(&t[2], Token::Ident(Cow::Owned(s)) if s == "abc"));
665 assert!(matches!(&t[4], Token::String(Cow::Owned(s)) if s == "x'y"));
666 assert!(matches!(&t[6], Token::Url(Cow::Owned(s)) if s == "q)r"));
667 }
668
669 #[test]
670 fn crlf_counts_as_one_newline() {
671 assert_eq!(tokens("\\41\r\nb"), [Token::Ident("Ab".into())]);
673 assert_eq!(tokens("'a\\\r\nb'"), [Token::String("ab".into())]);
674 assert_eq!(
675 tokens("'a\rb"),
676 [
677 Token::BadString,
678 Token::Whitespace,
679 Token::Ident("b".into())
680 ]
681 );
682 }
683
684 #[test]
685 fn nul_is_u_fffd() {
686 assert_eq!(tokens("\0a"), [Token::Ident("\u{fffd}a".into())]);
687 assert_eq!(tokens("'\0'"), [Token::String("\u{fffd}".into())]);
688 }
689
690 #[test]
691 fn unicode_ranges_only_when_asked() {
692 assert_eq!(tokens("u+1f")[0], Token::Ident("u".into()));
693 let t: Vec<_> = Tokenizer::with_unicode_ranges("u+1f-2F U+4??")
694 .map(|t| t.0)
695 .collect();
696 assert_eq!(
697 t,
698 [
699 Token::UnicodeRange {
700 start: 0x1f,
701 end: 0x2f
702 },
703 Token::Whitespace,
704 Token::UnicodeRange {
705 start: 0x400,
706 end: 0x4ff
707 }
708 ]
709 );
710 }
711}