1use std::{collections::VecDeque, str};
2
3use bitflags::bitflags;
4use memchr::{memchr2, memchr3};
5use rustc_hash::FxHashSet;
6use smallvec::SmallVec;
7
8use crate::{
9 Range,
10 css_syntax::{
11 MAX_CSS_KEYWORD_LEN, decode_css_keyword, is_css_modules_pure_magic_comment,
12 lowercase_ascii_keyword, strip_vendor_prefix,
13 },
14 dependencies::{PropertyKind, special_value_is_candidate},
15 parser::{
16 BlockItemCandidates, BlockItemKind, BlockItemProgress, ComponentValues, RecoveryBoundary,
17 },
18};
19
20const fn build_plain_ascii_name_byte_table() -> [bool; 256] {
21 let mut table = [false; 256];
22 let mut byte = 0usize;
23 while byte < table.len() {
24 table[byte] = (byte >= b'0' as usize && byte <= b'9' as usize)
25 || (byte >= b'A' as usize && byte <= b'Z' as usize)
26 || (byte >= b'a' as usize && byte <= b'z' as usize)
27 || byte == b'_' as usize
28 || byte == b'-' as usize;
29 byte += 1;
30 }
31 table
32}
33
34const PLAIN_ASCII_NAME_BYTE: [bool; 256] = build_plain_ascii_name_byte_table();
35
36pub const C_LINE_FEED: u8 = b'\n';
37pub const C_CARRIAGE_RETURN: u8 = b'\r';
38pub const C_FORM_FEED: u8 = b'\x0c';
39
40pub const C_TAB: u8 = b'\t';
41pub const C_SPACE: u8 = b' ';
42
43pub const C_SOLIDUS: u8 = b'/';
44pub const C_REVERSE_SOLIDUS: u8 = b'\\';
45pub const C_ASTERISK: u8 = b'*';
46
47pub const C_LEFT_PARENTHESIS: u8 = b'(';
48pub const C_RIGHT_PARENTHESIS: u8 = b')';
49pub const C_LEFT_CURLY: u8 = b'{';
50pub const C_RIGHT_CURLY: u8 = b'}';
51pub const C_LEFT_SQUARE: u8 = b'[';
52pub const C_RIGHT_SQUARE: u8 = b']';
53
54pub const C_QUOTATION_MARK: u8 = b'"';
55pub const C_APOSTROPHE: u8 = b'\'';
56
57pub const C_FULL_STOP: u8 = b'.';
58pub const C_COLON: u8 = b':';
59pub const C_SEMICOLON: u8 = b';';
60pub const C_COMMA: u8 = b',';
61pub const C_PERCENTAGE: u8 = b'%';
62pub const C_AT_SIGN: u8 = b'@';
63
64pub const C_LOW_LINE: u8 = b'_';
65pub const C_LOWER_E: u8 = b'e';
66pub const C_UPPER_E: u8 = b'E';
67
68pub const C_NUMBER_SIGN: u8 = b'#';
69pub const C_PLUS_SIGN: u8 = b'+';
70pub const C_HYPHEN_MINUS: u8 = b'-';
71
72pub type Pos = u32;
73
74#[derive(Debug, Clone, Copy, PartialEq, Eq)]
80pub enum TokenKind {
81 Eof,
82 Ident,
83 AtKeyword,
84 Hash,
85 IdHash,
86 QuotedString,
87 Url,
88 Function,
89 Number,
90 Percentage,
91 Dimension,
92 WhiteSpace,
93 Comment,
94 BadComment,
95 BadString,
96 BadUrl,
97 Delim,
98 Colon,
99 Semicolon,
100 Comma,
101 LeftParenthesis,
102 RightParenthesis,
103 LeftSquareBracket,
104 RightSquareBracket,
105 LeftCurlyBracket,
106 RightCurlyBracket,
107 IncludeMatch,
108 DashMatch,
109 PrefixMatch,
110 SuffixMatch,
111 SubstringMatch,
112}
113
114impl TokenKind {
115 #[inline]
116 pub fn is_trivia(self) -> bool {
117 matches!(self, Self::WhiteSpace | Self::Comment)
118 }
119
120 #[inline]
121 pub fn is_scan_error(self) -> bool {
122 matches!(self, Self::BadComment | Self::BadString | Self::BadUrl)
123 }
124}
125
126#[derive(Debug, Clone, Copy, PartialEq, Eq)]
128pub struct Token {
129 pub kind: TokenKind,
130 pub range: Range,
131 pub value_range: Range,
132 pub flags: TokenFlags,
133}
134
135impl Token {
136 #[inline]
137 pub const fn new(kind: TokenKind, range: Range, value_range: Range) -> Self {
138 Self::with_flags(kind, range, value_range, TokenFlags::ascii())
139 }
140
141 #[inline]
142 pub const fn with_flags(
143 kind: TokenKind,
144 range: Range,
145 value_range: Range,
146 flags: TokenFlags,
147 ) -> Self {
148 Self {
149 kind,
150 range,
151 value_range,
152 flags,
153 }
154 }
155}
156
157bitflags! {
158 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
161 pub struct TokenFlags: u8 {
162 const HAS_ESCAPE = 1 << 0;
163 const HAS_NULL = 1 << 1;
164 const IS_ASCII = 1 << 2;
165 }
166}
167
168impl TokenFlags {
169 #[inline]
170 pub const fn ascii() -> Self {
171 Self::IS_ASCII
172 }
173
174 #[inline]
175 pub const fn has_escape(self) -> bool {
176 self.contains(Self::HAS_ESCAPE)
177 }
178
179 #[inline]
180 pub const fn has_null(self) -> bool {
181 self.contains(Self::HAS_NULL)
182 }
183
184 #[inline]
185 pub const fn is_ascii(self) -> bool {
186 self.contains(Self::IS_ASCII)
187 }
188
189 #[inline]
190 fn mark_escape(&mut self) {
191 self.insert(Self::HAS_ESCAPE);
192 }
193
194 #[inline]
195 fn mark_null(&mut self) {
196 self.insert(Self::HAS_NULL);
197 }
198
199 #[inline]
200 fn mark_non_ascii(&mut self) {
201 self.remove(Self::IS_ASCII);
202 }
203
204 #[inline]
205 fn merge(&mut self, other: Self) {
206 let is_ascii = self.is_ascii() && other.is_ascii();
207 self.insert(other);
208 if !is_ascii {
209 self.mark_non_ascii();
210 }
211 }
212}
213
214#[derive(Debug, Clone, Copy, PartialEq, Eq)]
215pub struct Trivia {
216 pub range: Range,
217 pub end: Pos,
218 pub first_comment_start: Option<Pos>,
219 pub has_white_space: bool,
220}
221
222impl Trivia {
223 #[inline]
224 pub fn has_whitespace(self) -> bool {
225 self.has_white_space
226 }
227}
228
229#[derive(Debug, Clone, Copy, PartialEq, Eq)]
232pub struct TokenWithTrivia {
233 pub token: Token,
234 pub leading: Trivia,
235}
236
237pub trait LexerVisitor {
238 fn visit_ident(&mut self, name: &str, range: Range);
239}
240
241impl LexerVisitor for () {
242 #[inline(always)]
243 fn visit_ident(&mut self, _name: &str, _range: Range) {}
244}
245
246#[derive(Debug)]
247pub struct Lexer<'s, V: LexerVisitor = ()> {
248 value: &'s [u8],
249 scan_pos: Pos,
250 visitor: V,
251 function_kind: FunctionKind,
252}
253
254impl<'s, V: LexerVisitor> Lexer<'s, V> {
255 pub fn new(value: &'s str, visitor: V) -> Self {
256 assert!(value.len() <= Pos::MAX as usize, "CSS input is too large");
257 Self {
258 value: value.as_bytes(),
259 scan_pos: 0,
260 visitor,
261 function_kind: FunctionKind::Other,
262 }
263 }
264
265 #[inline]
266 pub(crate) fn visitor_mut(&mut self) -> &mut V {
267 &mut self.visitor
268 }
269
270 #[inline]
271 fn visit_ident(&mut self, kind: TokenKind, range: Range) {
272 if !matches!(kind, TokenKind::Ident | TokenKind::Function) {
273 return;
274 }
275 let value = &self.value[range.start as usize..range.end as usize];
276 let value = unsafe { str::from_utf8_unchecked(value) };
278 self.visitor.visit_ident(value, range);
279 }
280
281 #[inline]
282 pub(crate) fn source_end(&self) -> Pos {
283 self.value.len() as Pos
284 }
285
286 #[inline(always)]
287 pub(crate) fn byte_at(&self, position: Pos) -> Option<u8> {
288 self.value.get(position as usize).copied()
289 }
290
291 #[inline(always)]
292 pub(crate) fn could_start_ident_at(&self, position: Pos) -> bool {
293 self.byte_at(position).is_some_and(|byte| {
294 matches!(byte, C_HYPHEN_MINUS | C_REVERSE_SOLIDUS) || is_name_start_byte(byte)
295 })
296 }
297
298 fn scan_flat_value(&self, start: usize) -> Option<usize> {
301 let rest = &self.value[start..];
302 let short = rest.len().min(32);
303 for (index, byte) in rest[..short].iter().copied().enumerate() {
304 if matches!(byte, C_LEFT_CURLY | C_SEMICOLON | C_RIGHT_CURLY) {
305 return (byte != C_LEFT_CURLY).then_some(start + index);
306 }
307 if is_non_flat_value_byte(byte) {
308 return None;
309 }
310 }
311 let boundary = memchr3(C_LEFT_CURLY, C_SEMICOLON, C_RIGHT_CURLY, &rest[short..])
312 .map_or(rest.len(), |index| short + index);
313 if rest.get(boundary) == Some(&C_LEFT_CURLY)
314 || rest[short..boundary]
315 .iter()
316 .copied()
317 .any(is_non_flat_value_byte)
318 {
319 return None;
320 }
321 Some(start + boundary)
322 }
323
324 fn fast_forward_generic_value<F, C>(
330 &mut self,
331 state: &mut GenericValueScanState,
332 options: GenericValueScanOptions,
333 is_candidate: F,
334 is_comment_candidate: C,
335 icss_symbols: Option<&FxHashSet<&str>>,
336 block_item: Option<BlockItemScan<'_>>,
337 ) -> Option<BufferedToken>
338 where
339 F: FnMut(&str) -> bool,
340 C: FnMut(&str) -> bool,
341 {
342 self.scan_value(
343 state,
344 options,
345 is_candidate,
346 is_comment_candidate,
347 icss_symbols,
348 block_item,
349 )
350 }
351
352 fn scan_value<F, C>(
353 &mut self,
354 state: &mut GenericValueScanState,
355 options: GenericValueScanOptions,
356 mut is_candidate: F,
357 mut is_comment_candidate: C,
358 icss_symbols: Option<&FxHashSet<&str>>,
359 block_item: Option<BlockItemScan<'_>>,
360 ) -> Option<BufferedToken>
361 where
362 F: FnMut(&str) -> bool,
363 C: FnMut(&str) -> bool,
364 {
365 let mut block_item = block_item;
366 let mut position = self.scan_pos as usize;
367 let scan_start = position;
368 let mut candidate_token = None;
369 if block_item
370 .as_ref()
371 .is_some_and(|candidate| !candidate.can_scan_value())
372 {
373 return None;
374 }
375 if options.allow_flat_scan
378 && !options.keep_comments
379 && !options.preserve_strings
380 && !options.preserve_delimiters
381 && !value_is_nested(state, &block_item)
382 && let Some(candidate) = block_item.as_mut()
383 && candidate.flat_probe_start != Some(position as Pos)
384 {
385 candidate.flat_probe_start = Some(position as Pos);
386 if let Some(end) = self.scan_flat_value(position)
387 && (!candidate.collect_dashed
388 || !self.value[position..end]
389 .windows(2)
390 .any(|bytes| bytes == b"--"))
391 {
392 if end > position {
393 note_opaque_value(&mut block_item);
394 }
395 self.scan_pos = end as Pos;
396 return None;
397 }
398 }
399 let preserve_selector = block_item
400 .as_ref()
401 .is_some_and(|candidate| candidate.preserve_selector_tokens());
402 let defer_rule_events =
403 preserve_selector && options.property.is_none() && !options.preserve_delimiters;
404 let preserve_delimiters =
405 options.preserve_delimiters || (preserve_selector && !defer_rule_events);
406 let validating = block_item.is_some();
407 let mut trivia_start = position;
408 let mut trivia_comment = None;
409 let mut trivia_whitespace = false;
410 while position < self.value.len() {
411 let byte = self.value[position];
412 if is_white_space(byte) {
413 trivia_whitespace = true;
414 position += 1;
415 continue;
416 }
417
418 if byte == C_SOLIDUS && self.value.get(position + 1) == Some(&C_ASTERISK) {
419 let (kind, end, _, _, _) = self.scan_comment(position);
420 if options.keep_comments && kind == TokenKind::Comment {
421 let content = &self.value[position + 2..end.saturating_sub(2)];
422 let mut first = 0usize;
423 while content.get(first).is_some_and(|byte| is_white_space(*byte)) {
424 first += 1;
425 }
426 if content.get(first) == Some(&b'c') {
427 let content = unsafe { str::from_utf8_unchecked(content) };
428 if is_comment_candidate(content) {
429 break;
430 }
431 }
432 }
433 trivia_comment.get_or_insert(position as Pos);
434 position = end;
435 continue;
436 }
437
438 let leading = Trivia {
439 range: Range::new(trivia_start as Pos, position as Pos),
440 end: position as Pos,
441 first_comment_start: trivia_comment,
442 has_white_space: trivia_whitespace,
443 };
444 if matches!(byte, C_QUOTATION_MARK | C_APOSTROPHE) {
445 if options.preserve_strings {
446 break;
447 }
448 let (kind, end, _, _, _) = self.scan_string(position, byte);
449 if kind == TokenKind::BadString && block_item.is_some() {
450 break;
451 }
452
453 note_opaque_value(&mut block_item);
454 position = end;
455 trivia_start = position;
456 trivia_comment = None;
457 trivia_whitespace = false;
458 continue;
459 }
460
461 let starts_ident = match byte {
462 C_HYPHEN_MINUS | C_REVERSE_SOLIDUS => self.starts_ident_at(position),
463 _ => is_name_start_byte(byte),
464 };
465 if starts_ident {
466 let end = self.scan_plain_ascii_name(position);
467 if end == position || self.raw_name_needs_tokenizer(end) {
468 break;
469 }
470 let flags = TokenFlags::ascii();
471 let name = unsafe { str::from_utf8_unchecked(&self.value[position..end]) };
472 let is_function = self.value.get(end) == Some(&C_LEFT_PARENTHESIS);
473 let function = if is_function {
474 FunctionKind::classify(name, flags)
475 } else {
476 FunctionKind::Other
477 };
478 let is_candidate = !flags.has_escape()
479 && (is_candidate(name)
480 || options
481 .property
482 .is_some_and(|property| special_value_is_candidate(property, name, icss_symbols)));
483 if block_item.as_ref().is_some_and(|candidate| {
484 candidate.preserve_selector_tokens()
485 && ((is_function && !defer_rule_events)
486 || name.starts_with("--")
487 || (!is_function && candidate.preserve_ident()))
488 }) || flags.has_escape()
489 || is_candidate
490 || (is_function && function.is_dependency())
491 || (is_function && preserve_delimiters)
492 {
493 let is_url = function == FunctionKind::Url;
494 if !is_url {
495 let token_end = if is_function { end + 1 } else { end };
496 candidate_token = Some(BufferedToken::new(
497 TokenWithTrivia {
498 token: Token::with_flags(
499 if is_function {
500 TokenKind::Function
501 } else {
502 TokenKind::Ident
503 },
504 Range::new(position as Pos, token_end as Pos),
505 Range::new(position as Pos, end as Pos),
506 flags,
507 ),
508 leading,
509 },
510 function,
511 ));
512 position = token_end;
513 }
514 break;
515 }
516 if is_function {
517 observe_opaque_value(
518 &mut block_item,
519 TokenKind::Function,
520 position,
521 end + 1,
522 self.value,
523 leading,
524 function,
525 );
526 } else {
527 note_opaque_value(&mut block_item);
528 }
529 self
530 .visitor
531 .visit_ident(name, Range::new(position as Pos, end as Pos));
532 if is_function {
533 if !validating {
534 state.consume(TokenKind::LeftParenthesis);
535 }
536 position = end + 1;
537 } else {
538 position = end;
539 }
540 trivia_start = position;
541 trivia_comment = None;
542 trivia_whitespace = false;
543 continue;
544 }
545
546 let starts_number = match byte {
547 b'0'..=b'9' => true,
548 C_PLUS_SIGN | C_HYPHEN_MINUS | C_FULL_STOP => self.starts_number_at(position),
549 _ => false,
550 };
551 if starts_number {
552 let Some(end) = self.scan_raw_numeric_end(position) else {
553 break;
554 };
555 note_opaque_value(&mut block_item);
556 position = end;
557 trivia_start = position;
558 trivia_comment = None;
559 trivia_whitespace = false;
560 continue;
561 }
562
563 if preserve_selector && matches!(byte, C_NUMBER_SIGN | C_FULL_STOP | C_COLON | C_AT_SIGN) {
564 break;
565 }
566 if validating
567 && matches!(byte, C_LEFT_CURLY | C_RIGHT_PARENTHESIS | C_RIGHT_SQUARE)
568 && !value_is_nested(state, &block_item)
569 {
570 break;
571 }
572 match byte {
573 C_COMMA if preserve_selector => {
574 observe_opaque_value(
576 &mut block_item,
577 TokenKind::Comma,
578 position,
579 position + 1,
580 self.value,
581 leading,
582 FunctionKind::Other,
583 );
584 position += 1;
585 }
586 C_NUMBER_SIGN => {
587 note_opaque_value(&mut block_item);
588 let name_start = position + 1;
589 if self.starts_ident_at(name_start)
590 || self
591 .value
592 .get(name_start)
593 .is_some_and(|byte| is_digit(*byte) || *byte == C_HYPHEN_MINUS)
594 {
595 let end = self.scan_plain_ascii_name(name_start);
596 if end == name_start || self.raw_name_needs_tokenizer(end) {
597 break;
598 }
599 position = end;
600 } else {
601 position = name_start;
602 }
603 }
604 C_LEFT_PARENTHESIS => {
605 if preserve_delimiters {
606 break;
607 }
608 observe_opaque_value(
609 &mut block_item,
610 TokenKind::LeftParenthesis,
611 position,
612 position + 1,
613 self.value,
614 leading,
615 FunctionKind::Other,
616 );
617 if !validating {
618 state.consume(TokenKind::LeftParenthesis);
619 }
620 position += 1;
621 }
622 C_RIGHT_PARENTHESIS => {
623 if preserve_delimiters {
624 break;
625 }
626 observe_opaque_value(
627 &mut block_item,
628 TokenKind::RightParenthesis,
629 position,
630 position + 1,
631 self.value,
632 leading,
633 FunctionKind::Other,
634 );
635 if !validating {
636 state.consume(TokenKind::RightParenthesis);
637 }
638 position += 1;
639 }
640 C_LEFT_SQUARE => {
641 if preserve_delimiters {
642 break;
643 }
644 observe_opaque_value(
645 &mut block_item,
646 TokenKind::LeftSquareBracket,
647 position,
648 position + 1,
649 self.value,
650 leading,
651 FunctionKind::Other,
652 );
653 if !validating {
654 state.consume(TokenKind::LeftSquareBracket);
655 }
656 position += 1;
657 }
658 C_RIGHT_SQUARE => {
659 if preserve_delimiters {
660 break;
661 }
662 observe_opaque_value(
663 &mut block_item,
664 TokenKind::RightSquareBracket,
665 position,
666 position + 1,
667 self.value,
668 leading,
669 FunctionKind::Other,
670 );
671 if !validating {
672 state.consume(TokenKind::RightSquareBracket);
673 }
674 position += 1;
675 }
676 C_LEFT_CURLY
677 if options.stop_at_top_level_left_curly && !value_is_nested(state, &block_item) =>
678 {
679 break;
680 }
681 C_LEFT_CURLY => {
682 if preserve_delimiters {
683 break;
684 }
685 observe_opaque_value(
686 &mut block_item,
687 TokenKind::LeftCurlyBracket,
688 position,
689 position + 1,
690 self.value,
691 leading,
692 FunctionKind::Other,
693 );
694 if !validating {
695 state.consume(TokenKind::LeftCurlyBracket);
696 }
697 position += 1;
698 }
699 C_RIGHT_CURLY
700 if value_expects(state, &block_item, TokenKind::RightCurlyBracket)
701 && !preserve_delimiters =>
702 {
703 observe_opaque_value(
704 &mut block_item,
705 TokenKind::RightCurlyBracket,
706 position,
707 position + 1,
708 self.value,
709 leading,
710 FunctionKind::Other,
711 );
712 if !validating {
713 state.consume(TokenKind::RightCurlyBracket);
714 }
715 position += 1;
716 if block_item
717 .as_ref()
718 .is_some_and(|candidate| !candidate.can_scan_value())
719 {
720 break;
721 }
722 }
723 C_RIGHT_CURLY if preserve_delimiters || !value_is_nested(state, &block_item) => {
724 break;
725 }
726 C_RIGHT_CURLY => position += 1,
727 C_SEMICOLON
728 if RecoveryBoundary::at(TokenKind::Semicolon, value_is_nested(state, &block_item))
729 .is_some() =>
730 {
731 break;
732 }
733 _ => {
734 note_opaque_value(&mut block_item);
735 position += 1;
736 }
737 }
738 trivia_start = position;
739 trivia_comment = None;
740 trivia_whitespace = false;
741 }
742 if preserve_selector && candidate_token.is_none() && position < self.value.len() {
745 position = trivia_start;
746 }
747 self.scan_pos = position as Pos;
748 debug_assert!(
749 self.scan_pos >= scan_start as Pos,
750 "fast_forward moved scan_pos backward from {scan_start} to {position}"
751 );
752 candidate_token
753 }
754
755 pub(crate) fn fast_forward_selector<F, C>(
760 &mut self,
761 square_depth: &mut u32,
762 keep_comments: bool,
763 has_mode: bool,
764 mut is_ident_candidate: F,
765 mut is_comment_candidate: C,
766 ) -> bool
767 where
768 F: FnMut(&str) -> bool,
769 C: FnMut(&str) -> bool,
770 {
771 let mut position = self.scan_pos as usize;
772 let scan_start = position;
773 let mut invalidates_composes = false;
774 let mut trivia_start = None;
775
776 while position < self.value.len() {
777 let byte = self.value[position];
778
779 if is_white_space(byte) {
780 trivia_start.get_or_insert(position);
781 position += 1;
782 continue;
783 }
784
785 if byte == C_SOLIDUS && self.value.get(position + 1) == Some(&C_ASTERISK) {
786 let (kind, end, _, _, _) = self.scan_comment(position);
787 if keep_comments && kind == TokenKind::Comment {
788 let content = &self.value[position + 2..end.saturating_sub(2)];
789 let content = unsafe { str::from_utf8_unchecked(content) };
791 if is_comment_candidate(content) {
792 break;
793 }
794 }
795 trivia_start.get_or_insert(position);
796 position = end;
797 continue;
798 }
799
800 if *square_depth > 0 {
801 if byte == C_REVERSE_SOLIDUS && self.is_valid_escape_at(position) {
802 position = self.scan_escape(position);
803 trivia_start = None;
804 continue;
805 }
806 match byte {
807 C_QUOTATION_MARK | C_APOSTROPHE => {
808 position = self.scan_string(position, byte).1;
809 }
810 C_LEFT_SQUARE => {
811 *square_depth += 1;
812 position += 1;
813 }
814 C_RIGHT_SQUARE => {
815 *square_depth -= 1;
816 position += 1;
817 }
818 _ => position += 1,
819 }
820 trivia_start = None;
821 continue;
822 }
823
824 let starts_ident = match byte {
825 C_HYPHEN_MINUS | C_REVERSE_SOLIDUS => self.starts_ident_at(position),
826 _ => is_name_start_byte(byte),
827 };
828 if starts_ident {
829 let end = self.scan_name(position);
830 if end == position {
831 break;
832 }
833 if self.value.get(end) == Some(&C_LEFT_PARENTHESIS) {
839 break;
840 }
841 let name = &self.value[position..end];
842 let name = unsafe { str::from_utf8_unchecked(name) };
844 if is_ident_candidate(name) {
845 break;
846 }
847 self
848 .visitor
849 .visit_ident(name, Range::new(position as Pos, end as Pos));
850 invalidates_composes = true;
851 trivia_start = None;
852 position = end;
853 continue;
854 }
855
856 let starts_number = match byte {
857 b'0'..=b'9' => true,
858 C_PLUS_SIGN | C_HYPHEN_MINUS | C_FULL_STOP => self.starts_number_at(position),
859 _ => false,
860 };
861 if starts_number {
862 invalidates_composes = true;
863 trivia_start = None;
864 position = self.scan_numeric(position).1;
865 continue;
866 }
867
868 match byte {
869 C_LEFT_SQUARE => {
870 invalidates_composes = true;
871 *square_depth = 1;
872 trivia_start = None;
873 position += 1;
874 }
875 C_QUOTATION_MARK | C_APOSTROPHE => break,
879 C_FULL_STOP | C_NUMBER_SIGN | C_COLON if has_mode => break,
880 C_NUMBER_SIGN => {
881 let name_start = position + 1;
882 position = if self.starts_ident_at(name_start)
883 || self
884 .value
885 .get(name_start)
886 .is_some_and(|byte| is_digit(*byte) || *byte == C_HYPHEN_MINUS)
887 {
888 self.scan_name(name_start)
889 } else {
890 name_start
891 };
892 invalidates_composes = true;
893 trivia_start = None;
894 }
895 C_RIGHT_PARENTHESIS => {
896 position = trivia_start.unwrap_or(position);
897 break;
898 }
899 C_LEFT_PARENTHESIS | C_LEFT_CURLY | C_RIGHT_CURLY | C_COMMA | C_SEMICOLON | C_AT_SIGN => {
900 break;
901 }
902 _ => {
903 invalidates_composes = true;
904 trivia_start = None;
905 position += 1;
906 }
907 }
908 }
909
910 self.scan_pos = position as Pos;
911 debug_assert!(
912 self.scan_pos >= scan_start as Pos,
913 "selector fast-forward moved scan_pos backward from {scan_start} to {position}"
914 );
915 invalidates_composes
916 }
917
918 pub(crate) fn fast_forward(&mut self, end: TokenKind) -> Option<Range> {
934 match end {
935 TokenKind::RightParenthesis
936 | TokenKind::RightSquareBracket
937 | TokenKind::RightCurlyBracket => {}
938 _ => unreachable!("fast_forward accepts only closing bracket kinds"),
939 }
940 let open_start = self.scan_pos;
941 let mut stack: SmallVec<[TokenKind; 8]> = SmallVec::new();
942 stack.push(end);
943 let mut position = open_start as usize;
944 let bytes = self.value;
945 while position < bytes.len() {
946 let byte = bytes[position];
947 match byte {
948 C_QUOTATION_MARK | C_APOSTROPHE => {
949 let (kind, string_end, _, _, _) = self.scan_string(position, byte);
950 if kind == TokenKind::BadString {
951 return None;
952 }
953 position = string_end;
954 }
955 C_SOLIDUS if bytes.get(position + 1) == Some(&C_ASTERISK) => {
956 let (kind, comment_end, _, _, _) = self.scan_comment(position);
957 if kind == TokenKind::BadComment {
958 return None;
959 }
960 position = comment_end;
961 }
962 C_REVERSE_SOLIDUS if self.is_valid_escape_at(position) => {
963 position = self.scan_escape(position);
964 }
965 C_LEFT_PARENTHESIS => {
966 stack.push(TokenKind::RightParenthesis);
967 position += 1;
968 }
969 C_LEFT_SQUARE => {
970 stack.push(TokenKind::RightSquareBracket);
971 position += 1;
972 }
973 C_LEFT_CURLY => {
974 stack.push(TokenKind::RightCurlyBracket);
975 position += 1;
976 }
977 C_RIGHT_PARENTHESIS | C_RIGHT_SQUARE | C_RIGHT_CURLY => {
978 if *stack.last().expect("delimiter stack never empties") != kind_to_right(byte) {
979 return None;
980 }
981 stack.pop();
982 position += 1;
983 if stack.is_empty() {
984 self.scan_pos = position as Pos;
985 debug_assert!(
986 self.scan_pos >= open_start,
987 "fast_forward moved scan_pos backward from {open_start} to {position}"
988 );
989 return Some(Range::new(open_start, position as Pos));
990 }
991 }
992 C_SEMICOLON => return None,
996 _ if byte >= 0x80 => {
997 position += self.utf8_width_at(position);
998 }
999 _ => position += 1,
1000 }
1001 }
1002 None
1003 }
1004
1005 pub(crate) fn slice(&self, start: Pos, end: Pos) -> Option<&'s str> {
1006 let range = Range::new(start, end);
1007 let start = start as usize;
1008 let end = end as usize;
1009 if start > end || end > self.value.len() {
1010 return None;
1011 }
1012 Some(unsafe { slice_unchecked(self.value, &range) })
1015 }
1016
1017 #[inline(always)]
1018 pub(crate) fn slice_trusted(&self, start: Pos, end: Pos) -> &'s str {
1019 debug_assert!(start <= end && end <= self.value.len() as Pos);
1020 unsafe { slice_unchecked(self.value, &Range::new(start, end)) }
1023 }
1024}
1025
1026impl Lexer<'_, ()> {
1027 pub fn slice_range<'a>(input: &'a str, range: &Range) -> Option<&'a str> {
1028 let start = range.start as usize;
1029 let end = range.end as usize;
1030 if start > end
1031 || end > input.len()
1032 || !input.is_char_boundary(start)
1033 || !input.is_char_boundary(end)
1034 {
1035 return None;
1036 }
1037 Some(unsafe { slice_unchecked(input.as_bytes(), range) })
1039 }
1040}
1041
1042unsafe fn slice_unchecked<'a>(input: &'a [u8], range: &Range) -> &'a str {
1043 unsafe {
1044 let value = input.get_unchecked(range.start as usize..range.end as usize);
1045 str::from_utf8_unchecked(value)
1046 }
1047}
1048
1049impl<V: LexerVisitor> Lexer<'_, V> {
1050 #[inline]
1051 pub(crate) fn scan_pos(&self) -> Pos {
1052 self.scan_pos
1053 }
1054}
1055
1056impl<'s, V: LexerVisitor> Lexer<'s, V> {
1057 #[inline]
1063 pub fn next_token(&mut self) -> Token {
1064 let token = self.next_token_without_visit();
1065 self.visit_ident(token.kind, token.value_range);
1066 token
1067 }
1068
1069 fn next_token_without_visit(&mut self) -> Token {
1070 self.function_kind = FunctionKind::Other;
1071 let start = self.scan_pos as usize;
1072 let len = self.value.len();
1073 if start >= len {
1074 self.scan_pos = len as Pos;
1075 return Token::new(
1076 TokenKind::Eof,
1077 Range::new(len as Pos, len as Pos),
1078 Range::new(len as Pos, len as Pos),
1079 );
1080 }
1081
1082 let byte = self.value[start];
1083 let (kind, end, value_start, value_end, flags) = match byte {
1084 C_SPACE | C_TAB | C_LINE_FEED | C_CARRIAGE_RETURN | C_FORM_FEED => {
1085 self.scan_whitespace(start)
1086 }
1087 C_QUOTATION_MARK => self.scan_string(start, C_QUOTATION_MARK),
1088 C_APOSTROPHE => self.scan_string(start, C_APOSTROPHE),
1089 C_NUMBER_SIGN => {
1090 let value_start = start + 1;
1091 if self.starts_ident_at(value_start) {
1092 let (end, flags) = self.scan_name_with_flags(value_start);
1093 (TokenKind::IdHash, end, value_start, end, flags)
1094 } else if self
1095 .value
1096 .get(value_start)
1097 .is_some_and(|byte| is_digit(*byte) || *byte == C_HYPHEN_MINUS)
1098 {
1099 let (end, flags) = self.scan_name_with_flags(value_start);
1100 (TokenKind::Hash, end, value_start, end, flags)
1101 } else {
1102 (
1103 TokenKind::Delim,
1104 start + 1,
1105 start,
1106 start + 1,
1107 TokenFlags::ascii(),
1108 )
1109 }
1110 }
1111 C_LEFT_PARENTHESIS => (
1112 TokenKind::LeftParenthesis,
1113 start + 1,
1114 start,
1115 start + 1,
1116 TokenFlags::ascii(),
1117 ),
1118 C_RIGHT_PARENTHESIS => (
1119 TokenKind::RightParenthesis,
1120 start + 1,
1121 start,
1122 start + 1,
1123 TokenFlags::ascii(),
1124 ),
1125 C_LEFT_SQUARE => (
1126 TokenKind::LeftSquareBracket,
1127 start + 1,
1128 start,
1129 start + 1,
1130 TokenFlags::ascii(),
1131 ),
1132 C_RIGHT_SQUARE => (
1133 TokenKind::RightSquareBracket,
1134 start + 1,
1135 start,
1136 start + 1,
1137 TokenFlags::ascii(),
1138 ),
1139 C_LEFT_CURLY => (
1140 TokenKind::LeftCurlyBracket,
1141 start + 1,
1142 start,
1143 start + 1,
1144 TokenFlags::ascii(),
1145 ),
1146 C_RIGHT_CURLY => (
1147 TokenKind::RightCurlyBracket,
1148 start + 1,
1149 start,
1150 start + 1,
1151 TokenFlags::ascii(),
1152 ),
1153 C_COLON => (
1154 TokenKind::Colon,
1155 start + 1,
1156 start,
1157 start + 1,
1158 TokenFlags::ascii(),
1159 ),
1160 C_SEMICOLON => (
1161 TokenKind::Semicolon,
1162 start + 1,
1163 start,
1164 start + 1,
1165 TokenFlags::ascii(),
1166 ),
1167 C_COMMA => (
1168 TokenKind::Comma,
1169 start + 1,
1170 start,
1171 start + 1,
1172 TokenFlags::ascii(),
1173 ),
1174 C_PLUS_SIGN => {
1175 if self.starts_number_at(start) {
1176 self.scan_numeric(start)
1177 } else {
1178 (
1179 TokenKind::Delim,
1180 start + 1,
1181 start,
1182 start + 1,
1183 TokenFlags::ascii(),
1184 )
1185 }
1186 }
1187 C_HYPHEN_MINUS => {
1188 if self.starts_number_at(start) {
1189 self.scan_numeric(start)
1190 } else if self.starts_ident_at(start) {
1191 self.scan_ident_like(start)
1192 } else {
1193 (
1194 TokenKind::Delim,
1195 start + 1,
1196 start,
1197 start + 1,
1198 TokenFlags::ascii(),
1199 )
1200 }
1201 }
1202 C_FULL_STOP => {
1203 if self.starts_number_at(start) {
1204 self.scan_numeric(start)
1205 } else {
1206 (
1207 TokenKind::Delim,
1208 start + 1,
1209 start,
1210 start + 1,
1211 TokenFlags::ascii(),
1212 )
1213 }
1214 }
1215 C_SOLIDUS => {
1216 if self.value.get(start + 1) == Some(&C_ASTERISK) {
1217 self.scan_comment(start)
1218 } else {
1219 (
1220 TokenKind::Delim,
1221 start + 1,
1222 start,
1223 start + 1,
1224 TokenFlags::ascii(),
1225 )
1226 }
1227 }
1228 C_AT_SIGN => {
1229 let value_start = start + 1;
1230 if self.starts_ident_at(value_start) {
1231 let (end, flags) = self.scan_name_with_flags(value_start);
1232 (TokenKind::AtKeyword, end, value_start, end, flags)
1233 } else {
1234 (
1235 TokenKind::Delim,
1236 start + 1,
1237 start,
1238 start + 1,
1239 TokenFlags::ascii(),
1240 )
1241 }
1242 }
1243 C_REVERSE_SOLIDUS => {
1244 if self.is_valid_escape_at(start) {
1245 self.scan_ident_like(start)
1246 } else {
1247 (
1248 TokenKind::Delim,
1249 start + 1,
1250 start,
1251 start + 1,
1252 TokenFlags::ascii(),
1253 )
1254 }
1255 }
1256 b'0'..=b'9' => self.scan_numeric(start),
1257 b'$' if self.value[start..].starts_with(b"$=") => (
1258 TokenKind::SuffixMatch,
1259 start + 2,
1260 start,
1261 start + 2,
1262 TokenFlags::ascii(),
1263 ),
1264 b'^' if self.value[start..].starts_with(b"^=") => (
1265 TokenKind::PrefixMatch,
1266 start + 2,
1267 start,
1268 start + 2,
1269 TokenFlags::ascii(),
1270 ),
1271 b'|' if self.value[start..].starts_with(b"|=") => (
1272 TokenKind::DashMatch,
1273 start + 2,
1274 start,
1275 start + 2,
1276 TokenFlags::ascii(),
1277 ),
1278 b'~' if self.value[start..].starts_with(b"~=") => (
1279 TokenKind::IncludeMatch,
1280 start + 2,
1281 start,
1282 start + 2,
1283 TokenFlags::ascii(),
1284 ),
1285 b'*' if self.value[start..].starts_with(b"*=") => (
1286 TokenKind::SubstringMatch,
1287 start + 2,
1288 start,
1289 start + 2,
1290 TokenFlags::ascii(),
1291 ),
1292 _ if self.starts_ident_at(start) => self.scan_ident_like(start),
1293 _ => (
1294 TokenKind::Delim,
1295 start + 1,
1296 start,
1297 start + 1,
1298 TokenFlags::ascii(),
1299 ),
1300 };
1301
1302 self.scan_pos = end as Pos;
1303 debug_assert!(
1304 self.scan_pos >= start as Pos,
1305 "scan_pos moved backward from {start} to {}",
1306 self.scan_pos
1307 );
1308 Token::with_flags(
1309 kind,
1310 Range::new(start as Pos, end as Pos),
1311 Range::new(value_start as Pos, value_end as Pos),
1312 flags,
1313 )
1314 }
1315
1316 #[inline]
1317 fn scan_whitespace(&self, start: usize) -> (TokenKind, usize, usize, usize, TokenFlags) {
1318 let bytes = self.value;
1319 let mut end = start;
1320 while end < bytes.len() && is_white_space(bytes[end]) {
1321 end += 1;
1322 }
1323 (TokenKind::WhiteSpace, end, start, end, TokenFlags::ascii())
1324 }
1325
1326 fn scan_comment(&self, start: usize) -> (TokenKind, usize, usize, usize, TokenFlags) {
1327 let bytes = self.value;
1328 let mut end = start + 2;
1329 let mut flags = TokenFlags::ascii();
1330 while end < bytes.len() {
1331 if bytes[end] == C_ASTERISK && bytes.get(end + 1) == Some(&C_SOLIDUS) {
1332 return (TokenKind::Comment, end + 2, start + 2, end, flags);
1333 }
1334 if bytes[end] == 0 {
1335 flags.mark_null();
1336 } else if !bytes[end].is_ascii() {
1337 flags.mark_non_ascii();
1338 }
1339 end += if bytes[end] < 0x80 {
1340 1
1341 } else {
1342 self.utf8_width_at(end)
1343 };
1344 }
1345 (
1346 TokenKind::BadComment,
1347 bytes.len(),
1348 start + 2,
1349 bytes.len(),
1350 flags,
1351 )
1352 }
1353
1354 fn scan_string(&self, start: usize, quote: u8) -> (TokenKind, usize, usize, usize, TokenFlags) {
1355 let mut end = start + 1;
1356 let mut flags = TokenFlags::ascii();
1357 loop {
1358 let short_end = end.saturating_add(32).min(self.value.len());
1362 while end < short_end {
1363 let byte = self.value[end];
1364 if byte == quote {
1365 return (TokenKind::QuotedString, end + 1, start + 1, end, flags);
1366 }
1367 if is_new_line(byte) {
1368 return (TokenKind::BadString, end, start + 1, end, flags);
1369 }
1370 if byte == 0 {
1371 flags.mark_null();
1372 } else if !byte.is_ascii() {
1373 flags.mark_non_ascii();
1374 }
1375 if byte == C_REVERSE_SOLIDUS {
1376 flags.mark_escape();
1377 if self.value.get(end + 1).is_some_and(|next| !next.is_ascii()) {
1378 flags.mark_non_ascii();
1379 }
1380 if self
1381 .value
1382 .get(end + 1)
1383 .is_some_and(|next| is_new_line(*next))
1384 {
1385 end += 1;
1386 if self.value.get(end) == Some(&C_CARRIAGE_RETURN)
1387 && self.value.get(end + 1) == Some(&C_LINE_FEED)
1388 {
1389 end += 1;
1390 }
1391 end += 1;
1392 } else if self.is_valid_escape_at(end) {
1393 end = self.scan_escape(end);
1394 } else {
1395 end += 1;
1396 }
1397 } else {
1398 end += self.utf8_width_at(end);
1399 }
1400 }
1401 if end == self.value.len() {
1402 return (TokenKind::BadString, end, start + 1, end, flags);
1403 }
1404 let rest = &self.value[end..];
1407 let end_rel = memchr3(quote, C_REVERSE_SOLIDUS, C_LINE_FEED, rest).unwrap_or(rest.len());
1408 let segment_end =
1411 memchr2(C_CARRIAGE_RETURN, C_FORM_FEED, &rest[..end_rel]).unwrap_or(end_rel);
1412 if memchr::memchr(b'\0', &rest[..segment_end]).is_some() {
1416 flags.mark_null();
1417 }
1418 if !rest[..segment_end].is_ascii() {
1419 flags.mark_non_ascii();
1420 }
1421 end += segment_end;
1422 if segment_end != end_rel {
1423 return (TokenKind::BadString, end, start + 1, end, flags);
1424 }
1425 }
1428 }
1429
1430 fn scan_ident_like(&mut self, start: usize) -> (TokenKind, usize, usize, usize, TokenFlags) {
1431 let (name_end, name_flags) = self.scan_name_with_flags(start);
1432 if self.value.get(name_end) != Some(&C_LEFT_PARENTHESIS) {
1433 return (TokenKind::Ident, name_end, start, name_end, name_flags);
1434 }
1435
1436 let open_end = name_end + 1;
1437 let name = &self.value[start..name_end];
1438 let name = unsafe { str::from_utf8_unchecked(name) };
1440 self.function_kind = FunctionKind::classify(name, name_flags);
1441 let is_url = self.function_kind == FunctionKind::Url;
1442 if is_url {
1443 let content_start = self.skip_white_space(open_end);
1444 if !matches!(
1445 self.value.get(content_start),
1446 Some(&C_QUOTATION_MARK | &C_APOSTROPHE)
1447 ) {
1448 let (kind, end, value_start, value_end, url_flags) = self.scan_url(start, content_start);
1449 let mut flags = name_flags;
1450 flags.merge(url_flags);
1451 return (kind, end, value_start, value_end, flags);
1452 }
1453 }
1454 (TokenKind::Function, open_end, start, name_end, name_flags)
1455 }
1456
1457 fn scan_url(
1458 &self,
1459 start: usize,
1460 content_start: usize,
1461 ) -> (TokenKind, usize, usize, usize, TokenFlags) {
1462 let mut end = content_start;
1463 let mut flags = TokenFlags::ascii();
1464 'scan: loop {
1465 let short_end = end.saturating_add(32).min(self.value.len());
1468 while end < short_end {
1469 let byte = self.value[end];
1470 if byte == C_RIGHT_PARENTHESIS {
1471 return (TokenKind::Url, end + 1, content_start, end, flags);
1472 }
1473 if is_white_space(byte) {
1474 let content_end = end;
1475 let close = self.skip_white_space(end);
1476 if self.value.get(close) == Some(&C_RIGHT_PARENTHESIS) {
1477 return (TokenKind::Url, close + 1, content_start, content_end, flags);
1478 }
1479 if close == self.value.len() {
1481 return (TokenKind::Url, close, content_start, content_end, flags);
1482 }
1483 return self.scan_bad_url(start, close, content_start, flags);
1484 }
1485 if byte == 0 {
1486 flags.mark_null();
1487 } else if !byte.is_ascii() {
1488 flags.mark_non_ascii();
1489 }
1490 if byte == C_QUOTATION_MARK
1491 || byte == C_APOSTROPHE
1492 || byte == C_LEFT_PARENTHESIS
1493 || is_non_printable(byte)
1494 {
1495 return self.scan_bad_url(start, end, content_start, flags);
1496 }
1497 if byte == C_REVERSE_SOLIDUS {
1498 flags.mark_escape();
1499 if self.value.get(end + 1).is_some_and(|next| !next.is_ascii()) {
1500 flags.mark_non_ascii();
1501 }
1502 if self.is_valid_escape_at(end) {
1503 end = self.scan_escape(end);
1504 } else if self
1505 .value
1506 .get(end + 1)
1507 .is_some_and(|next| is_new_line(*next))
1508 {
1509 end += 2;
1510 if self.value.get(end - 1) == Some(&C_CARRIAGE_RETURN)
1511 && self.value.get(end) == Some(&C_LINE_FEED)
1512 {
1513 end += 1;
1514 }
1515 } else {
1516 return self.scan_bad_url(start, end, content_start, flags);
1517 }
1518 } else {
1519 end += self.utf8_width_at(end);
1520 }
1521 }
1522 if end == self.value.len() {
1523 return (
1524 TokenKind::Url,
1525 self.value.len(),
1526 content_start,
1527 self.value.len(),
1528 flags,
1529 );
1530 }
1531 let rest = &self.value[end..];
1537 let close_rel = memchr::memchr(C_RIGHT_PARENTHESIS, rest).unwrap_or(rest.len());
1538 let segment = &rest[..close_rel];
1539 if !segment.is_ascii() {
1540 flags.mark_non_ascii();
1541 }
1542 let bad_rel = segment.iter().position(|byte| {
1543 *byte <= C_SPACE
1544 || *byte == C_QUOTATION_MARK
1545 || *byte == C_APOSTROPHE
1546 || *byte == C_LEFT_PARENTHESIS
1547 || *byte == C_REVERSE_SOLIDUS
1548 || *byte == 0x7F
1549 });
1550 if let Some(offset) = bad_rel {
1551 end += offset;
1552 continue 'scan;
1553 }
1554 end += close_rel;
1555 if close_rel == rest.len() {
1556 return (
1557 TokenKind::Url,
1558 self.value.len(),
1559 content_start,
1560 self.value.len(),
1561 flags,
1562 );
1563 }
1564 return (TokenKind::Url, end + 1, content_start, end, flags);
1565 }
1566 }
1567
1568 fn scan_bad_url(
1569 &self,
1570 _start: usize,
1571 mut end: usize,
1572 content_start: usize,
1573 mut flags: TokenFlags,
1574 ) -> (TokenKind, usize, usize, usize, TokenFlags) {
1575 loop {
1576 let short_end = end.saturating_add(32).min(self.value.len());
1577 while end < short_end {
1578 let byte = self.value[end];
1579 if byte == 0 {
1580 flags.mark_null();
1581 } else if !byte.is_ascii() {
1582 flags.mark_non_ascii();
1583 }
1584 if byte == C_RIGHT_PARENTHESIS {
1585 end += 1;
1586 return (TokenKind::BadUrl, end, content_start, end, flags);
1587 }
1588 if byte == C_REVERSE_SOLIDUS {
1589 flags.mark_escape();
1590 if self.value.get(end + 1).is_some_and(|next| !next.is_ascii()) {
1591 flags.mark_non_ascii();
1592 }
1593 if self.is_valid_escape_at(end) {
1594 end = self.scan_escape(end);
1595 } else if self
1596 .value
1597 .get(end + 1)
1598 .is_some_and(|next| is_new_line(*next))
1599 {
1600 end += 2;
1601 if self.value.get(end - 1) == Some(&C_CARRIAGE_RETURN)
1602 && self.value.get(end) == Some(&C_LINE_FEED)
1603 {
1604 end += 1;
1605 }
1606 } else {
1607 end += 1;
1608 }
1609 } else {
1610 end += self.utf8_width_at(end);
1611 }
1612 }
1613 if end == self.value.len() {
1614 break;
1615 }
1616 let rest = &self.value[end..];
1619 let rel = memchr2(C_RIGHT_PARENTHESIS, C_REVERSE_SOLIDUS, rest).unwrap_or(rest.len());
1620 let segment = &rest[..rel];
1621 if memchr::memchr(b'\0', segment).is_some() {
1622 flags.mark_null();
1623 }
1624 if !segment.is_ascii() {
1625 flags.mark_non_ascii();
1626 }
1627 end += rel;
1628 }
1631 (TokenKind::BadUrl, end, content_start, end, flags)
1632 }
1633
1634 fn scan_numeric(&self, start: usize) -> (TokenKind, usize, usize, usize, TokenFlags) {
1635 let end = self.scan_number_end(start);
1636 if self.value.get(end) == Some(&C_PERCENTAGE) {
1637 return (
1638 TokenKind::Percentage,
1639 end + 1,
1640 start,
1641 end + 1,
1642 TokenFlags::ascii(),
1643 );
1644 }
1645 if self.starts_ident_at(end) {
1646 let (end, flags) = self.scan_name_with_flags(end);
1647 return (TokenKind::Dimension, end, start, end, flags);
1648 }
1649 (TokenKind::Number, end, start, end, TokenFlags::ascii())
1650 }
1651
1652 #[inline]
1653 fn scan_number_end(&self, start: usize) -> usize {
1654 let mut end = start;
1655 if matches!(self.value.get(end), Some(&C_PLUS_SIGN | &C_HYPHEN_MINUS)) {
1656 end += 1;
1657 }
1658 while self.value.get(end).is_some_and(|byte| is_digit(*byte)) {
1659 end += 1;
1660 }
1661 if self.value.get(end) == Some(&C_FULL_STOP)
1662 && self.value.get(end + 1).is_some_and(|byte| is_digit(*byte))
1663 {
1664 end += 1;
1665 while self.value.get(end).is_some_and(|byte| is_digit(*byte)) {
1666 end += 1;
1667 }
1668 }
1669 if matches!(self.value.get(end), Some(&C_LOWER_E | &C_UPPER_E)) {
1670 let exponent_start = end;
1671 let mut exponent_end = end + 1;
1672 if matches!(
1673 self.value.get(exponent_end),
1674 Some(&C_PLUS_SIGN | &C_HYPHEN_MINUS)
1675 ) {
1676 exponent_end += 1;
1677 }
1678 if self
1679 .value
1680 .get(exponent_end)
1681 .is_some_and(|byte| is_digit(*byte))
1682 {
1683 end = exponent_end + 1;
1684 while self.value.get(end).is_some_and(|byte| is_digit(*byte)) {
1685 end += 1;
1686 }
1687 } else {
1688 end = exponent_start;
1689 }
1690 }
1691 end
1692 }
1693
1694 #[inline]
1695 fn skip_white_space(&self, mut position: usize) -> usize {
1696 while self
1697 .value
1698 .get(position)
1699 .is_some_and(|byte| is_white_space(*byte))
1700 {
1701 position += 1;
1702 }
1703 position
1704 }
1705
1706 #[inline]
1707 fn scan_plain_ascii_name(&self, mut position: usize) -> usize {
1708 while self
1709 .value
1710 .get(position)
1711 .is_some_and(|byte| PLAIN_ASCII_NAME_BYTE[*byte as usize])
1712 {
1713 position += 1;
1714 }
1715 position
1716 }
1717
1718 #[inline]
1719 fn raw_name_needs_tokenizer(&self, position: usize) -> bool {
1720 self
1721 .value
1722 .get(position)
1723 .is_some_and(|byte| *byte == 0 || *byte == C_REVERSE_SOLIDUS || !byte.is_ascii())
1724 }
1725
1726 #[inline]
1727 fn scan_raw_numeric_end(&self, start: usize) -> Option<usize> {
1728 let end = self.scan_number_end(start);
1729 if self.value.get(end) == Some(&C_PERCENTAGE) {
1730 return Some(end + 1);
1731 }
1732 if self.starts_ident_at(end) {
1733 let name_end = self.scan_plain_ascii_name(end);
1734 if name_end == end || self.raw_name_needs_tokenizer(name_end) {
1735 return None;
1736 }
1737 return Some(name_end);
1738 }
1739 if self.raw_name_needs_tokenizer(end) {
1740 return None;
1741 }
1742 Some(end)
1743 }
1744
1745 #[inline]
1746 fn scan_name(&self, position: usize) -> usize {
1747 self.scan_name_impl(position, false).0
1748 }
1749
1750 #[inline]
1751 fn scan_name_with_flags(&self, position: usize) -> (usize, TokenFlags) {
1752 self.scan_name_impl(position, true)
1753 }
1754
1755 #[inline]
1756 fn scan_name_impl(&self, mut position: usize, track_flags: bool) -> (usize, TokenFlags) {
1757 let bytes = self.value;
1758 let mut flags = TokenFlags::ascii();
1759 while position < bytes.len() {
1760 while position < bytes.len() {
1761 let byte = bytes[position];
1762 if PLAIN_ASCII_NAME_BYTE[byte as usize] {
1763 position += 1;
1764 } else if byte == 0 {
1765 if track_flags {
1766 flags.mark_null();
1767 }
1768 position += 1;
1769 } else {
1770 break;
1771 }
1772 }
1773 if position == bytes.len() {
1774 break;
1775 }
1776
1777 let byte = bytes[position];
1778 if byte == C_REVERSE_SOLIDUS {
1779 if self.is_valid_escape_at(position) {
1780 if track_flags {
1781 flags.mark_escape();
1782 if bytes.get(position + 1).is_some_and(|next| !next.is_ascii()) {
1783 flags.mark_non_ascii();
1784 }
1785 }
1786 position = self.scan_escape(position);
1787 } else {
1788 break;
1789 }
1790 } else if byte.is_ascii() {
1791 break;
1792 } else {
1793 if track_flags {
1794 flags.mark_non_ascii();
1795 }
1796 position += self.utf8_width_at(position);
1797 }
1798 }
1799 (position, flags)
1800 }
1801
1802 #[inline]
1803 fn starts_ident_at(&self, position: usize) -> bool {
1804 let Some(byte) = self.value.get(position).copied() else {
1805 return false;
1806 };
1807 match byte {
1808 C_HYPHEN_MINUS => match self.value.get(position + 1).copied() {
1809 Some(next) => {
1810 is_name_start_byte(next)
1811 || next == C_HYPHEN_MINUS
1812 || self.is_valid_escape_at(position + 1)
1813 }
1814 None => false,
1815 },
1816 C_REVERSE_SOLIDUS => self.is_valid_escape_at(position),
1817 _ => is_name_start_byte(byte),
1818 }
1819 }
1820
1821 #[inline]
1822 fn starts_number_at(&self, position: usize) -> bool {
1823 let Some(first) = self.value.get(position).copied() else {
1824 return false;
1825 };
1826 let second = self.value.get(position + 1).copied();
1827 let third = self.value.get(position + 2).copied();
1828 match first {
1829 C_PLUS_SIGN | C_HYPHEN_MINUS => {
1830 second.is_some_and(is_digit) || (second == Some(C_FULL_STOP) && third.is_some_and(is_digit))
1831 }
1832 C_FULL_STOP => second.is_some_and(is_digit),
1833 _ => is_digit(first),
1834 }
1835 }
1836
1837 #[inline]
1838 fn is_valid_escape_at(&self, position: usize) -> bool {
1839 self.value.get(position) == Some(&C_REVERSE_SOLIDUS)
1840 && self
1841 .value
1842 .get(position + 1)
1843 .is_some_and(|byte| !is_new_line(*byte))
1844 }
1845
1846 #[inline]
1847 fn scan_escape(&self, position: usize) -> usize {
1848 debug_assert!(self.is_valid_escape_at(position));
1849 let mut end = position + 1;
1850 if self.value[end].is_ascii_hexdigit() {
1851 let mut digits = 0;
1852 while end < self.value.len() && digits < 6 && self.value[end].is_ascii_hexdigit() {
1853 end += 1;
1854 digits += 1;
1855 }
1856 if self
1857 .value
1858 .get(end)
1859 .is_some_and(|byte| is_white_space(*byte))
1860 {
1861 end += 1;
1862 if self.value.get(end - 1) == Some(&C_CARRIAGE_RETURN)
1863 && self.value.get(end) == Some(&C_LINE_FEED)
1864 {
1865 end += 1;
1866 }
1867 }
1868 return end;
1869 }
1870 end + self.utf8_width_at(end)
1871 }
1872
1873 #[inline]
1874 fn utf8_width_at(&self, position: usize) -> usize {
1875 let byte = self.value[position];
1876 if byte < 0x80 {
1877 1
1878 } else if byte < 0xE0 {
1879 2
1880 } else if byte < 0xF0 {
1881 3
1882 } else {
1883 4
1884 }
1885 }
1886}
1887
1888pub(crate) struct TokenStream<'a, 's, V: LexerVisitor = ()> {
1909 lexer: &'a mut Lexer<'s, V>,
1910 consumed: Pos,
1911 buffered: VecDeque<BufferedToken>,
1912 value_state: GenericValueScanState,
1913 at_rule_state: GenericValueScanState,
1914 pub(crate) block_item: Option<BlockItemCandidates>,
1915 rule_events: Vec<BufferedToken>,
1916 function_kind: FunctionKind,
1917 parsed_value_end: Option<Pos>,
1918}
1919
1920struct BlockItemScan<'a> {
1923 candidate: &'a mut BlockItemCandidates,
1924 events: &'a mut Vec<BufferedToken>,
1925}
1926
1927impl std::ops::Deref for BlockItemScan<'_> {
1928 type Target = BlockItemCandidates;
1929
1930 fn deref(&self) -> &Self::Target {
1931 self.candidate
1932 }
1933}
1934
1935impl std::ops::DerefMut for BlockItemScan<'_> {
1936 fn deref_mut(&mut self) -> &mut Self::Target {
1937 self.candidate
1938 }
1939}
1940
1941#[inline]
1943fn value_is_nested(state: &ComponentValues, candidate: &Option<BlockItemScan<'_>>) -> bool {
1944 candidate
1945 .as_ref()
1946 .map_or_else(|| state.is_nested(), |candidate| candidate.is_nested())
1947}
1948
1949#[inline]
1950fn value_expects(
1951 state: &ComponentValues,
1952 candidate: &Option<BlockItemScan<'_>>,
1953 kind: TokenKind,
1954) -> bool {
1955 candidate
1956 .as_ref()
1957 .map_or_else(|| state.expects(kind), |candidate| candidate.expects(kind))
1958}
1959
1960#[inline]
1961fn note_opaque_value(candidate: &mut Option<BlockItemScan<'_>>) {
1962 if let Some(candidate) = candidate {
1963 candidate.opaque_value();
1964 }
1965}
1966
1967#[inline]
1968fn observe_opaque_value(
1969 candidate: &mut Option<BlockItemScan<'_>>,
1970 kind: TokenKind,
1971 start: usize,
1972 end: usize,
1973 source: &[u8],
1974 leading: Trivia,
1975 function: FunctionKind,
1976) {
1977 if let Some(candidate) = candidate {
1978 let range = Range::new(start as Pos, end as Pos);
1979 let value_range = if kind == TokenKind::Function {
1980 Range::new(start as Pos, end as Pos - 1)
1981 } else {
1982 range
1983 };
1984 let token = Token::with_flags(kind, range, value_range, TokenFlags::ascii());
1985 candidate.observe(token, source);
1986 if candidate.preserve_selector_tokens() {
1987 candidate.events.push(BufferedToken::new(
1988 TokenWithTrivia { token, leading },
1989 function,
1990 ));
1991 }
1992 }
1993}
1994
1995pub(crate) type GenericValueScanState = ComponentValues;
1996
1997#[derive(Debug, Clone, Copy)]
1998struct GenericValueScanOptions {
1999 keep_comments: bool,
2000 preserve_strings: bool,
2001 preserve_delimiters: bool,
2002 property: Option<PropertyKind>,
2003 stop_at_top_level_left_curly: bool,
2004 allow_flat_scan: bool,
2005}
2006
2007#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2009pub(crate) enum FunctionKind {
2010 Other,
2011 Url,
2012 ImageSet,
2013 Layer,
2014 Supports,
2015 PaletteMix,
2016 Var,
2017 Local,
2018 Global,
2019 Dashed,
2020}
2021
2022impl FunctionKind {
2023 #[inline]
2024 fn classify(name: &str, flags: TokenFlags) -> Self {
2025 let mut lowercase = [0; MAX_CSS_KEYWORD_LEN];
2026 let normalized = if flags.has_escape() {
2027 decode_css_keyword(name, &mut lowercase)
2028 } else {
2029 lowercase_ascii_keyword(name, &mut lowercase)
2030 };
2031 match normalized {
2032 Some("url") => Self::Url,
2033 Some("image-set") => Self::ImageSet,
2034 Some("layer") => Self::Layer,
2035 Some("supports") => Self::Supports,
2036 Some("palette-mix") => Self::PaletteMix,
2037 Some("var") => Self::Var,
2038 Some("local") => Self::Local,
2039 Some("global") => Self::Global,
2040 Some(name) if strip_vendor_prefix(name) == Some("image-set") => Self::ImageSet,
2041 _ if name.starts_with("--") => Self::Dashed,
2042 _ => Self::Other,
2043 }
2044 }
2045
2046 #[inline]
2047 fn is_dependency(self) -> bool {
2048 matches!(self, Self::Url | Self::ImageSet | Self::Var | Self::Dashed)
2049 }
2050}
2051
2052#[derive(Debug, Clone, Copy)]
2055struct BufferedToken {
2056 token: Token,
2057 leading_start: Pos,
2058 first_comment_start: Pos,
2059 has_whitespace: bool,
2060 function: FunctionKind,
2061}
2062
2063impl BufferedToken {
2064 fn new(item: TokenWithTrivia, function: FunctionKind) -> Self {
2065 debug_assert_eq!(item.leading.end, item.token.range.start);
2066 debug_assert_eq!(item.leading.range.end, item.token.range.start);
2067 Self {
2068 token: item.token,
2069 leading_start: item.leading.range.start,
2070 first_comment_start: item.leading.first_comment_start.unwrap_or(Pos::MAX),
2071 has_whitespace: item.leading.has_white_space,
2072 function,
2073 }
2074 }
2075
2076 #[inline]
2077 fn into_item(self) -> TokenWithTrivia {
2078 TokenWithTrivia {
2079 token: self.token,
2080 leading: Trivia {
2081 range: Range::new(self.leading_start, self.token.range.start),
2082 end: self.token.range.start,
2083 first_comment_start: (self.first_comment_start != Pos::MAX)
2084 .then_some(self.first_comment_start),
2085 has_white_space: self.has_whitespace,
2086 },
2087 }
2088 }
2089}
2090
2091#[inline(always)]
2092fn never_fast_forward_candidate(_: &str) -> bool {
2093 false
2094}
2095
2096impl<'a, 's, V: LexerVisitor> TokenStream<'a, 's, V> {
2097 pub(crate) fn from_lexer(lexer: &'a mut Lexer<'s, V>) -> Self {
2098 Self {
2099 lexer,
2100 consumed: 0,
2101 buffered: VecDeque::new(),
2102 value_state: GenericValueScanState::default(),
2103 at_rule_state: GenericValueScanState::default(),
2104 block_item: None,
2105 rule_events: Vec::new(),
2106 function_kind: FunctionKind::Other,
2107 parsed_value_end: None,
2108 }
2109 }
2110
2111 #[inline(always)]
2112 pub(crate) fn next(&mut self, keep_comments: bool) -> TokenWithTrivia {
2113 let token = if let Some(token) = self.buffered.pop_front() {
2114 token
2115 } else {
2116 self.read_significant(keep_comments)
2117 };
2118 debug_assert!(
2119 token.token.range.start >= self.consumed,
2120 "token range starts before consumed_pos: {:?} < {}",
2121 token.token.range,
2122 self.consumed
2123 );
2124 if let Some(candidate) = &mut self.block_item
2125 && !matches!(candidate.progress, BlockItemProgress::Finished(_))
2126 {
2127 candidate.observe(token.token, self.lexer.value);
2128 if candidate.retain_rule_event(token.token.kind) {
2129 self.rule_events.push(token);
2130 }
2131 }
2132 self.function_kind = token.function;
2133 self
2134 .lexer
2135 .visit_ident(token.token.kind, token.token.value_range);
2136 self.consumed = token.token.range.end;
2137 debug_assert!(
2138 self.consumed <= self.lexer.scan_pos(),
2139 "consumed_pos ({}) exceeded scan_pos ({})",
2140 self.consumed,
2141 self.lexer.scan_pos()
2142 );
2143 token.into_item()
2144 }
2145
2146 #[inline]
2147 pub(crate) fn function_kind(&self) -> FunctionKind {
2148 self.function_kind
2149 }
2150
2151 #[inline(always)]
2152 pub(crate) fn next_with_progress(
2153 &mut self,
2154 keep_comments: bool,
2155 ) -> (TokenWithTrivia, BlockItemProgress) {
2156 let token = self.next(keep_comments);
2157 (token, self.block_item_progress())
2158 }
2159
2160 #[inline(always)]
2161 pub(crate) fn block_item_progress(&self) -> BlockItemProgress {
2162 self
2163 .block_item
2164 .as_ref()
2165 .map_or(BlockItemProgress::Continue, |candidate| candidate.progress)
2166 }
2167
2168 pub(crate) fn prepare_declaration(
2173 &mut self,
2174 ident: Token,
2175 custom: bool,
2176 flat: bool,
2177 collect_dashed: bool,
2178 ) -> (bool, Option<Pos>) {
2179 let mut flat_probe_start = None;
2180 if self.byte_at(ident.range.end) != Some(C_COLON) {
2181 return (false, flat_probe_start);
2182 }
2183 if !custom && !flat {
2184 return (false, flat_probe_start);
2185 }
2186 let end = if custom {
2187 None
2188 } else {
2189 flat_probe_start = Some(ident.range.end + 1);
2190 let Some(end) = self.lexer.scan_flat_value(ident.range.end as usize + 1) else {
2191 return (false, flat_probe_start);
2192 };
2193 if collect_dashed
2194 && self.lexer.value[ident.range.end as usize + 1..end]
2195 .windows(2)
2196 .any(|bytes| bytes == b"--")
2197 {
2198 return (false, flat_probe_start);
2199 }
2200 Some(end as Pos)
2201 };
2202 if self.buffered.len() != 1 {
2204 return (false, flat_probe_start);
2205 }
2206 let colon = self.read_significant(false);
2207 debug_assert_eq!(colon.token.kind, TokenKind::Colon);
2208 self.buffered.push_back(colon);
2209 self.parsed_value_end = end;
2210 (true, None)
2211 }
2212
2213 pub(crate) fn begin_block_item(
2214 &mut self,
2215 custom: bool,
2216 has_mode: bool,
2217 collect_dashed: bool,
2218 flat_probe_start: Option<Pos>,
2219 ) {
2220 let mut candidate = BlockItemCandidates::new(custom, has_mode);
2221 candidate.flat_probe_start = flat_probe_start;
2222 candidate.collect_dashed = collect_dashed;
2223 self.block_item = Some(candidate);
2224 }
2225
2226 pub(crate) fn finish_block_item(&mut self) -> BlockItemKind {
2229 let candidate = self.block_item.take().expect("an active block item");
2230 let BlockItemProgress::Finished(selected) = candidate.progress else {
2231 unreachable!("the block item has been resolved");
2232 };
2233 match selected {
2234 BlockItemKind::Declaration => {}
2235 BlockItemKind::Rule => {
2236 self.consumed = self
2237 .rule_events
2238 .first()
2239 .expect("a rule has a prefix")
2240 .token
2241 .range
2242 .start;
2243 for event in self.rule_events.drain(..).rev() {
2244 self.buffered.push_front(event);
2245 }
2246 self.reset_value_scan_states();
2247 }
2248 BlockItemKind::Invalid => {
2249 let boundary = *self
2250 .rule_events
2251 .last()
2252 .expect("an invalid item ends at a boundary");
2253 self.consumed = boundary.token.range.start;
2254 self.buffered.push_front(boundary);
2255 self.reset_value_scan_states();
2256 }
2257 }
2258 self.rule_events.clear();
2259 selected
2260 }
2261
2262 pub(crate) fn reset_value_scan_states(&mut self) {
2263 self.value_state = GenericValueScanState::default();
2264 }
2265
2266 pub(crate) fn value_is_nested(&self) -> bool {
2267 self.value_state.is_nested()
2268 || self
2269 .block_item
2270 .as_ref()
2271 .is_some_and(BlockItemCandidates::is_nested)
2272 }
2273
2274 #[inline]
2279 pub(crate) fn next_parser_token(&mut self) -> TokenWithTrivia {
2280 let mut item = self.next(true);
2281 if item.token.kind != TokenKind::Comment {
2282 return item;
2283 }
2284
2285 let start = item.leading.range.start;
2286 let first_comment_start = item
2287 .leading
2288 .first_comment_start
2289 .or(Some(item.token.range.start));
2290 let mut has_white_space = item.leading.has_white_space;
2291 loop {
2292 item = self.next(true);
2293 has_white_space |= item.leading.has_white_space;
2294 if item.token.kind == TokenKind::Comment {
2295 continue;
2296 }
2297 let end = item.leading.end;
2298 item.leading = Trivia {
2299 range: Range::new(start, end),
2300 end,
2301 first_comment_start,
2302 has_white_space,
2303 };
2304 return item;
2305 }
2306 }
2307
2308 #[inline]
2309 pub(crate) fn peek(&mut self, keep_comments: bool) -> TokenWithTrivia {
2310 if self.buffered.is_empty() {
2311 let item = self.read_significant(keep_comments);
2312 self.buffered.push_back(item);
2313 }
2314 self
2315 .buffered
2316 .front()
2317 .expect("peek must buffer an item before reading the front")
2318 .into_item()
2319 }
2320
2321 #[inline]
2325 pub(crate) fn peek_parser_token(&mut self) -> TokenWithTrivia {
2326 self.peek(true);
2327 let mut leading_start = None;
2328 let mut first_comment_start = None;
2329 let mut has_white_space = false;
2330 let mut index = 0usize;
2331 loop {
2332 if index == self.buffered.len() {
2333 let item = self.read_significant(true);
2334 self.buffered.push_back(item);
2335 }
2336 let item = self.buffered[index].into_item();
2337 leading_start.get_or_insert(item.leading.range.start);
2338 has_white_space |= item.leading.has_whitespace();
2339 if matches!(item.token.kind, TokenKind::Comment | TokenKind::BadComment) {
2340 first_comment_start.get_or_insert(item.token.range.start);
2341 index += 1;
2342 continue;
2343 }
2344
2345 let end = item.leading.end;
2346 let mut result = item;
2347 result.leading = Trivia {
2348 range: Range::new(leading_start.unwrap_or(end), end),
2349 end,
2350 first_comment_start: first_comment_start.or(item.leading.first_comment_start),
2351 has_white_space,
2352 };
2353 return result;
2354 }
2355 }
2356
2357 #[inline]
2360 pub(crate) fn peek_significant_skipping_comments(
2361 &mut self,
2362 keep_comments: bool,
2363 ) -> TokenWithTrivia {
2364 self.peek(keep_comments);
2365 for item in &self.buffered {
2366 if !matches!(item.token.kind, TokenKind::Comment | TokenKind::BadComment) {
2367 return item.into_item();
2368 }
2369 }
2370 loop {
2371 let item = self.read_significant(keep_comments);
2372 self.buffered.push_back(item);
2373 if !matches!(item.token.kind, TokenKind::Comment | TokenKind::BadComment) {
2374 return item.into_item();
2375 }
2376 }
2377 }
2378
2379 #[inline]
2382 pub(crate) fn fast_forward_generic_value_if_buffer_empty<F, C>(
2383 &mut self,
2384 keep_comments: bool,
2385 preserve_strings: bool,
2386 preserve_delimiters: bool,
2387 mut is_candidate: F,
2388 mut is_comment_candidate: C,
2389 ) where
2390 F: FnMut(&str) -> bool,
2391 C: FnMut(&str) -> bool,
2392 {
2393 if !self.buffered.is_empty() {
2394 return;
2395 }
2396 let candidate = self.lexer.fast_forward_generic_value(
2397 &mut self.value_state,
2398 GenericValueScanOptions {
2399 keep_comments,
2400 preserve_strings,
2401 preserve_delimiters,
2402 property: None,
2403 stop_at_top_level_left_curly: false,
2404 allow_flat_scan: false,
2405 },
2406 &mut is_candidate,
2407 &mut is_comment_candidate,
2408 None,
2409 self.block_item.as_mut().map(|candidate| BlockItemScan {
2410 candidate,
2411 events: &mut self.rule_events,
2412 }),
2413 );
2414 self.finish_value_fast_forward(candidate);
2415 }
2416
2417 #[inline]
2418 pub(crate) fn fast_forward_generic_value_without_candidates_if_buffer_empty(
2419 &mut self,
2420 preserve_strings: bool,
2421 preserve_delimiters: bool,
2422 ) {
2423 if !self.buffered.is_empty() {
2424 return;
2425 }
2426 if let Some(end) = self.parsed_value_end.take() {
2427 debug_assert!(self.lexer.scan_pos() <= end);
2428 self.lexer.scan_pos = end;
2429 self.consumed = end;
2430 return;
2431 }
2432 let candidate = self.lexer.fast_forward_generic_value(
2433 &mut self.value_state,
2434 GenericValueScanOptions {
2435 keep_comments: false,
2436 preserve_strings,
2437 preserve_delimiters,
2438 property: None,
2439 stop_at_top_level_left_curly: false,
2440 allow_flat_scan: true,
2441 },
2442 never_fast_forward_candidate,
2443 never_fast_forward_candidate,
2444 None,
2445 self.block_item.as_mut().map(|candidate| BlockItemScan {
2446 candidate,
2447 events: &mut self.rule_events,
2448 }),
2449 );
2450 self.finish_value_fast_forward(candidate);
2451 }
2452
2453 #[inline]
2454 pub(crate) fn fast_forward_at_rule_if_buffer_empty<F, C>(
2455 &mut self,
2456 keep_comments: bool,
2457 preserve_strings: bool,
2458 preserve_delimiters: bool,
2459 mut is_candidate: F,
2460 mut is_comment_candidate: C,
2461 ) where
2462 F: FnMut(&str) -> bool,
2463 C: FnMut(&str) -> bool,
2464 {
2465 if !self.buffered.is_empty() {
2466 return;
2467 }
2468 let candidate = self.lexer.fast_forward_generic_value(
2469 &mut self.at_rule_state,
2470 GenericValueScanOptions {
2471 keep_comments,
2472 preserve_strings,
2473 preserve_delimiters,
2474 property: None,
2475 stop_at_top_level_left_curly: true,
2476 allow_flat_scan: false,
2477 },
2478 &mut is_candidate,
2479 &mut is_comment_candidate,
2480 None,
2481 None,
2482 );
2483 self.finish_value_fast_forward(candidate);
2484 }
2485
2486 #[inline]
2487 pub(crate) fn reset_at_rule_scan_state(&mut self) {
2488 self.at_rule_state = GenericValueScanState::default();
2489 }
2490
2491 #[inline]
2492 pub(crate) fn fast_forward_special_value_if_buffer_empty(
2493 &mut self,
2494 keep_comments: bool,
2495 preserve_strings: bool,
2496 preserve_delimiters: bool,
2497 property: PropertyKind,
2498 icss_symbols: Option<&FxHashSet<&str>>,
2499 ) {
2500 if !self.buffered.is_empty() {
2501 return;
2502 }
2503 let candidate = self.lexer.fast_forward_generic_value(
2504 &mut self.value_state,
2505 GenericValueScanOptions {
2506 keep_comments,
2507 preserve_strings,
2508 preserve_delimiters,
2509 property: Some(property),
2510 stop_at_top_level_left_curly: false,
2511 allow_flat_scan: false,
2512 },
2513 never_fast_forward_candidate,
2514 is_css_modules_pure_magic_comment,
2515 icss_symbols,
2516 self.block_item.as_mut().map(|candidate| BlockItemScan {
2517 candidate,
2518 events: &mut self.rule_events,
2519 }),
2520 );
2521 self.finish_value_fast_forward(candidate);
2522 }
2523
2524 #[inline]
2525 fn finish_value_fast_forward(&mut self, candidate: Option<BufferedToken>) {
2526 let Some(candidate) = candidate else {
2527 self.consumed = self.lexer.scan_pos();
2528 return;
2529 };
2530 debug_assert_eq!(candidate.token.range.end, self.lexer.scan_pos());
2531 self.consumed = candidate.token.range.start;
2532 self.buffered.push_back(candidate);
2533 }
2534
2535 #[inline]
2536 pub(crate) fn fast_forward_selector_if_buffer_empty<F, C>(
2537 &mut self,
2538 square_depth: &mut u32,
2539 keep_comments: bool,
2540 has_mode: bool,
2541 is_ident_candidate: F,
2542 is_comment_candidate: C,
2543 ) -> bool
2544 where
2545 F: FnMut(&str) -> bool,
2546 C: FnMut(&str) -> bool,
2547 {
2548 if !self.buffered.is_empty() {
2549 return false;
2550 }
2551 if *square_depth == 0 {
2552 let scan_pos = self.lexer.scan_pos();
2553 let Some(next) = self.lexer.byte_at(scan_pos) else {
2554 return false;
2555 };
2556 if matches!(
2557 next,
2558 b'"' | b'\'' | b'(' | b')' | b'{' | b'}' | b',' | b';' | b'@'
2559 ) || (has_mode && matches!(next, b'.' | b'#' | b':'))
2560 {
2561 return false;
2562 }
2563
2564 let mut probe = scan_pos;
2569 while probe - scan_pos < 4 {
2570 let Some(byte) = self.lexer.byte_at(probe) else {
2571 return false;
2572 };
2573 if matches!(byte, b'/' | b'[') {
2574 break;
2575 }
2576 if probe != scan_pos
2577 && (matches!(
2578 byte,
2579 b'"' | b'\'' | b'(' | b')' | b'{' | b'}' | b',' | b';' | b'@'
2580 ) || (has_mode && matches!(byte, b'.' | b'#' | b':')))
2581 {
2582 return false;
2583 }
2584 probe += 1;
2585 }
2586 }
2587 let invalidates_composes = self.lexer.fast_forward_selector(
2588 square_depth,
2589 keep_comments,
2590 has_mode,
2591 is_ident_candidate,
2592 is_comment_candidate,
2593 );
2594 self.consumed = self.lexer.scan_pos();
2595 invalidates_composes
2596 }
2597
2598 pub(crate) fn fast_forward(&mut self, end: TokenKind) -> Option<Range> {
2605 if !self.buffered.is_empty() {
2606 return None;
2607 }
2608 let old_scan_pos = self.lexer.scan_pos();
2609 let old_consumed = self.consumed;
2610 let range = self.lexer.fast_forward(end)?;
2611 self.consumed = range.end;
2612 debug_assert!(old_scan_pos <= self.lexer.scan_pos());
2613 debug_assert!(old_consumed <= self.consumed);
2614 debug_assert!(self.consumed <= self.lexer.scan_pos());
2615 Some(range)
2616 }
2617
2618 #[inline]
2621 pub(crate) fn consumed_pos(&self) -> Pos {
2622 self.consumed
2623 }
2624
2625 #[inline]
2626 pub(crate) fn lexer(&self) -> &Lexer<'s, V> {
2627 self.lexer
2628 }
2629
2630 #[inline]
2631 pub(crate) fn lexer_mut(&mut self) -> &mut Lexer<'s, V> {
2632 self.lexer
2633 }
2634
2635 #[inline]
2636 pub(crate) fn source_end(&self) -> Pos {
2637 self.lexer.source_end()
2638 }
2639
2640 #[inline(always)]
2641 pub(crate) fn byte_at(&self, position: Pos) -> Option<u8> {
2642 self.lexer.byte_at(position)
2643 }
2644
2645 #[inline]
2646 pub(crate) fn slice(&self, start: Pos, end: Pos) -> Option<&'s str> {
2647 self.lexer.slice(start, end)
2648 }
2649
2650 #[inline(always)]
2651 pub(crate) fn slice_trusted(&self, start: Pos, end: Pos) -> &'s str {
2652 self.lexer.slice_trusted(start, end)
2653 }
2654
2655 #[inline(always)]
2656 fn read_significant(&mut self, keep_comments: bool) -> BufferedToken {
2657 let start = self.lexer.scan_pos();
2658 let mut end = start;
2659 let mut first_comment_start = None;
2660 let mut has_white_space = false;
2661 loop {
2662 let token = self.lexer.next_token_without_visit();
2663 debug_assert!(token.range.start >= end);
2664 if token.kind == TokenKind::WhiteSpace {
2665 has_white_space = true;
2666 end = token.range.end;
2667 continue;
2668 }
2669 if !keep_comments && matches!(token.kind, TokenKind::Comment | TokenKind::BadComment) {
2670 first_comment_start.get_or_insert(token.range.start);
2671 end = token.range.end;
2672 continue;
2673 }
2674 let leading = Trivia {
2675 range: Range::new(start, end),
2676 end,
2677 first_comment_start,
2678 has_white_space,
2679 };
2680 return BufferedToken::new(TokenWithTrivia { token, leading }, self.lexer.function_kind);
2681 }
2682 }
2683}
2684
2685pub fn is_new_line(c: u8) -> bool {
2686 c == C_LINE_FEED || c == C_CARRIAGE_RETURN || c == C_FORM_FEED
2687}
2688
2689#[inline]
2690fn kind_to_right(byte: u8) -> TokenKind {
2691 match byte {
2692 C_RIGHT_PARENTHESIS => TokenKind::RightParenthesis,
2693 C_RIGHT_SQUARE => TokenKind::RightSquareBracket,
2694 C_RIGHT_CURLY => TokenKind::RightCurlyBracket,
2695 _ => unreachable!("fast_forward only sees closing bracket bytes"),
2696 }
2697}
2698
2699pub fn is_space(c: u8) -> bool {
2700 c == C_TAB || c == C_SPACE
2701}
2702
2703pub fn is_white_space(c: u8) -> bool {
2704 is_new_line(c) || is_space(c)
2705}
2706
2707pub fn is_digit(c: u8) -> bool {
2708 c.is_ascii_digit()
2709}
2710
2711#[inline]
2712fn is_non_flat_value_byte(byte: u8) -> bool {
2713 matches!(
2714 byte,
2715 C_QUOTATION_MARK
2716 | C_APOSTROPHE
2717 | C_SOLIDUS
2718 | C_REVERSE_SOLIDUS
2719 | C_LEFT_PARENTHESIS
2720 | C_RIGHT_PARENTHESIS
2721 | C_LEFT_SQUARE
2722 | C_RIGHT_SQUARE
2723 )
2724}
2725
2726#[inline]
2727fn is_name_start_byte(c: u8) -> bool {
2728 c == 0 || c == C_LOW_LINE || c.is_ascii_alphabetic() || !c.is_ascii()
2729}
2730
2731#[inline]
2732fn is_non_printable(c: u8) -> bool {
2733 matches!(c, 0x00..=0x08 | 0x0B | 0x0E..=0x1F | 0x7F)
2734}
2735
2736#[cfg(test)]
2737#[path = "../tests/lexer_tests.rs"]
2738mod tests;