Skip to main content

css_module_lexer/
lexer.rs

1use std::{collections::VecDeque, str};
2
3use bitflags::bitflags;
4use memchr::{memchr2, memchr3};
5use rustc_hash::FxHashSet;
6use smallvec::SmallVec;
7
8use crate::{
9  Range,
10  css_syntax::{
11    MAX_CSS_KEYWORD_LEN, decode_css_keyword, is_css_modules_pure_magic_comment,
12    lowercase_ascii_keyword, strip_vendor_prefix,
13  },
14  dependencies::{PropertyKind, special_value_is_candidate},
15  parser::{
16    BlockItemCandidates, BlockItemKind, BlockItemProgress, ComponentValues, RecoveryBoundary,
17  },
18};
19
20const fn build_plain_ascii_name_byte_table() -> [bool; 256] {
21  let mut table = [false; 256];
22  let mut byte = 0usize;
23  while byte < table.len() {
24    table[byte] = (byte >= b'0' as usize && byte <= b'9' as usize)
25      || (byte >= b'A' as usize && byte <= b'Z' as usize)
26      || (byte >= b'a' as usize && byte <= b'z' as usize)
27      || byte == b'_' as usize
28      || byte == b'-' as usize;
29    byte += 1;
30  }
31  table
32}
33
34const PLAIN_ASCII_NAME_BYTE: [bool; 256] = build_plain_ascii_name_byte_table();
35
36pub const C_LINE_FEED: u8 = b'\n';
37pub const C_CARRIAGE_RETURN: u8 = b'\r';
38pub const C_FORM_FEED: u8 = b'\x0c';
39
40pub const C_TAB: u8 = b'\t';
41pub const C_SPACE: u8 = b' ';
42
43pub const C_SOLIDUS: u8 = b'/';
44pub const C_REVERSE_SOLIDUS: u8 = b'\\';
45pub const C_ASTERISK: u8 = b'*';
46
47pub const C_LEFT_PARENTHESIS: u8 = b'(';
48pub const C_RIGHT_PARENTHESIS: u8 = b')';
49pub const C_LEFT_CURLY: u8 = b'{';
50pub const C_RIGHT_CURLY: u8 = b'}';
51pub const C_LEFT_SQUARE: u8 = b'[';
52pub const C_RIGHT_SQUARE: u8 = b']';
53
54pub const C_QUOTATION_MARK: u8 = b'"';
55pub const C_APOSTROPHE: u8 = b'\'';
56
57pub const C_FULL_STOP: u8 = b'.';
58pub const C_COLON: u8 = b':';
59pub const C_SEMICOLON: u8 = b';';
60pub const C_COMMA: u8 = b',';
61pub const C_PERCENTAGE: u8 = b'%';
62pub const C_AT_SIGN: u8 = b'@';
63
64pub const C_LOW_LINE: u8 = b'_';
65pub const C_LOWER_E: u8 = b'e';
66pub const C_UPPER_E: u8 = b'E';
67
68pub const C_NUMBER_SIGN: u8 = b'#';
69pub const C_PLUS_SIGN: u8 = b'+';
70pub const C_HYPHEN_MINUS: u8 = b'-';
71
72pub type Pos = u32;
73
74/// The lexical kind of a CSS token.
75///
76/// Tokens retain only byte ranges into the original source.  Comments are
77/// emitted deliberately so that CSS Modules magic comments can be interpreted
78/// without a second scan of the input.
79#[derive(Debug, Clone, Copy, PartialEq, Eq)]
80pub enum TokenKind {
81  Eof,
82  Ident,
83  AtKeyword,
84  Hash,
85  IdHash,
86  QuotedString,
87  Url,
88  Function,
89  Number,
90  Percentage,
91  Dimension,
92  WhiteSpace,
93  Comment,
94  BadComment,
95  BadString,
96  BadUrl,
97  Delim,
98  Colon,
99  Semicolon,
100  Comma,
101  LeftParenthesis,
102  RightParenthesis,
103  LeftSquareBracket,
104  RightSquareBracket,
105  LeftCurlyBracket,
106  RightCurlyBracket,
107  IncludeMatch,
108  DashMatch,
109  PrefixMatch,
110  SuffixMatch,
111  SubstringMatch,
112}
113
114impl TokenKind {
115  #[inline]
116  pub fn is_trivia(self) -> bool {
117    matches!(self, Self::WhiteSpace | Self::Comment)
118  }
119
120  #[inline]
121  pub fn is_scan_error(self) -> bool {
122    matches!(self, Self::BadComment | Self::BadString | Self::BadUrl)
123  }
124}
125
126/// A CSS token represented by ranges into the input string.
127#[derive(Debug, Clone, Copy, PartialEq, Eq)]
128pub struct Token {
129  pub kind: TokenKind,
130  pub range: Range,
131  pub value_range: Range,
132  pub flags: TokenFlags,
133}
134
135impl Token {
136  #[inline]
137  pub const fn new(kind: TokenKind, range: Range, value_range: Range) -> Self {
138    Self::with_flags(kind, range, value_range, TokenFlags::ascii())
139  }
140
141  #[inline]
142  pub const fn with_flags(
143    kind: TokenKind,
144    range: Range,
145    value_range: Range,
146    flags: TokenFlags,
147  ) -> Self {
148    Self {
149      kind,
150      range,
151      value_range,
152      flags,
153    }
154  }
155}
156
157bitflags! {
158    /// Properties collected while scanning a token. They let dependency parsing
159    /// skip escape/null checks for the overwhelmingly common plain-ASCII path.
160    #[derive(Debug, Clone, Copy, PartialEq, Eq)]
161    pub struct TokenFlags: u8 {
162        const HAS_ESCAPE = 1 << 0;
163        const HAS_NULL = 1 << 1;
164        const IS_ASCII = 1 << 2;
165    }
166}
167
168impl TokenFlags {
169  #[inline]
170  pub const fn ascii() -> Self {
171    Self::IS_ASCII
172  }
173
174  #[inline]
175  pub const fn has_escape(self) -> bool {
176    self.contains(Self::HAS_ESCAPE)
177  }
178
179  #[inline]
180  pub const fn has_null(self) -> bool {
181    self.contains(Self::HAS_NULL)
182  }
183
184  #[inline]
185  pub const fn is_ascii(self) -> bool {
186    self.contains(Self::IS_ASCII)
187  }
188
189  #[inline]
190  fn mark_escape(&mut self) {
191    self.insert(Self::HAS_ESCAPE);
192  }
193
194  #[inline]
195  fn mark_null(&mut self) {
196    self.insert(Self::HAS_NULL);
197  }
198
199  #[inline]
200  fn mark_non_ascii(&mut self) {
201    self.remove(Self::IS_ASCII);
202  }
203
204  #[inline]
205  fn merge(&mut self, other: Self) {
206    let is_ascii = self.is_ascii() && other.is_ascii();
207    self.insert(other);
208    if !is_ascii {
209      self.mark_non_ascii();
210    }
211  }
212}
213
214#[derive(Debug, Clone, Copy, PartialEq, Eq)]
215pub struct Trivia {
216  pub range: Range,
217  pub end: Pos,
218  pub first_comment_start: Option<Pos>,
219  pub has_white_space: bool,
220}
221
222impl Trivia {
223  #[inline]
224  pub fn has_whitespace(self) -> bool {
225    self.has_white_space
226  }
227}
228
229/// A significant token together with all immediately preceding whitespace and
230/// comments.  The leading trivia is consumed once and never rescanned.
231#[derive(Debug, Clone, Copy, PartialEq, Eq)]
232pub struct TokenWithTrivia {
233  pub token: Token,
234  pub leading: Trivia,
235}
236
237pub trait LexerVisitor {
238  fn visit_ident(&mut self, name: &str, range: Range);
239}
240
241impl LexerVisitor for () {
242  #[inline(always)]
243  fn visit_ident(&mut self, _name: &str, _range: Range) {}
244}
245
246#[derive(Debug)]
247pub struct Lexer<'s, V: LexerVisitor = ()> {
248  value: &'s [u8],
249  scan_pos: Pos,
250  visitor: V,
251  function_kind: FunctionKind,
252}
253
254impl<'s, V: LexerVisitor> Lexer<'s, V> {
255  pub fn new(value: &'s str, visitor: V) -> Self {
256    assert!(value.len() <= Pos::MAX as usize, "CSS input is too large");
257    Self {
258      value: value.as_bytes(),
259      scan_pos: 0,
260      visitor,
261      function_kind: FunctionKind::Other,
262    }
263  }
264
265  #[inline]
266  pub(crate) fn visitor_mut(&mut self) -> &mut V {
267    &mut self.visitor
268  }
269
270  #[inline]
271  fn visit_ident(&mut self, kind: TokenKind, range: Range) {
272    if !matches!(kind, TokenKind::Ident | TokenKind::Function) {
273      return;
274    }
275    let value = &self.value[range.start as usize..range.end as usize];
276    // SAFETY: token value ranges always lie on UTF-8 boundaries.
277    let value = unsafe { str::from_utf8_unchecked(value) };
278    self.visitor.visit_ident(value, range);
279  }
280
281  #[inline]
282  pub(crate) fn source_end(&self) -> Pos {
283    self.value.len() as Pos
284  }
285
286  #[inline(always)]
287  pub(crate) fn byte_at(&self, position: Pos) -> Option<u8> {
288    self.value.get(position as usize).copied()
289  }
290
291  #[inline(always)]
292  pub(crate) fn could_start_ident_at(&self, position: Pos) -> bool {
293    self.byte_at(position).is_some_and(|byte| {
294      matches!(byte, C_HYPHEN_MINUS | C_REVERSE_SOLIDUS) || is_name_start_byte(byte)
295    })
296  }
297
298  /// A flat run has no semantic value events. Check short runs in one pass;
299  /// use memchr for long runs, retaining the validated endpoint for the caller.
300  fn scan_flat_value(&self, start: usize) -> Option<usize> {
301    let rest = &self.value[start..];
302    let short = rest.len().min(32);
303    for (index, byte) in rest[..short].iter().copied().enumerate() {
304      if matches!(byte, C_LEFT_CURLY | C_SEMICOLON | C_RIGHT_CURLY) {
305        return (byte != C_LEFT_CURLY).then_some(start + index);
306      }
307      if is_non_flat_value_byte(byte) {
308        return None;
309      }
310    }
311    let boundary = memchr3(C_LEFT_CURLY, C_SEMICOLON, C_RIGHT_CURLY, &rest[short..])
312      .map_or(rest.len(), |index| short + index);
313    if rest.get(boundary) == Some(&C_LEFT_CURLY)
314      || rest[short..boundary]
315        .iter()
316        .copied()
317        .any(is_non_flat_value_byte)
318    {
319      return None;
320    }
321    Some(start + boundary)
322  }
323
324  /// Skip value text that cannot produce a dependency or change the block
325  /// structure. Delimiters of ordinary functions are tracked without
326  /// manufacturing tokens, and the scanner stops before dependency-bearing
327  /// functions, ICSS symbols, selector-independent magic comments, or a
328  /// declaration boundary.
329  fn fast_forward_generic_value<F, C>(
330    &mut self,
331    state: &mut GenericValueScanState,
332    options: GenericValueScanOptions,
333    is_candidate: F,
334    is_comment_candidate: C,
335    icss_symbols: Option<&FxHashSet<&str>>,
336    block_item: Option<BlockItemScan<'_>>,
337  ) -> Option<BufferedToken>
338  where
339    F: FnMut(&str) -> bool,
340    C: FnMut(&str) -> bool,
341  {
342    self.scan_value(
343      state,
344      options,
345      is_candidate,
346      is_comment_candidate,
347      icss_symbols,
348      block_item,
349    )
350  }
351
352  fn scan_value<F, C>(
353    &mut self,
354    state: &mut GenericValueScanState,
355    options: GenericValueScanOptions,
356    mut is_candidate: F,
357    mut is_comment_candidate: C,
358    icss_symbols: Option<&FxHashSet<&str>>,
359    block_item: Option<BlockItemScan<'_>>,
360  ) -> Option<BufferedToken>
361  where
362    F: FnMut(&str) -> bool,
363    C: FnMut(&str) -> bool,
364  {
365    let mut block_item = block_item;
366    let mut position = self.scan_pos as usize;
367    let scan_start = position;
368    let mut candidate_token = None;
369    if block_item
370      .as_ref()
371      .is_some_and(|candidate| !candidate.can_scan_value())
372    {
373      return None;
374    }
375    // Each suffix gets at most one flat-value proof. A failed prefix proof
376    // from prepare_declaration is carried here, rather than being repeated.
377    if options.allow_flat_scan
378      && !options.keep_comments
379      && !options.preserve_strings
380      && !options.preserve_delimiters
381      && !value_is_nested(state, &block_item)
382      && let Some(candidate) = block_item.as_mut()
383      && candidate.flat_probe_start != Some(position as Pos)
384    {
385      candidate.flat_probe_start = Some(position as Pos);
386      if let Some(end) = self.scan_flat_value(position)
387        && (!candidate.collect_dashed
388          || !self.value[position..end]
389            .windows(2)
390            .any(|bytes| bytes == b"--"))
391      {
392        if end > position {
393          note_opaque_value(&mut block_item);
394        }
395        self.scan_pos = end as Pos;
396        return None;
397      }
398    }
399    let preserve_selector = block_item
400      .as_ref()
401      .is_some_and(|candidate| candidate.preserve_selector_tokens());
402    let defer_rule_events =
403      preserve_selector && options.property.is_none() && !options.preserve_delimiters;
404    let preserve_delimiters =
405      options.preserve_delimiters || (preserve_selector && !defer_rule_events);
406    let validating = block_item.is_some();
407    let mut trivia_start = position;
408    let mut trivia_comment = None;
409    let mut trivia_whitespace = false;
410    while position < self.value.len() {
411      let byte = self.value[position];
412      if is_white_space(byte) {
413        trivia_whitespace = true;
414        position += 1;
415        continue;
416      }
417
418      if byte == C_SOLIDUS && self.value.get(position + 1) == Some(&C_ASTERISK) {
419        let (kind, end, _, _, _) = self.scan_comment(position);
420        if options.keep_comments && kind == TokenKind::Comment {
421          let content = &self.value[position + 2..end.saturating_sub(2)];
422          let mut first = 0usize;
423          while content.get(first).is_some_and(|byte| is_white_space(*byte)) {
424            first += 1;
425          }
426          if content.get(first) == Some(&b'c') {
427            let content = unsafe { str::from_utf8_unchecked(content) };
428            if is_comment_candidate(content) {
429              break;
430            }
431          }
432        }
433        trivia_comment.get_or_insert(position as Pos);
434        position = end;
435        continue;
436      }
437
438      let leading = Trivia {
439        range: Range::new(trivia_start as Pos, position as Pos),
440        end: position as Pos,
441        first_comment_start: trivia_comment,
442        has_white_space: trivia_whitespace,
443      };
444      if matches!(byte, C_QUOTATION_MARK | C_APOSTROPHE) {
445        if options.preserve_strings {
446          break;
447        }
448        let (kind, end, _, _, _) = self.scan_string(position, byte);
449        if kind == TokenKind::BadString && block_item.is_some() {
450          break;
451        }
452
453        note_opaque_value(&mut block_item);
454        position = end;
455        trivia_start = position;
456        trivia_comment = None;
457        trivia_whitespace = false;
458        continue;
459      }
460
461      let starts_ident = match byte {
462        C_HYPHEN_MINUS | C_REVERSE_SOLIDUS => self.starts_ident_at(position),
463        _ => is_name_start_byte(byte),
464      };
465      if starts_ident {
466        let end = self.scan_plain_ascii_name(position);
467        if end == position || self.raw_name_needs_tokenizer(end) {
468          break;
469        }
470        let flags = TokenFlags::ascii();
471        let name = unsafe { str::from_utf8_unchecked(&self.value[position..end]) };
472        let is_function = self.value.get(end) == Some(&C_LEFT_PARENTHESIS);
473        let function = if is_function {
474          FunctionKind::classify(name, flags)
475        } else {
476          FunctionKind::Other
477        };
478        let is_candidate = !flags.has_escape()
479          && (is_candidate(name)
480            || options
481              .property
482              .is_some_and(|property| special_value_is_candidate(property, name, icss_symbols)));
483        if block_item.as_ref().is_some_and(|candidate| {
484          candidate.preserve_selector_tokens()
485            && ((is_function && !defer_rule_events)
486              || name.starts_with("--")
487              || (!is_function && candidate.preserve_ident()))
488        }) || flags.has_escape()
489          || is_candidate
490          || (is_function && function.is_dependency())
491          || (is_function && preserve_delimiters)
492        {
493          let is_url = function == FunctionKind::Url;
494          if !is_url {
495            let token_end = if is_function { end + 1 } else { end };
496            candidate_token = Some(BufferedToken::new(
497              TokenWithTrivia {
498                token: Token::with_flags(
499                  if is_function {
500                    TokenKind::Function
501                  } else {
502                    TokenKind::Ident
503                  },
504                  Range::new(position as Pos, token_end as Pos),
505                  Range::new(position as Pos, end as Pos),
506                  flags,
507                ),
508                leading,
509              },
510              function,
511            ));
512            position = token_end;
513          }
514          break;
515        }
516        if is_function {
517          observe_opaque_value(
518            &mut block_item,
519            TokenKind::Function,
520            position,
521            end + 1,
522            self.value,
523            leading,
524            function,
525          );
526        } else {
527          note_opaque_value(&mut block_item);
528        }
529        self
530          .visitor
531          .visit_ident(name, Range::new(position as Pos, end as Pos));
532        if is_function {
533          if !validating {
534            state.consume(TokenKind::LeftParenthesis);
535          }
536          position = end + 1;
537        } else {
538          position = end;
539        }
540        trivia_start = position;
541        trivia_comment = None;
542        trivia_whitespace = false;
543        continue;
544      }
545
546      let starts_number = match byte {
547        b'0'..=b'9' => true,
548        C_PLUS_SIGN | C_HYPHEN_MINUS | C_FULL_STOP => self.starts_number_at(position),
549        _ => false,
550      };
551      if starts_number {
552        let Some(end) = self.scan_raw_numeric_end(position) else {
553          break;
554        };
555        note_opaque_value(&mut block_item);
556        position = end;
557        trivia_start = position;
558        trivia_comment = None;
559        trivia_whitespace = false;
560        continue;
561      }
562
563      if preserve_selector && matches!(byte, C_NUMBER_SIGN | C_FULL_STOP | C_COLON | C_AT_SIGN) {
564        break;
565      }
566      if validating
567        && matches!(byte, C_LEFT_CURLY | C_RIGHT_PARENTHESIS | C_RIGHT_SQUARE)
568        && !value_is_nested(state, &block_item)
569      {
570        break;
571      }
572      match byte {
573        C_COMMA if preserve_selector => {
574          // Selector arms need the comma when a declaration candidate is replayed.
575          observe_opaque_value(
576            &mut block_item,
577            TokenKind::Comma,
578            position,
579            position + 1,
580            self.value,
581            leading,
582            FunctionKind::Other,
583          );
584          position += 1;
585        }
586        C_NUMBER_SIGN => {
587          note_opaque_value(&mut block_item);
588          let name_start = position + 1;
589          if self.starts_ident_at(name_start)
590            || self
591              .value
592              .get(name_start)
593              .is_some_and(|byte| is_digit(*byte) || *byte == C_HYPHEN_MINUS)
594          {
595            let end = self.scan_plain_ascii_name(name_start);
596            if end == name_start || self.raw_name_needs_tokenizer(end) {
597              break;
598            }
599            position = end;
600          } else {
601            position = name_start;
602          }
603        }
604        C_LEFT_PARENTHESIS => {
605          if preserve_delimiters {
606            break;
607          }
608          observe_opaque_value(
609            &mut block_item,
610            TokenKind::LeftParenthesis,
611            position,
612            position + 1,
613            self.value,
614            leading,
615            FunctionKind::Other,
616          );
617          if !validating {
618            state.consume(TokenKind::LeftParenthesis);
619          }
620          position += 1;
621        }
622        C_RIGHT_PARENTHESIS => {
623          if preserve_delimiters {
624            break;
625          }
626          observe_opaque_value(
627            &mut block_item,
628            TokenKind::RightParenthesis,
629            position,
630            position + 1,
631            self.value,
632            leading,
633            FunctionKind::Other,
634          );
635          if !validating {
636            state.consume(TokenKind::RightParenthesis);
637          }
638          position += 1;
639        }
640        C_LEFT_SQUARE => {
641          if preserve_delimiters {
642            break;
643          }
644          observe_opaque_value(
645            &mut block_item,
646            TokenKind::LeftSquareBracket,
647            position,
648            position + 1,
649            self.value,
650            leading,
651            FunctionKind::Other,
652          );
653          if !validating {
654            state.consume(TokenKind::LeftSquareBracket);
655          }
656          position += 1;
657        }
658        C_RIGHT_SQUARE => {
659          if preserve_delimiters {
660            break;
661          }
662          observe_opaque_value(
663            &mut block_item,
664            TokenKind::RightSquareBracket,
665            position,
666            position + 1,
667            self.value,
668            leading,
669            FunctionKind::Other,
670          );
671          if !validating {
672            state.consume(TokenKind::RightSquareBracket);
673          }
674          position += 1;
675        }
676        C_LEFT_CURLY
677          if options.stop_at_top_level_left_curly && !value_is_nested(state, &block_item) =>
678        {
679          break;
680        }
681        C_LEFT_CURLY => {
682          if preserve_delimiters {
683            break;
684          }
685          observe_opaque_value(
686            &mut block_item,
687            TokenKind::LeftCurlyBracket,
688            position,
689            position + 1,
690            self.value,
691            leading,
692            FunctionKind::Other,
693          );
694          if !validating {
695            state.consume(TokenKind::LeftCurlyBracket);
696          }
697          position += 1;
698        }
699        C_RIGHT_CURLY
700          if value_expects(state, &block_item, TokenKind::RightCurlyBracket)
701            && !preserve_delimiters =>
702        {
703          observe_opaque_value(
704            &mut block_item,
705            TokenKind::RightCurlyBracket,
706            position,
707            position + 1,
708            self.value,
709            leading,
710            FunctionKind::Other,
711          );
712          if !validating {
713            state.consume(TokenKind::RightCurlyBracket);
714          }
715          position += 1;
716          if block_item
717            .as_ref()
718            .is_some_and(|candidate| !candidate.can_scan_value())
719          {
720            break;
721          }
722        }
723        C_RIGHT_CURLY if preserve_delimiters || !value_is_nested(state, &block_item) => {
724          break;
725        }
726        C_RIGHT_CURLY => position += 1,
727        C_SEMICOLON
728          if RecoveryBoundary::at(TokenKind::Semicolon, value_is_nested(state, &block_item))
729            .is_some() =>
730        {
731          break;
732        }
733        _ => {
734          note_opaque_value(&mut block_item);
735          position += 1;
736        }
737      }
738      trivia_start = position;
739      trivia_comment = None;
740      trivia_whitespace = false;
741    }
742    // A surviving selector uses this trivia for :local/:global whitespace
743    // validation. It has not been tokenized, so leave it to the next token.
744    if preserve_selector && candidate_token.is_none() && position < self.value.len() {
745      position = trivia_start;
746    }
747    self.scan_pos = position as Pos;
748    debug_assert!(
749      self.scan_pos >= scan_start as Pos,
750      "fast_forward moved scan_pos backward from {scan_start} to {position}"
751    );
752    candidate_token
753  }
754
755  /// Skip selector text that cannot produce a CSS Modules dependency.
756  /// Attribute selectors remain opaque across calls through `square_depth`;
757  /// the scanner stops before dependency candidates and structural tokens so
758  /// the normal token stream remains the sole owner of parser state changes.
759  pub(crate) fn fast_forward_selector<F, C>(
760    &mut self,
761    square_depth: &mut u32,
762    keep_comments: bool,
763    has_mode: bool,
764    mut is_ident_candidate: F,
765    mut is_comment_candidate: C,
766  ) -> bool
767  where
768    F: FnMut(&str) -> bool,
769    C: FnMut(&str) -> bool,
770  {
771    let mut position = self.scan_pos as usize;
772    let scan_start = position;
773    let mut invalidates_composes = false;
774    let mut trivia_start = None;
775
776    while position < self.value.len() {
777      let byte = self.value[position];
778
779      if is_white_space(byte) {
780        trivia_start.get_or_insert(position);
781        position += 1;
782        continue;
783      }
784
785      if byte == C_SOLIDUS && self.value.get(position + 1) == Some(&C_ASTERISK) {
786        let (kind, end, _, _, _) = self.scan_comment(position);
787        if keep_comments && kind == TokenKind::Comment {
788          let content = &self.value[position + 2..end.saturating_sub(2)];
789          // SAFETY: comments are slices of the original UTF-8 input.
790          let content = unsafe { str::from_utf8_unchecked(content) };
791          if is_comment_candidate(content) {
792            break;
793          }
794        }
795        trivia_start.get_or_insert(position);
796        position = end;
797        continue;
798      }
799
800      if *square_depth > 0 {
801        if byte == C_REVERSE_SOLIDUS && self.is_valid_escape_at(position) {
802          position = self.scan_escape(position);
803          trivia_start = None;
804          continue;
805        }
806        match byte {
807          C_QUOTATION_MARK | C_APOSTROPHE => {
808            position = self.scan_string(position, byte).1;
809          }
810          C_LEFT_SQUARE => {
811            *square_depth += 1;
812            position += 1;
813          }
814          C_RIGHT_SQUARE => {
815            *square_depth -= 1;
816            position += 1;
817          }
818          _ => position += 1,
819        }
820        trivia_start = None;
821        continue;
822      }
823
824      let starts_ident = match byte {
825        C_HYPHEN_MINUS | C_REVERSE_SOLIDUS => self.starts_ident_at(position),
826        _ => is_name_start_byte(byte),
827      };
828      if starts_ident {
829        let end = self.scan_name(position);
830        if end == position {
831          break;
832        }
833        // Functions own balanced-stack state in the dependency
834        // scanner. In particular, `url()` and `image-set()` can
835        // produce dependencies even when invalid CSS made them
836        // appear while recovering from a selector. Leave the name
837        // to the regular tokenizer so it can emit `Function`.
838        if self.value.get(end) == Some(&C_LEFT_PARENTHESIS) {
839          break;
840        }
841        let name = &self.value[position..end];
842        // SAFETY: scanned names start and end on UTF-8 boundaries.
843        let name = unsafe { str::from_utf8_unchecked(name) };
844        if is_ident_candidate(name) {
845          break;
846        }
847        self
848          .visitor
849          .visit_ident(name, Range::new(position as Pos, end as Pos));
850        invalidates_composes = true;
851        trivia_start = None;
852        position = end;
853        continue;
854      }
855
856      let starts_number = match byte {
857        b'0'..=b'9' => true,
858        C_PLUS_SIGN | C_HYPHEN_MINUS | C_FULL_STOP => self.starts_number_at(position),
859        _ => false,
860      };
861      if starts_number {
862        invalidates_composes = true;
863        trivia_start = None;
864        position = self.scan_numeric(position).1;
865        continue;
866      }
867
868      match byte {
869        C_LEFT_SQUARE => {
870          invalidates_composes = true;
871          *square_depth = 1;
872          trivia_start = None;
873          position += 1;
874        }
875        // Outside an attribute selector a string may belong to
876        // `url()` or `image-set()`, whose balanced-stack kind decides
877        // whether it creates a dependency.
878        C_QUOTATION_MARK | C_APOSTROPHE => break,
879        C_FULL_STOP | C_NUMBER_SIGN | C_COLON if has_mode => break,
880        C_NUMBER_SIGN => {
881          let name_start = position + 1;
882          position = if self.starts_ident_at(name_start)
883            || self
884              .value
885              .get(name_start)
886              .is_some_and(|byte| is_digit(*byte) || *byte == C_HYPHEN_MINUS)
887          {
888            self.scan_name(name_start)
889          } else {
890            name_start
891          };
892          invalidates_composes = true;
893          trivia_start = None;
894        }
895        C_RIGHT_PARENTHESIS => {
896          position = trivia_start.unwrap_or(position);
897          break;
898        }
899        C_LEFT_PARENTHESIS | C_LEFT_CURLY | C_RIGHT_CURLY | C_COMMA | C_SEMICOLON | C_AT_SIGN => {
900          break;
901        }
902        _ => {
903          invalidates_composes = true;
904          trivia_start = None;
905          position += 1;
906        }
907      }
908    }
909
910    self.scan_pos = position as Pos;
911    debug_assert!(
912      self.scan_pos >= scan_start as Pos,
913      "selector fast-forward moved scan_pos backward from {scan_start} to {position}"
914    );
915    invalidates_composes
916  }
917
918  /// Skip a balanced delimiter run whose opening token has already been
919  /// consumed. Only `RightParenthesis`, `RightSquareBracket`, and
920  /// `RightCurlyBracket` are accepted as `end`.
921  ///
922  /// Returns the range from the opening token through the matching closing
923  /// token on success. The caller is responsible for passing the position
924  /// just past the opening token.
925  ///
926  /// Nested delimiters are tracked on a local stack. Strings, comments,
927  /// escapes, and non-ASCII code points are skipped without tokenizing so
928  /// their content can never close the run. The scan is transactional: on
929  /// failure (EOF or an unbalanced closing token) `scan_pos` is left
930  /// unchanged and `None` is returned so the caller can fall back to the
931  /// regular tokenizer, which remains the sole owner of parser state
932  /// changes.
933  pub(crate) fn fast_forward(&mut self, end: TokenKind) -> Option<Range> {
934    match end {
935      TokenKind::RightParenthesis
936      | TokenKind::RightSquareBracket
937      | TokenKind::RightCurlyBracket => {}
938      _ => unreachable!("fast_forward accepts only closing bracket kinds"),
939    }
940    let open_start = self.scan_pos;
941    let mut stack: SmallVec<[TokenKind; 8]> = SmallVec::new();
942    stack.push(end);
943    let mut position = open_start as usize;
944    let bytes = self.value;
945    while position < bytes.len() {
946      let byte = bytes[position];
947      match byte {
948        C_QUOTATION_MARK | C_APOSTROPHE => {
949          let (kind, string_end, _, _, _) = self.scan_string(position, byte);
950          if kind == TokenKind::BadString {
951            return None;
952          }
953          position = string_end;
954        }
955        C_SOLIDUS if bytes.get(position + 1) == Some(&C_ASTERISK) => {
956          let (kind, comment_end, _, _, _) = self.scan_comment(position);
957          if kind == TokenKind::BadComment {
958            return None;
959          }
960          position = comment_end;
961        }
962        C_REVERSE_SOLIDUS if self.is_valid_escape_at(position) => {
963          position = self.scan_escape(position);
964        }
965        C_LEFT_PARENTHESIS => {
966          stack.push(TokenKind::RightParenthesis);
967          position += 1;
968        }
969        C_LEFT_SQUARE => {
970          stack.push(TokenKind::RightSquareBracket);
971          position += 1;
972        }
973        C_LEFT_CURLY => {
974          stack.push(TokenKind::RightCurlyBracket);
975          position += 1;
976        }
977        C_RIGHT_PARENTHESIS | C_RIGHT_SQUARE | C_RIGHT_CURLY => {
978          if *stack.last().expect("delimiter stack never empties") != kind_to_right(byte) {
979            return None;
980          }
981          stack.pop();
982          position += 1;
983          if stack.is_empty() {
984            self.scan_pos = position as Pos;
985            debug_assert!(
986              self.scan_pos >= open_start,
987              "fast_forward moved scan_pos backward from {open_start} to {position}"
988            );
989            return Some(Range::new(open_start, position as Pos));
990          }
991        }
992        // A semicolon ends an at-rule prelude regardless of nesting
993        // depth. Bail so the regular tokenizer resumes and the
994        // dependency scanner completes the at-rule at the semicolon.
995        C_SEMICOLON => return None,
996        _ if byte >= 0x80 => {
997          position += self.utf8_width_at(position);
998        }
999        _ => position += 1,
1000      }
1001    }
1002    None
1003  }
1004
1005  pub(crate) fn slice(&self, start: Pos, end: Pos) -> Option<&'s str> {
1006    let range = Range::new(start, end);
1007    let start = start as usize;
1008    let end = end as usize;
1009    if start > end || end > self.value.len() {
1010      return None;
1011    }
1012    // SAFETY: Lexer-generated positions always delimit complete tokens or
1013    // source fragments and therefore stay on UTF-8 code point boundaries.
1014    Some(unsafe { slice_unchecked(self.value, &range) })
1015  }
1016
1017  #[inline(always)]
1018  pub(crate) fn slice_trusted(&self, start: Pos, end: Pos) -> &'s str {
1019    debug_assert!(start <= end && end <= self.value.len() as Pos);
1020    // SAFETY: Token ranges and parser positions originate from this
1021    // lexer, so they delimit valid UTF-8 source boundaries.
1022    unsafe { slice_unchecked(self.value, &Range::new(start, end)) }
1023  }
1024}
1025
1026impl Lexer<'_, ()> {
1027  pub fn slice_range<'a>(input: &'a str, range: &Range) -> Option<&'a str> {
1028    let start = range.start as usize;
1029    let end = range.end as usize;
1030    if start > end
1031      || end > input.len()
1032      || !input.is_char_boundary(start)
1033      || !input.is_char_boundary(end)
1034    {
1035      return None;
1036    }
1037    // SAFETY: The range was checked against the original `str` above.
1038    Some(unsafe { slice_unchecked(input.as_bytes(), range) })
1039  }
1040}
1041
1042unsafe fn slice_unchecked<'a>(input: &'a [u8], range: &Range) -> &'a str {
1043  unsafe {
1044    let value = input.get_unchecked(range.start as usize..range.end as usize);
1045    str::from_utf8_unchecked(value)
1046  }
1047}
1048
1049impl<V: LexerVisitor> Lexer<'_, V> {
1050  #[inline]
1051  pub(crate) fn scan_pos(&self) -> Pos {
1052    self.scan_pos
1053  }
1054}
1055
1056impl<'s, V: LexerVisitor> Lexer<'s, V> {
1057  /// Return the next CSS token without allocating or decoding its value.
1058  ///
1059  /// `TokenKind::Eof` is returned after the input is exhausted. Unterminated
1060  /// comments, strings, and URLs are represented by the corresponding
1061  /// `Bad*` token kinds, so EOF and tokenizer errors are never conflated.
1062  #[inline]
1063  pub fn next_token(&mut self) -> Token {
1064    let token = self.next_token_without_visit();
1065    self.visit_ident(token.kind, token.value_range);
1066    token
1067  }
1068
1069  fn next_token_without_visit(&mut self) -> Token {
1070    self.function_kind = FunctionKind::Other;
1071    let start = self.scan_pos as usize;
1072    let len = self.value.len();
1073    if start >= len {
1074      self.scan_pos = len as Pos;
1075      return Token::new(
1076        TokenKind::Eof,
1077        Range::new(len as Pos, len as Pos),
1078        Range::new(len as Pos, len as Pos),
1079      );
1080    }
1081
1082    let byte = self.value[start];
1083    let (kind, end, value_start, value_end, flags) = match byte {
1084      C_SPACE | C_TAB | C_LINE_FEED | C_CARRIAGE_RETURN | C_FORM_FEED => {
1085        self.scan_whitespace(start)
1086      }
1087      C_QUOTATION_MARK => self.scan_string(start, C_QUOTATION_MARK),
1088      C_APOSTROPHE => self.scan_string(start, C_APOSTROPHE),
1089      C_NUMBER_SIGN => {
1090        let value_start = start + 1;
1091        if self.starts_ident_at(value_start) {
1092          let (end, flags) = self.scan_name_with_flags(value_start);
1093          (TokenKind::IdHash, end, value_start, end, flags)
1094        } else if self
1095          .value
1096          .get(value_start)
1097          .is_some_and(|byte| is_digit(*byte) || *byte == C_HYPHEN_MINUS)
1098        {
1099          let (end, flags) = self.scan_name_with_flags(value_start);
1100          (TokenKind::Hash, end, value_start, end, flags)
1101        } else {
1102          (
1103            TokenKind::Delim,
1104            start + 1,
1105            start,
1106            start + 1,
1107            TokenFlags::ascii(),
1108          )
1109        }
1110      }
1111      C_LEFT_PARENTHESIS => (
1112        TokenKind::LeftParenthesis,
1113        start + 1,
1114        start,
1115        start + 1,
1116        TokenFlags::ascii(),
1117      ),
1118      C_RIGHT_PARENTHESIS => (
1119        TokenKind::RightParenthesis,
1120        start + 1,
1121        start,
1122        start + 1,
1123        TokenFlags::ascii(),
1124      ),
1125      C_LEFT_SQUARE => (
1126        TokenKind::LeftSquareBracket,
1127        start + 1,
1128        start,
1129        start + 1,
1130        TokenFlags::ascii(),
1131      ),
1132      C_RIGHT_SQUARE => (
1133        TokenKind::RightSquareBracket,
1134        start + 1,
1135        start,
1136        start + 1,
1137        TokenFlags::ascii(),
1138      ),
1139      C_LEFT_CURLY => (
1140        TokenKind::LeftCurlyBracket,
1141        start + 1,
1142        start,
1143        start + 1,
1144        TokenFlags::ascii(),
1145      ),
1146      C_RIGHT_CURLY => (
1147        TokenKind::RightCurlyBracket,
1148        start + 1,
1149        start,
1150        start + 1,
1151        TokenFlags::ascii(),
1152      ),
1153      C_COLON => (
1154        TokenKind::Colon,
1155        start + 1,
1156        start,
1157        start + 1,
1158        TokenFlags::ascii(),
1159      ),
1160      C_SEMICOLON => (
1161        TokenKind::Semicolon,
1162        start + 1,
1163        start,
1164        start + 1,
1165        TokenFlags::ascii(),
1166      ),
1167      C_COMMA => (
1168        TokenKind::Comma,
1169        start + 1,
1170        start,
1171        start + 1,
1172        TokenFlags::ascii(),
1173      ),
1174      C_PLUS_SIGN => {
1175        if self.starts_number_at(start) {
1176          self.scan_numeric(start)
1177        } else {
1178          (
1179            TokenKind::Delim,
1180            start + 1,
1181            start,
1182            start + 1,
1183            TokenFlags::ascii(),
1184          )
1185        }
1186      }
1187      C_HYPHEN_MINUS => {
1188        if self.starts_number_at(start) {
1189          self.scan_numeric(start)
1190        } else if self.starts_ident_at(start) {
1191          self.scan_ident_like(start)
1192        } else {
1193          (
1194            TokenKind::Delim,
1195            start + 1,
1196            start,
1197            start + 1,
1198            TokenFlags::ascii(),
1199          )
1200        }
1201      }
1202      C_FULL_STOP => {
1203        if self.starts_number_at(start) {
1204          self.scan_numeric(start)
1205        } else {
1206          (
1207            TokenKind::Delim,
1208            start + 1,
1209            start,
1210            start + 1,
1211            TokenFlags::ascii(),
1212          )
1213        }
1214      }
1215      C_SOLIDUS => {
1216        if self.value.get(start + 1) == Some(&C_ASTERISK) {
1217          self.scan_comment(start)
1218        } else {
1219          (
1220            TokenKind::Delim,
1221            start + 1,
1222            start,
1223            start + 1,
1224            TokenFlags::ascii(),
1225          )
1226        }
1227      }
1228      C_AT_SIGN => {
1229        let value_start = start + 1;
1230        if self.starts_ident_at(value_start) {
1231          let (end, flags) = self.scan_name_with_flags(value_start);
1232          (TokenKind::AtKeyword, end, value_start, end, flags)
1233        } else {
1234          (
1235            TokenKind::Delim,
1236            start + 1,
1237            start,
1238            start + 1,
1239            TokenFlags::ascii(),
1240          )
1241        }
1242      }
1243      C_REVERSE_SOLIDUS => {
1244        if self.is_valid_escape_at(start) {
1245          self.scan_ident_like(start)
1246        } else {
1247          (
1248            TokenKind::Delim,
1249            start + 1,
1250            start,
1251            start + 1,
1252            TokenFlags::ascii(),
1253          )
1254        }
1255      }
1256      b'0'..=b'9' => self.scan_numeric(start),
1257      b'$' if self.value[start..].starts_with(b"$=") => (
1258        TokenKind::SuffixMatch,
1259        start + 2,
1260        start,
1261        start + 2,
1262        TokenFlags::ascii(),
1263      ),
1264      b'^' if self.value[start..].starts_with(b"^=") => (
1265        TokenKind::PrefixMatch,
1266        start + 2,
1267        start,
1268        start + 2,
1269        TokenFlags::ascii(),
1270      ),
1271      b'|' if self.value[start..].starts_with(b"|=") => (
1272        TokenKind::DashMatch,
1273        start + 2,
1274        start,
1275        start + 2,
1276        TokenFlags::ascii(),
1277      ),
1278      b'~' if self.value[start..].starts_with(b"~=") => (
1279        TokenKind::IncludeMatch,
1280        start + 2,
1281        start,
1282        start + 2,
1283        TokenFlags::ascii(),
1284      ),
1285      b'*' if self.value[start..].starts_with(b"*=") => (
1286        TokenKind::SubstringMatch,
1287        start + 2,
1288        start,
1289        start + 2,
1290        TokenFlags::ascii(),
1291      ),
1292      _ if self.starts_ident_at(start) => self.scan_ident_like(start),
1293      _ => (
1294        TokenKind::Delim,
1295        start + 1,
1296        start,
1297        start + 1,
1298        TokenFlags::ascii(),
1299      ),
1300    };
1301
1302    self.scan_pos = end as Pos;
1303    debug_assert!(
1304      self.scan_pos >= start as Pos,
1305      "scan_pos moved backward from {start} to {}",
1306      self.scan_pos
1307    );
1308    Token::with_flags(
1309      kind,
1310      Range::new(start as Pos, end as Pos),
1311      Range::new(value_start as Pos, value_end as Pos),
1312      flags,
1313    )
1314  }
1315
1316  #[inline]
1317  fn scan_whitespace(&self, start: usize) -> (TokenKind, usize, usize, usize, TokenFlags) {
1318    let bytes = self.value;
1319    let mut end = start;
1320    while end < bytes.len() && is_white_space(bytes[end]) {
1321      end += 1;
1322    }
1323    (TokenKind::WhiteSpace, end, start, end, TokenFlags::ascii())
1324  }
1325
1326  fn scan_comment(&self, start: usize) -> (TokenKind, usize, usize, usize, TokenFlags) {
1327    let bytes = self.value;
1328    let mut end = start + 2;
1329    let mut flags = TokenFlags::ascii();
1330    while end < bytes.len() {
1331      if bytes[end] == C_ASTERISK && bytes.get(end + 1) == Some(&C_SOLIDUS) {
1332        return (TokenKind::Comment, end + 2, start + 2, end, flags);
1333      }
1334      if bytes[end] == 0 {
1335        flags.mark_null();
1336      } else if !bytes[end].is_ascii() {
1337        flags.mark_non_ascii();
1338      }
1339      end += if bytes[end] < 0x80 {
1340        1
1341      } else {
1342        self.utf8_width_at(end)
1343      };
1344    }
1345    (
1346      TokenKind::BadComment,
1347      bytes.len(),
1348      start + 2,
1349      bytes.len(),
1350      flags,
1351    )
1352  }
1353
1354  fn scan_string(&self, start: usize, quote: u8) -> (TokenKind, usize, usize, usize, TokenFlags) {
1355    let mut end = start + 1;
1356    let mut flags = TokenFlags::ascii();
1357    loop {
1358      // Scan short runs with a bounded scalar loop. This keeps the common
1359      // short-string path branch-friendly, preserves the exact first error
1360      // position, and pays the bulk search cost only for long runs.
1361      let short_end = end.saturating_add(32).min(self.value.len());
1362      while end < short_end {
1363        let byte = self.value[end];
1364        if byte == quote {
1365          return (TokenKind::QuotedString, end + 1, start + 1, end, flags);
1366        }
1367        if is_new_line(byte) {
1368          return (TokenKind::BadString, end, start + 1, end, flags);
1369        }
1370        if byte == 0 {
1371          flags.mark_null();
1372        } else if !byte.is_ascii() {
1373          flags.mark_non_ascii();
1374        }
1375        if byte == C_REVERSE_SOLIDUS {
1376          flags.mark_escape();
1377          if self.value.get(end + 1).is_some_and(|next| !next.is_ascii()) {
1378            flags.mark_non_ascii();
1379          }
1380          if self
1381            .value
1382            .get(end + 1)
1383            .is_some_and(|next| is_new_line(*next))
1384          {
1385            end += 1;
1386            if self.value.get(end) == Some(&C_CARRIAGE_RETURN)
1387              && self.value.get(end + 1) == Some(&C_LINE_FEED)
1388            {
1389              end += 1;
1390            }
1391            end += 1;
1392          } else if self.is_valid_escape_at(end) {
1393            end = self.scan_escape(end);
1394          } else {
1395            end += 1;
1396          }
1397        } else {
1398          end += self.utf8_width_at(end);
1399        }
1400      }
1401      if end == self.value.len() {
1402        return (TokenKind::BadString, end, start + 1, end, flags);
1403      }
1404      // Bulk search: skip ordinary string content with a vectorized search
1405      // for the next quote, escape, or line feed.
1406      let rest = &self.value[end..];
1407      let end_rel = memchr3(quote, C_REVERSE_SOLIDUS, C_LINE_FEED, rest).unwrap_or(rest.len());
1408      // A carriage return or form feed ends the string before any later
1409      // quote, escape, or line feed, so report it at its exact position.
1410      let segment_end =
1411        memchr2(C_CARRIAGE_RETURN, C_FORM_FEED, &rest[..end_rel]).unwrap_or(end_rel);
1412      // Null and non-ASCII bytes inside the skipped run only affect token
1413      // flags; their exact positions do not matter. Only the bytes before a
1414      // terminating carriage return or form feed belong to the string.
1415      if memchr::memchr(b'\0', &rest[..segment_end]).is_some() {
1416        flags.mark_null();
1417      }
1418      if !rest[..segment_end].is_ascii() {
1419        flags.mark_non_ascii();
1420      }
1421      end += segment_end;
1422      if segment_end != end_rel {
1423        return (TokenKind::BadString, end, start + 1, end, flags);
1424      }
1425      // end now points at a quote, escape, line feed, or EOF. The scalar
1426      // loop handles it with the original per-byte logic.
1427    }
1428  }
1429
1430  fn scan_ident_like(&mut self, start: usize) -> (TokenKind, usize, usize, usize, TokenFlags) {
1431    let (name_end, name_flags) = self.scan_name_with_flags(start);
1432    if self.value.get(name_end) != Some(&C_LEFT_PARENTHESIS) {
1433      return (TokenKind::Ident, name_end, start, name_end, name_flags);
1434    }
1435
1436    let open_end = name_end + 1;
1437    let name = &self.value[start..name_end];
1438    // SAFETY: a scanned name starts and ends on UTF-8 boundaries.
1439    let name = unsafe { str::from_utf8_unchecked(name) };
1440    self.function_kind = FunctionKind::classify(name, name_flags);
1441    let is_url = self.function_kind == FunctionKind::Url;
1442    if is_url {
1443      let content_start = self.skip_white_space(open_end);
1444      if !matches!(
1445        self.value.get(content_start),
1446        Some(&C_QUOTATION_MARK | &C_APOSTROPHE)
1447      ) {
1448        let (kind, end, value_start, value_end, url_flags) = self.scan_url(start, content_start);
1449        let mut flags = name_flags;
1450        flags.merge(url_flags);
1451        return (kind, end, value_start, value_end, flags);
1452      }
1453    }
1454    (TokenKind::Function, open_end, start, name_end, name_flags)
1455  }
1456
1457  fn scan_url(
1458    &self,
1459    start: usize,
1460    content_start: usize,
1461  ) -> (TokenKind, usize, usize, usize, TokenFlags) {
1462    let mut end = content_start;
1463    let mut flags = TokenFlags::ascii();
1464    'scan: loop {
1465      // Scan short runs with a bounded scalar loop before falling back to
1466      // the bulk search, matching the short-string strategy in scan_string.
1467      let short_end = end.saturating_add(32).min(self.value.len());
1468      while end < short_end {
1469        let byte = self.value[end];
1470        if byte == C_RIGHT_PARENTHESIS {
1471          return (TokenKind::Url, end + 1, content_start, end, flags);
1472        }
1473        if is_white_space(byte) {
1474          let content_end = end;
1475          let close = self.skip_white_space(end);
1476          if self.value.get(close) == Some(&C_RIGHT_PARENTHESIS) {
1477            return (TokenKind::Url, close + 1, content_start, content_end, flags);
1478          }
1479          // CSS Syntax §4.3.6: EOF ends a url-token rather than making it bad.
1480          if close == self.value.len() {
1481            return (TokenKind::Url, close, content_start, content_end, flags);
1482          }
1483          return self.scan_bad_url(start, close, content_start, flags);
1484        }
1485        if byte == 0 {
1486          flags.mark_null();
1487        } else if !byte.is_ascii() {
1488          flags.mark_non_ascii();
1489        }
1490        if byte == C_QUOTATION_MARK
1491          || byte == C_APOSTROPHE
1492          || byte == C_LEFT_PARENTHESIS
1493          || is_non_printable(byte)
1494        {
1495          return self.scan_bad_url(start, end, content_start, flags);
1496        }
1497        if byte == C_REVERSE_SOLIDUS {
1498          flags.mark_escape();
1499          if self.value.get(end + 1).is_some_and(|next| !next.is_ascii()) {
1500            flags.mark_non_ascii();
1501          }
1502          if self.is_valid_escape_at(end) {
1503            end = self.scan_escape(end);
1504          } else if self
1505            .value
1506            .get(end + 1)
1507            .is_some_and(|next| is_new_line(*next))
1508          {
1509            end += 2;
1510            if self.value.get(end - 1) == Some(&C_CARRIAGE_RETURN)
1511              && self.value.get(end) == Some(&C_LINE_FEED)
1512            {
1513              end += 1;
1514            }
1515          } else {
1516            return self.scan_bad_url(start, end, content_start, flags);
1517          }
1518        } else {
1519          end += self.utf8_width_at(end);
1520        }
1521      }
1522      if end == self.value.len() {
1523        return (
1524          TokenKind::Url,
1525          self.value.len(),
1526          content_start,
1527          self.value.len(),
1528          flags,
1529        );
1530      }
1531      // Bulk search: the closing parenthesis is almost always the next byte
1532      // that needs attention, so find it with a vectorized search and then
1533      // verify the skipped run. Any byte before it that requires per-byte
1534      // handling (whitespace, quote, parenthesis, escape, or non-printable)
1535      // hands control back to the scalar loop at that exact position.
1536      let rest = &self.value[end..];
1537      let close_rel = memchr::memchr(C_RIGHT_PARENTHESIS, rest).unwrap_or(rest.len());
1538      let segment = &rest[..close_rel];
1539      if !segment.is_ascii() {
1540        flags.mark_non_ascii();
1541      }
1542      let bad_rel = segment.iter().position(|byte| {
1543        *byte <= C_SPACE
1544          || *byte == C_QUOTATION_MARK
1545          || *byte == C_APOSTROPHE
1546          || *byte == C_LEFT_PARENTHESIS
1547          || *byte == C_REVERSE_SOLIDUS
1548          || *byte == 0x7F
1549      });
1550      if let Some(offset) = bad_rel {
1551        end += offset;
1552        continue 'scan;
1553      }
1554      end += close_rel;
1555      if close_rel == rest.len() {
1556        return (
1557          TokenKind::Url,
1558          self.value.len(),
1559          content_start,
1560          self.value.len(),
1561          flags,
1562        );
1563      }
1564      return (TokenKind::Url, end + 1, content_start, end, flags);
1565    }
1566  }
1567
1568  fn scan_bad_url(
1569    &self,
1570    _start: usize,
1571    mut end: usize,
1572    content_start: usize,
1573    mut flags: TokenFlags,
1574  ) -> (TokenKind, usize, usize, usize, TokenFlags) {
1575    loop {
1576      let short_end = end.saturating_add(32).min(self.value.len());
1577      while end < short_end {
1578        let byte = self.value[end];
1579        if byte == 0 {
1580          flags.mark_null();
1581        } else if !byte.is_ascii() {
1582          flags.mark_non_ascii();
1583        }
1584        if byte == C_RIGHT_PARENTHESIS {
1585          end += 1;
1586          return (TokenKind::BadUrl, end, content_start, end, flags);
1587        }
1588        if byte == C_REVERSE_SOLIDUS {
1589          flags.mark_escape();
1590          if self.value.get(end + 1).is_some_and(|next| !next.is_ascii()) {
1591            flags.mark_non_ascii();
1592          }
1593          if self.is_valid_escape_at(end) {
1594            end = self.scan_escape(end);
1595          } else if self
1596            .value
1597            .get(end + 1)
1598            .is_some_and(|next| is_new_line(*next))
1599          {
1600            end += 2;
1601            if self.value.get(end - 1) == Some(&C_CARRIAGE_RETURN)
1602              && self.value.get(end) == Some(&C_LINE_FEED)
1603            {
1604              end += 1;
1605            }
1606          } else {
1607            end += 1;
1608          }
1609        } else {
1610          end += self.utf8_width_at(end);
1611        }
1612      }
1613      if end == self.value.len() {
1614        break;
1615      }
1616      // Bulk search: skip ordinary content with a vectorized search for the
1617      // next closing parenthesis or escape.
1618      let rest = &self.value[end..];
1619      let rel = memchr2(C_RIGHT_PARENTHESIS, C_REVERSE_SOLIDUS, rest).unwrap_or(rest.len());
1620      let segment = &rest[..rel];
1621      if memchr::memchr(b'\0', segment).is_some() {
1622        flags.mark_null();
1623      }
1624      if !segment.is_ascii() {
1625        flags.mark_non_ascii();
1626      }
1627      end += rel;
1628      // end now points at a closing parenthesis, escape, or EOF. The scalar
1629      // loop handles it with the original per-byte logic.
1630    }
1631    (TokenKind::BadUrl, end, content_start, end, flags)
1632  }
1633
1634  fn scan_numeric(&self, start: usize) -> (TokenKind, usize, usize, usize, TokenFlags) {
1635    let end = self.scan_number_end(start);
1636    if self.value.get(end) == Some(&C_PERCENTAGE) {
1637      return (
1638        TokenKind::Percentage,
1639        end + 1,
1640        start,
1641        end + 1,
1642        TokenFlags::ascii(),
1643      );
1644    }
1645    if self.starts_ident_at(end) {
1646      let (end, flags) = self.scan_name_with_flags(end);
1647      return (TokenKind::Dimension, end, start, end, flags);
1648    }
1649    (TokenKind::Number, end, start, end, TokenFlags::ascii())
1650  }
1651
1652  #[inline]
1653  fn scan_number_end(&self, start: usize) -> usize {
1654    let mut end = start;
1655    if matches!(self.value.get(end), Some(&C_PLUS_SIGN | &C_HYPHEN_MINUS)) {
1656      end += 1;
1657    }
1658    while self.value.get(end).is_some_and(|byte| is_digit(*byte)) {
1659      end += 1;
1660    }
1661    if self.value.get(end) == Some(&C_FULL_STOP)
1662      && self.value.get(end + 1).is_some_and(|byte| is_digit(*byte))
1663    {
1664      end += 1;
1665      while self.value.get(end).is_some_and(|byte| is_digit(*byte)) {
1666        end += 1;
1667      }
1668    }
1669    if matches!(self.value.get(end), Some(&C_LOWER_E | &C_UPPER_E)) {
1670      let exponent_start = end;
1671      let mut exponent_end = end + 1;
1672      if matches!(
1673        self.value.get(exponent_end),
1674        Some(&C_PLUS_SIGN | &C_HYPHEN_MINUS)
1675      ) {
1676        exponent_end += 1;
1677      }
1678      if self
1679        .value
1680        .get(exponent_end)
1681        .is_some_and(|byte| is_digit(*byte))
1682      {
1683        end = exponent_end + 1;
1684        while self.value.get(end).is_some_and(|byte| is_digit(*byte)) {
1685          end += 1;
1686        }
1687      } else {
1688        end = exponent_start;
1689      }
1690    }
1691    end
1692  }
1693
1694  #[inline]
1695  fn skip_white_space(&self, mut position: usize) -> usize {
1696    while self
1697      .value
1698      .get(position)
1699      .is_some_and(|byte| is_white_space(*byte))
1700    {
1701      position += 1;
1702    }
1703    position
1704  }
1705
1706  #[inline]
1707  fn scan_plain_ascii_name(&self, mut position: usize) -> usize {
1708    while self
1709      .value
1710      .get(position)
1711      .is_some_and(|byte| PLAIN_ASCII_NAME_BYTE[*byte as usize])
1712    {
1713      position += 1;
1714    }
1715    position
1716  }
1717
1718  #[inline]
1719  fn raw_name_needs_tokenizer(&self, position: usize) -> bool {
1720    self
1721      .value
1722      .get(position)
1723      .is_some_and(|byte| *byte == 0 || *byte == C_REVERSE_SOLIDUS || !byte.is_ascii())
1724  }
1725
1726  #[inline]
1727  fn scan_raw_numeric_end(&self, start: usize) -> Option<usize> {
1728    let end = self.scan_number_end(start);
1729    if self.value.get(end) == Some(&C_PERCENTAGE) {
1730      return Some(end + 1);
1731    }
1732    if self.starts_ident_at(end) {
1733      let name_end = self.scan_plain_ascii_name(end);
1734      if name_end == end || self.raw_name_needs_tokenizer(name_end) {
1735        return None;
1736      }
1737      return Some(name_end);
1738    }
1739    if self.raw_name_needs_tokenizer(end) {
1740      return None;
1741    }
1742    Some(end)
1743  }
1744
1745  #[inline]
1746  fn scan_name(&self, position: usize) -> usize {
1747    self.scan_name_impl(position, false).0
1748  }
1749
1750  #[inline]
1751  fn scan_name_with_flags(&self, position: usize) -> (usize, TokenFlags) {
1752    self.scan_name_impl(position, true)
1753  }
1754
1755  #[inline]
1756  fn scan_name_impl(&self, mut position: usize, track_flags: bool) -> (usize, TokenFlags) {
1757    let bytes = self.value;
1758    let mut flags = TokenFlags::ascii();
1759    while position < bytes.len() {
1760      while position < bytes.len() {
1761        let byte = bytes[position];
1762        if PLAIN_ASCII_NAME_BYTE[byte as usize] {
1763          position += 1;
1764        } else if byte == 0 {
1765          if track_flags {
1766            flags.mark_null();
1767          }
1768          position += 1;
1769        } else {
1770          break;
1771        }
1772      }
1773      if position == bytes.len() {
1774        break;
1775      }
1776
1777      let byte = bytes[position];
1778      if byte == C_REVERSE_SOLIDUS {
1779        if self.is_valid_escape_at(position) {
1780          if track_flags {
1781            flags.mark_escape();
1782            if bytes.get(position + 1).is_some_and(|next| !next.is_ascii()) {
1783              flags.mark_non_ascii();
1784            }
1785          }
1786          position = self.scan_escape(position);
1787        } else {
1788          break;
1789        }
1790      } else if byte.is_ascii() {
1791        break;
1792      } else {
1793        if track_flags {
1794          flags.mark_non_ascii();
1795        }
1796        position += self.utf8_width_at(position);
1797      }
1798    }
1799    (position, flags)
1800  }
1801
1802  #[inline]
1803  fn starts_ident_at(&self, position: usize) -> bool {
1804    let Some(byte) = self.value.get(position).copied() else {
1805      return false;
1806    };
1807    match byte {
1808      C_HYPHEN_MINUS => match self.value.get(position + 1).copied() {
1809        Some(next) => {
1810          is_name_start_byte(next)
1811            || next == C_HYPHEN_MINUS
1812            || self.is_valid_escape_at(position + 1)
1813        }
1814        None => false,
1815      },
1816      C_REVERSE_SOLIDUS => self.is_valid_escape_at(position),
1817      _ => is_name_start_byte(byte),
1818    }
1819  }
1820
1821  #[inline]
1822  fn starts_number_at(&self, position: usize) -> bool {
1823    let Some(first) = self.value.get(position).copied() else {
1824      return false;
1825    };
1826    let second = self.value.get(position + 1).copied();
1827    let third = self.value.get(position + 2).copied();
1828    match first {
1829      C_PLUS_SIGN | C_HYPHEN_MINUS => {
1830        second.is_some_and(is_digit) || (second == Some(C_FULL_STOP) && third.is_some_and(is_digit))
1831      }
1832      C_FULL_STOP => second.is_some_and(is_digit),
1833      _ => is_digit(first),
1834    }
1835  }
1836
1837  #[inline]
1838  fn is_valid_escape_at(&self, position: usize) -> bool {
1839    self.value.get(position) == Some(&C_REVERSE_SOLIDUS)
1840      && self
1841        .value
1842        .get(position + 1)
1843        .is_some_and(|byte| !is_new_line(*byte))
1844  }
1845
1846  #[inline]
1847  fn scan_escape(&self, position: usize) -> usize {
1848    debug_assert!(self.is_valid_escape_at(position));
1849    let mut end = position + 1;
1850    if self.value[end].is_ascii_hexdigit() {
1851      let mut digits = 0;
1852      while end < self.value.len() && digits < 6 && self.value[end].is_ascii_hexdigit() {
1853        end += 1;
1854        digits += 1;
1855      }
1856      if self
1857        .value
1858        .get(end)
1859        .is_some_and(|byte| is_white_space(*byte))
1860      {
1861        end += 1;
1862        if self.value.get(end - 1) == Some(&C_CARRIAGE_RETURN)
1863          && self.value.get(end) == Some(&C_LINE_FEED)
1864        {
1865          end += 1;
1866        }
1867      }
1868      return end;
1869    }
1870    end + self.utf8_width_at(end)
1871  }
1872
1873  #[inline]
1874  fn utf8_width_at(&self, position: usize) -> usize {
1875    let byte = self.value[position];
1876    if byte < 0x80 {
1877      1
1878    } else if byte < 0xE0 {
1879      2
1880    } else if byte < 0xF0 {
1881      3
1882    } else {
1883      4
1884    }
1885  }
1886}
1887
1888/// The single forward stream used by dependency extraction.
1889///
1890/// The stream owns the scanner, buffered lookahead, and the rule alternative
1891/// events without cloning the lexer for every decision. Special
1892/// parsers use the same stream through `next_parser_token`, so consuming a
1893/// subgrammar never creates a second scanner over the source.
1894///
1895/// The source cursor is strictly one-directional:
1896///
1897/// ```text
1898/// Lexer::scan_pos         farthest tokenized position, only advances
1899/// TokenStream::consumed   semantic position of the currently selected grammar
1900/// buffered lookahead      tokens scanned but not yet consumed
1901/// ```
1902///
1903/// `next` consumes from the buffer first and only calls `lexer.next_token`
1904/// when the buffer is empty; it never moves the scanner backwards, so every
1905/// source range is tokenized at most once. A speculative declaration consumes
1906/// tokens immediately; a surviving rule can replay its saved semantic events
1907/// after declaration effects are rolled back. The source scanner never rewinds.
1908pub(crate) struct TokenStream<'a, 's, V: LexerVisitor = ()> {
1909  lexer: &'a mut Lexer<'s, V>,
1910  consumed: Pos,
1911  buffered: VecDeque<BufferedToken>,
1912  value_state: GenericValueScanState,
1913  at_rule_state: GenericValueScanState,
1914  pub(crate) block_item: Option<BlockItemCandidates>,
1915  rule_events: Vec<BufferedToken>,
1916  function_kind: FunctionKind,
1917  parsed_value_end: Option<Pos>,
1918}
1919
1920/// The raw scanner can save selector events without dispatching them through
1921/// declaration semantics. Syntax still advances on the same component stack.
1922struct BlockItemScan<'a> {
1923  candidate: &'a mut BlockItemCandidates,
1924  events: &'a mut Vec<BufferedToken>,
1925}
1926
1927impl std::ops::Deref for BlockItemScan<'_> {
1928  type Target = BlockItemCandidates;
1929
1930  fn deref(&self) -> &Self::Target {
1931    self.candidate
1932  }
1933}
1934
1935impl std::ops::DerefMut for BlockItemScan<'_> {
1936  fn deref_mut(&mut self) -> &mut Self::Target {
1937    self.candidate
1938  }
1939}
1940
1941/// While ambiguous, syntax and raw value scanning use the same component stack.
1942#[inline]
1943fn value_is_nested(state: &ComponentValues, candidate: &Option<BlockItemScan<'_>>) -> bool {
1944  candidate
1945    .as_ref()
1946    .map_or_else(|| state.is_nested(), |candidate| candidate.is_nested())
1947}
1948
1949#[inline]
1950fn value_expects(
1951  state: &ComponentValues,
1952  candidate: &Option<BlockItemScan<'_>>,
1953  kind: TokenKind,
1954) -> bool {
1955  candidate
1956    .as_ref()
1957    .map_or_else(|| state.expects(kind), |candidate| candidate.expects(kind))
1958}
1959
1960#[inline]
1961fn note_opaque_value(candidate: &mut Option<BlockItemScan<'_>>) {
1962  if let Some(candidate) = candidate {
1963    candidate.opaque_value();
1964  }
1965}
1966
1967#[inline]
1968fn observe_opaque_value(
1969  candidate: &mut Option<BlockItemScan<'_>>,
1970  kind: TokenKind,
1971  start: usize,
1972  end: usize,
1973  source: &[u8],
1974  leading: Trivia,
1975  function: FunctionKind,
1976) {
1977  if let Some(candidate) = candidate {
1978    let range = Range::new(start as Pos, end as Pos);
1979    let value_range = if kind == TokenKind::Function {
1980      Range::new(start as Pos, end as Pos - 1)
1981    } else {
1982      range
1983    };
1984    let token = Token::with_flags(kind, range, value_range, TokenFlags::ascii());
1985    candidate.observe(token, source);
1986    if candidate.preserve_selector_tokens() {
1987      candidate.events.push(BufferedToken::new(
1988        TokenWithTrivia { token, leading },
1989        function,
1990      ));
1991    }
1992  }
1993}
1994
1995pub(crate) type GenericValueScanState = ComponentValues;
1996
1997#[derive(Debug, Clone, Copy)]
1998struct GenericValueScanOptions {
1999  keep_comments: bool,
2000  preserve_strings: bool,
2001  preserve_delimiters: bool,
2002  property: Option<PropertyKind>,
2003  stop_at_top_level_left_curly: bool,
2004  allow_flat_scan: bool,
2005}
2006
2007/// Classification shared by tokenization, raw scanning and semantic parsing.
2008#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2009pub(crate) enum FunctionKind {
2010  Other,
2011  Url,
2012  ImageSet,
2013  Layer,
2014  Supports,
2015  PaletteMix,
2016  Var,
2017  Local,
2018  Global,
2019  Dashed,
2020}
2021
2022impl FunctionKind {
2023  #[inline]
2024  fn classify(name: &str, flags: TokenFlags) -> Self {
2025    let mut lowercase = [0; MAX_CSS_KEYWORD_LEN];
2026    let normalized = if flags.has_escape() {
2027      decode_css_keyword(name, &mut lowercase)
2028    } else {
2029      lowercase_ascii_keyword(name, &mut lowercase)
2030    };
2031    match normalized {
2032      Some("url") => Self::Url,
2033      Some("image-set") => Self::ImageSet,
2034      Some("layer") => Self::Layer,
2035      Some("supports") => Self::Supports,
2036      Some("palette-mix") => Self::PaletteMix,
2037      Some("var") => Self::Var,
2038      Some("local") => Self::Local,
2039      Some("global") => Self::Global,
2040      Some(name) if strip_vendor_prefix(name) == Some("image-set") => Self::ImageSet,
2041      _ if name.starts_with("--") => Self::Dashed,
2042      _ => Self::Other,
2043    }
2044  }
2045
2046  #[inline]
2047  fn is_dependency(self) -> bool {
2048    matches!(self, Self::Url | Self::ImageSet | Self::Var | Self::Dashed)
2049  }
2050}
2051
2052/// Compact metadata shared by lookahead, raw scanning and rule replay.
2053/// Trivia endpoints coincide with the token start and are reconstructed on use.
2054#[derive(Debug, Clone, Copy)]
2055struct BufferedToken {
2056  token: Token,
2057  leading_start: Pos,
2058  first_comment_start: Pos,
2059  has_whitespace: bool,
2060  function: FunctionKind,
2061}
2062
2063impl BufferedToken {
2064  fn new(item: TokenWithTrivia, function: FunctionKind) -> Self {
2065    debug_assert_eq!(item.leading.end, item.token.range.start);
2066    debug_assert_eq!(item.leading.range.end, item.token.range.start);
2067    Self {
2068      token: item.token,
2069      leading_start: item.leading.range.start,
2070      first_comment_start: item.leading.first_comment_start.unwrap_or(Pos::MAX),
2071      has_whitespace: item.leading.has_white_space,
2072      function,
2073    }
2074  }
2075
2076  #[inline]
2077  fn into_item(self) -> TokenWithTrivia {
2078    TokenWithTrivia {
2079      token: self.token,
2080      leading: Trivia {
2081        range: Range::new(self.leading_start, self.token.range.start),
2082        end: self.token.range.start,
2083        first_comment_start: (self.first_comment_start != Pos::MAX)
2084          .then_some(self.first_comment_start),
2085        has_white_space: self.has_whitespace,
2086      },
2087    }
2088  }
2089}
2090
2091#[inline(always)]
2092fn never_fast_forward_candidate(_: &str) -> bool {
2093  false
2094}
2095
2096impl<'a, 's, V: LexerVisitor> TokenStream<'a, 's, V> {
2097  pub(crate) fn from_lexer(lexer: &'a mut Lexer<'s, V>) -> Self {
2098    Self {
2099      lexer,
2100      consumed: 0,
2101      buffered: VecDeque::new(),
2102      value_state: GenericValueScanState::default(),
2103      at_rule_state: GenericValueScanState::default(),
2104      block_item: None,
2105      rule_events: Vec::new(),
2106      function_kind: FunctionKind::Other,
2107      parsed_value_end: None,
2108    }
2109  }
2110
2111  #[inline(always)]
2112  pub(crate) fn next(&mut self, keep_comments: bool) -> TokenWithTrivia {
2113    let token = if let Some(token) = self.buffered.pop_front() {
2114      token
2115    } else {
2116      self.read_significant(keep_comments)
2117    };
2118    debug_assert!(
2119      token.token.range.start >= self.consumed,
2120      "token range starts before consumed_pos: {:?} < {}",
2121      token.token.range,
2122      self.consumed
2123    );
2124    if let Some(candidate) = &mut self.block_item
2125      && !matches!(candidate.progress, BlockItemProgress::Finished(_))
2126    {
2127      candidate.observe(token.token, self.lexer.value);
2128      if candidate.retain_rule_event(token.token.kind) {
2129        self.rule_events.push(token);
2130      }
2131    }
2132    self.function_kind = token.function;
2133    self
2134      .lexer
2135      .visit_ident(token.token.kind, token.token.value_range);
2136    self.consumed = token.token.range.end;
2137    debug_assert!(
2138      self.consumed <= self.lexer.scan_pos(),
2139      "consumed_pos ({}) exceeded scan_pos ({})",
2140      self.consumed,
2141      self.lexer.scan_pos()
2142    );
2143    token.into_item()
2144  }
2145
2146  #[inline]
2147  pub(crate) fn function_kind(&self) -> FunctionKind {
2148    self.function_kind
2149  }
2150
2151  #[inline(always)]
2152  pub(crate) fn next_with_progress(
2153    &mut self,
2154    keep_comments: bool,
2155  ) -> (TokenWithTrivia, BlockItemProgress) {
2156    let token = self.next(keep_comments);
2157    (token, self.block_item_progress())
2158  }
2159
2160  #[inline(always)]
2161  pub(crate) fn block_item_progress(&self) -> BlockItemProgress {
2162    self
2163      .block_item
2164      .as_ref()
2165      .map_or(BlockItemProgress::Continue, |candidate| candidate.progress)
2166  }
2167
2168  /// Resolve cheap prefixes with their semantic result. The validated range
2169  /// is consumed by the value parser, so it never scans this text again.
2170  /// Return whether the declaration is prepared and, on failure, the attempted
2171  /// flat scan's start so candidate parsing can avoid repeating it.
2172  pub(crate) fn prepare_declaration(
2173    &mut self,
2174    ident: Token,
2175    custom: bool,
2176    flat: bool,
2177    collect_dashed: bool,
2178  ) -> (bool, Option<Pos>) {
2179    let mut flat_probe_start = None;
2180    if self.byte_at(ident.range.end) != Some(C_COLON) {
2181      return (false, flat_probe_start);
2182    }
2183    if !custom && !flat {
2184      return (false, flat_probe_start);
2185    }
2186    let end = if custom {
2187      None
2188    } else {
2189      flat_probe_start = Some(ident.range.end + 1);
2190      let Some(end) = self.lexer.scan_flat_value(ident.range.end as usize + 1) else {
2191        return (false, flat_probe_start);
2192      };
2193      if collect_dashed
2194        && self.lexer.value[ident.range.end as usize + 1..end]
2195          .windows(2)
2196          .any(|bytes| bytes == b"--")
2197      {
2198        return (false, flat_probe_start);
2199      }
2200      Some(end as Pos)
2201    };
2202    // `peek` has already buffered the identifier. Its colon is read once too.
2203    if self.buffered.len() != 1 {
2204      return (false, flat_probe_start);
2205    }
2206    let colon = self.read_significant(false);
2207    debug_assert_eq!(colon.token.kind, TokenKind::Colon);
2208    self.buffered.push_back(colon);
2209    self.parsed_value_end = end;
2210    (true, None)
2211  }
2212
2213  pub(crate) fn begin_block_item(
2214    &mut self,
2215    custom: bool,
2216    has_mode: bool,
2217    collect_dashed: bool,
2218    flat_probe_start: Option<Pos>,
2219  ) {
2220    let mut candidate = BlockItemCandidates::new(custom, has_mode);
2221    candidate.flat_probe_start = flat_probe_start;
2222    candidate.collect_dashed = collect_dashed;
2223    self.block_item = Some(candidate);
2224  }
2225
2226  /// Select the semantic result without re-tokenizing the source. A declaration
2227  /// is already consumed; only a surviving rule alternative needs its events.
2228  pub(crate) fn finish_block_item(&mut self) -> BlockItemKind {
2229    let candidate = self.block_item.take().expect("an active block item");
2230    let BlockItemProgress::Finished(selected) = candidate.progress else {
2231      unreachable!("the block item has been resolved");
2232    };
2233    match selected {
2234      BlockItemKind::Declaration => {}
2235      BlockItemKind::Rule => {
2236        self.consumed = self
2237          .rule_events
2238          .first()
2239          .expect("a rule has a prefix")
2240          .token
2241          .range
2242          .start;
2243        for event in self.rule_events.drain(..).rev() {
2244          self.buffered.push_front(event);
2245        }
2246        self.reset_value_scan_states();
2247      }
2248      BlockItemKind::Invalid => {
2249        let boundary = *self
2250          .rule_events
2251          .last()
2252          .expect("an invalid item ends at a boundary");
2253        self.consumed = boundary.token.range.start;
2254        self.buffered.push_front(boundary);
2255        self.reset_value_scan_states();
2256      }
2257    }
2258    self.rule_events.clear();
2259    selected
2260  }
2261
2262  pub(crate) fn reset_value_scan_states(&mut self) {
2263    self.value_state = GenericValueScanState::default();
2264  }
2265
2266  pub(crate) fn value_is_nested(&self) -> bool {
2267    self.value_state.is_nested()
2268      || self
2269        .block_item
2270        .as_ref()
2271        .is_some_and(BlockItemCandidates::is_nested)
2272  }
2273
2274  /// Consume the next parser token while folding comments into its leading
2275  /// trivia. This is the equivalent of the old isolated cursor's behavior,
2276  /// but it advances the main stream and therefore preserves one scanner
2277  /// ownership for all subgrammars.
2278  #[inline]
2279  pub(crate) fn next_parser_token(&mut self) -> TokenWithTrivia {
2280    let mut item = self.next(true);
2281    if item.token.kind != TokenKind::Comment {
2282      return item;
2283    }
2284
2285    let start = item.leading.range.start;
2286    let first_comment_start = item
2287      .leading
2288      .first_comment_start
2289      .or(Some(item.token.range.start));
2290    let mut has_white_space = item.leading.has_white_space;
2291    loop {
2292      item = self.next(true);
2293      has_white_space |= item.leading.has_white_space;
2294      if item.token.kind == TokenKind::Comment {
2295        continue;
2296      }
2297      let end = item.leading.end;
2298      item.leading = Trivia {
2299        range: Range::new(start, end),
2300        end,
2301        first_comment_start,
2302        has_white_space,
2303      };
2304      return item;
2305    }
2306  }
2307
2308  #[inline]
2309  pub(crate) fn peek(&mut self, keep_comments: bool) -> TokenWithTrivia {
2310    if self.buffered.is_empty() {
2311      let item = self.read_significant(keep_comments);
2312      self.buffered.push_back(item);
2313    }
2314    self
2315      .buffered
2316      .front()
2317      .expect("peek must buffer an item before reading the front")
2318      .into_item()
2319  }
2320
2321  /// Peek the next parser token (folding comments into its leading trivia)
2322  /// without consuming it or any preceding comments. The stream state is
2323  /// unchanged: `next` later returns the same tokens in the same order.
2324  #[inline]
2325  pub(crate) fn peek_parser_token(&mut self) -> TokenWithTrivia {
2326    self.peek(true);
2327    let mut leading_start = None;
2328    let mut first_comment_start = None;
2329    let mut has_white_space = false;
2330    let mut index = 0usize;
2331    loop {
2332      if index == self.buffered.len() {
2333        let item = self.read_significant(true);
2334        self.buffered.push_back(item);
2335      }
2336      let item = self.buffered[index].into_item();
2337      leading_start.get_or_insert(item.leading.range.start);
2338      has_white_space |= item.leading.has_whitespace();
2339      if matches!(item.token.kind, TokenKind::Comment | TokenKind::BadComment) {
2340        first_comment_start.get_or_insert(item.token.range.start);
2341        index += 1;
2342        continue;
2343      }
2344
2345      let end = item.leading.end;
2346      let mut result = item;
2347      result.leading = Trivia {
2348        range: Range::new(leading_start.unwrap_or(end), end),
2349        end,
2350        first_comment_start: first_comment_start.or(item.leading.first_comment_start),
2351        has_white_space,
2352      };
2353      return result;
2354    }
2355  }
2356
2357  /// Peek at the next significant token, leaving any skipped comments in
2358  /// the buffer for `next` to consume in source order.
2359  #[inline]
2360  pub(crate) fn peek_significant_skipping_comments(
2361    &mut self,
2362    keep_comments: bool,
2363  ) -> TokenWithTrivia {
2364    self.peek(keep_comments);
2365    for item in &self.buffered {
2366      if !matches!(item.token.kind, TokenKind::Comment | TokenKind::BadComment) {
2367        return item.into_item();
2368      }
2369    }
2370    loop {
2371      let item = self.read_significant(keep_comments);
2372      self.buffered.push_back(item);
2373      if !matches!(item.token.kind, TokenKind::Comment | TokenKind::BadComment) {
2374        return item.into_item();
2375      }
2376    }
2377  }
2378
2379  /// The position of the next yet-untokenized source byte. The buffer may
2380  /// still hold tokens; callers must drain `next` before fast-forwarding.
2381  #[inline]
2382  pub(crate) fn fast_forward_generic_value_if_buffer_empty<F, C>(
2383    &mut self,
2384    keep_comments: bool,
2385    preserve_strings: bool,
2386    preserve_delimiters: bool,
2387    mut is_candidate: F,
2388    mut is_comment_candidate: C,
2389  ) where
2390    F: FnMut(&str) -> bool,
2391    C: FnMut(&str) -> bool,
2392  {
2393    if !self.buffered.is_empty() {
2394      return;
2395    }
2396    let candidate = self.lexer.fast_forward_generic_value(
2397      &mut self.value_state,
2398      GenericValueScanOptions {
2399        keep_comments,
2400        preserve_strings,
2401        preserve_delimiters,
2402        property: None,
2403        stop_at_top_level_left_curly: false,
2404        allow_flat_scan: false,
2405      },
2406      &mut is_candidate,
2407      &mut is_comment_candidate,
2408      None,
2409      self.block_item.as_mut().map(|candidate| BlockItemScan {
2410        candidate,
2411        events: &mut self.rule_events,
2412      }),
2413    );
2414    self.finish_value_fast_forward(candidate);
2415  }
2416
2417  #[inline]
2418  pub(crate) fn fast_forward_generic_value_without_candidates_if_buffer_empty(
2419    &mut self,
2420    preserve_strings: bool,
2421    preserve_delimiters: bool,
2422  ) {
2423    if !self.buffered.is_empty() {
2424      return;
2425    }
2426    if let Some(end) = self.parsed_value_end.take() {
2427      debug_assert!(self.lexer.scan_pos() <= end);
2428      self.lexer.scan_pos = end;
2429      self.consumed = end;
2430      return;
2431    }
2432    let candidate = self.lexer.fast_forward_generic_value(
2433      &mut self.value_state,
2434      GenericValueScanOptions {
2435        keep_comments: false,
2436        preserve_strings,
2437        preserve_delimiters,
2438        property: None,
2439        stop_at_top_level_left_curly: false,
2440        allow_flat_scan: true,
2441      },
2442      never_fast_forward_candidate,
2443      never_fast_forward_candidate,
2444      None,
2445      self.block_item.as_mut().map(|candidate| BlockItemScan {
2446        candidate,
2447        events: &mut self.rule_events,
2448      }),
2449    );
2450    self.finish_value_fast_forward(candidate);
2451  }
2452
2453  #[inline]
2454  pub(crate) fn fast_forward_at_rule_if_buffer_empty<F, C>(
2455    &mut self,
2456    keep_comments: bool,
2457    preserve_strings: bool,
2458    preserve_delimiters: bool,
2459    mut is_candidate: F,
2460    mut is_comment_candidate: C,
2461  ) where
2462    F: FnMut(&str) -> bool,
2463    C: FnMut(&str) -> bool,
2464  {
2465    if !self.buffered.is_empty() {
2466      return;
2467    }
2468    let candidate = self.lexer.fast_forward_generic_value(
2469      &mut self.at_rule_state,
2470      GenericValueScanOptions {
2471        keep_comments,
2472        preserve_strings,
2473        preserve_delimiters,
2474        property: None,
2475        stop_at_top_level_left_curly: true,
2476        allow_flat_scan: false,
2477      },
2478      &mut is_candidate,
2479      &mut is_comment_candidate,
2480      None,
2481      None,
2482    );
2483    self.finish_value_fast_forward(candidate);
2484  }
2485
2486  #[inline]
2487  pub(crate) fn reset_at_rule_scan_state(&mut self) {
2488    self.at_rule_state = GenericValueScanState::default();
2489  }
2490
2491  #[inline]
2492  pub(crate) fn fast_forward_special_value_if_buffer_empty(
2493    &mut self,
2494    keep_comments: bool,
2495    preserve_strings: bool,
2496    preserve_delimiters: bool,
2497    property: PropertyKind,
2498    icss_symbols: Option<&FxHashSet<&str>>,
2499  ) {
2500    if !self.buffered.is_empty() {
2501      return;
2502    }
2503    let candidate = self.lexer.fast_forward_generic_value(
2504      &mut self.value_state,
2505      GenericValueScanOptions {
2506        keep_comments,
2507        preserve_strings,
2508        preserve_delimiters,
2509        property: Some(property),
2510        stop_at_top_level_left_curly: false,
2511        allow_flat_scan: false,
2512      },
2513      never_fast_forward_candidate,
2514      is_css_modules_pure_magic_comment,
2515      icss_symbols,
2516      self.block_item.as_mut().map(|candidate| BlockItemScan {
2517        candidate,
2518        events: &mut self.rule_events,
2519      }),
2520    );
2521    self.finish_value_fast_forward(candidate);
2522  }
2523
2524  #[inline]
2525  fn finish_value_fast_forward(&mut self, candidate: Option<BufferedToken>) {
2526    let Some(candidate) = candidate else {
2527      self.consumed = self.lexer.scan_pos();
2528      return;
2529    };
2530    debug_assert_eq!(candidate.token.range.end, self.lexer.scan_pos());
2531    self.consumed = candidate.token.range.start;
2532    self.buffered.push_back(candidate);
2533  }
2534
2535  #[inline]
2536  pub(crate) fn fast_forward_selector_if_buffer_empty<F, C>(
2537    &mut self,
2538    square_depth: &mut u32,
2539    keep_comments: bool,
2540    has_mode: bool,
2541    is_ident_candidate: F,
2542    is_comment_candidate: C,
2543  ) -> bool
2544  where
2545    F: FnMut(&str) -> bool,
2546    C: FnMut(&str) -> bool,
2547  {
2548    if !self.buffered.is_empty() {
2549      return false;
2550    }
2551    if *square_depth == 0 {
2552      let scan_pos = self.lexer.scan_pos();
2553      let Some(next) = self.lexer.byte_at(scan_pos) else {
2554        return false;
2555      };
2556      if matches!(
2557        next,
2558        b'"' | b'\'' | b'(' | b')' | b'{' | b'}' | b',' | b';' | b'@'
2559      ) || (has_mode && matches!(next, b'.' | b'#' | b':'))
2560      {
2561        return false;
2562      }
2563
2564      // A raw scan has a fixed call/setup cost. Keep short, dense
2565      // selector fragments on the regular tokenizer path and only
2566      // enter the scanner when it can skip a useful run. Comments and
2567      // attributes are exceptions because their contents are opaque.
2568      let mut probe = scan_pos;
2569      while probe - scan_pos < 4 {
2570        let Some(byte) = self.lexer.byte_at(probe) else {
2571          return false;
2572        };
2573        if matches!(byte, b'/' | b'[') {
2574          break;
2575        }
2576        if probe != scan_pos
2577          && (matches!(
2578            byte,
2579            b'"' | b'\'' | b'(' | b')' | b'{' | b'}' | b',' | b';' | b'@'
2580          ) || (has_mode && matches!(byte, b'.' | b'#' | b':')))
2581        {
2582          return false;
2583        }
2584        probe += 1;
2585      }
2586    }
2587    let invalidates_composes = self.lexer.fast_forward_selector(
2588      square_depth,
2589      keep_comments,
2590      has_mode,
2591      is_ident_candidate,
2592      is_comment_candidate,
2593    );
2594    self.consumed = self.lexer.scan_pos();
2595    invalidates_composes
2596  }
2597
2598  /// Skip a balanced delimiter run without manufacturing tokens. The opening
2599  /// token must already be consumed, and no further tokens may be buffered.
2600  ///
2601  /// On success `consumed` is advanced to the end of the closing token and
2602  /// the full range is returned. On failure (`None`) the lexer position is
2603  /// unchanged so the caller can re-tokenize the region normally.
2604  pub(crate) fn fast_forward(&mut self, end: TokenKind) -> Option<Range> {
2605    if !self.buffered.is_empty() {
2606      return None;
2607    }
2608    let old_scan_pos = self.lexer.scan_pos();
2609    let old_consumed = self.consumed;
2610    let range = self.lexer.fast_forward(end)?;
2611    self.consumed = range.end;
2612    debug_assert!(old_scan_pos <= self.lexer.scan_pos());
2613    debug_assert!(old_consumed <= self.consumed);
2614    debug_assert!(self.consumed <= self.lexer.scan_pos());
2615    Some(range)
2616  }
2617
2618  /// The farthest position consumed by `next`. This is the current semantic
2619  /// position of the dependency scanner.
2620  #[inline]
2621  pub(crate) fn consumed_pos(&self) -> Pos {
2622    self.consumed
2623  }
2624
2625  #[inline]
2626  pub(crate) fn lexer(&self) -> &Lexer<'s, V> {
2627    self.lexer
2628  }
2629
2630  #[inline]
2631  pub(crate) fn lexer_mut(&mut self) -> &mut Lexer<'s, V> {
2632    self.lexer
2633  }
2634
2635  #[inline]
2636  pub(crate) fn source_end(&self) -> Pos {
2637    self.lexer.source_end()
2638  }
2639
2640  #[inline(always)]
2641  pub(crate) fn byte_at(&self, position: Pos) -> Option<u8> {
2642    self.lexer.byte_at(position)
2643  }
2644
2645  #[inline]
2646  pub(crate) fn slice(&self, start: Pos, end: Pos) -> Option<&'s str> {
2647    self.lexer.slice(start, end)
2648  }
2649
2650  #[inline(always)]
2651  pub(crate) fn slice_trusted(&self, start: Pos, end: Pos) -> &'s str {
2652    self.lexer.slice_trusted(start, end)
2653  }
2654
2655  #[inline(always)]
2656  fn read_significant(&mut self, keep_comments: bool) -> BufferedToken {
2657    let start = self.lexer.scan_pos();
2658    let mut end = start;
2659    let mut first_comment_start = None;
2660    let mut has_white_space = false;
2661    loop {
2662      let token = self.lexer.next_token_without_visit();
2663      debug_assert!(token.range.start >= end);
2664      if token.kind == TokenKind::WhiteSpace {
2665        has_white_space = true;
2666        end = token.range.end;
2667        continue;
2668      }
2669      if !keep_comments && matches!(token.kind, TokenKind::Comment | TokenKind::BadComment) {
2670        first_comment_start.get_or_insert(token.range.start);
2671        end = token.range.end;
2672        continue;
2673      }
2674      let leading = Trivia {
2675        range: Range::new(start, end),
2676        end,
2677        first_comment_start,
2678        has_white_space,
2679      };
2680      return BufferedToken::new(TokenWithTrivia { token, leading }, self.lexer.function_kind);
2681    }
2682  }
2683}
2684
2685pub fn is_new_line(c: u8) -> bool {
2686  c == C_LINE_FEED || c == C_CARRIAGE_RETURN || c == C_FORM_FEED
2687}
2688
2689#[inline]
2690fn kind_to_right(byte: u8) -> TokenKind {
2691  match byte {
2692    C_RIGHT_PARENTHESIS => TokenKind::RightParenthesis,
2693    C_RIGHT_SQUARE => TokenKind::RightSquareBracket,
2694    C_RIGHT_CURLY => TokenKind::RightCurlyBracket,
2695    _ => unreachable!("fast_forward only sees closing bracket bytes"),
2696  }
2697}
2698
2699pub fn is_space(c: u8) -> bool {
2700  c == C_TAB || c == C_SPACE
2701}
2702
2703pub fn is_white_space(c: u8) -> bool {
2704  is_new_line(c) || is_space(c)
2705}
2706
2707pub fn is_digit(c: u8) -> bool {
2708  c.is_ascii_digit()
2709}
2710
2711#[inline]
2712fn is_non_flat_value_byte(byte: u8) -> bool {
2713  matches!(
2714    byte,
2715    C_QUOTATION_MARK
2716      | C_APOSTROPHE
2717      | C_SOLIDUS
2718      | C_REVERSE_SOLIDUS
2719      | C_LEFT_PARENTHESIS
2720      | C_RIGHT_PARENTHESIS
2721      | C_LEFT_SQUARE
2722      | C_RIGHT_SQUARE
2723  )
2724}
2725
2726#[inline]
2727fn is_name_start_byte(c: u8) -> bool {
2728  c == 0 || c == C_LOW_LINE || c.is_ascii_alphabetic() || !c.is_ascii()
2729}
2730
2731#[inline]
2732fn is_non_printable(c: u8) -> bool {
2733  matches!(c, 0x00..=0x08 | 0x0B | 0x0E..=0x1F | 0x7F)
2734}
2735
2736#[cfg(test)]
2737#[path = "../tests/lexer_tests.rs"]
2738mod tests;