omena_parser/

lex.rs

//! Public lexer result types and token wrappers.
//!
//! Lexing remains dialect-aware, but this module exposes a stable token surface
//! for summaries and parser fact collection.

use cstree::text::{TextRange, TextSize};
use omena_syntax::{StyleDialect, SyntaxKind};

use crate::{
    DialectExtension, ParseError, ParseErrorCode, TemplatePlaceholderMode,
    matches_ignore_ascii_case,
};

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct Token<'text> {
    pub(crate) kind: SyntaxKind,
    pub(crate) text: &'text str,
    pub(crate) range: TextRange,
}

pub(crate) struct Tokenizer<'text, 'extension, E> {
    pub(crate) text: &'text str,
    pub(crate) extension: &'extension E,
    pub(crate) offset: usize,
    pub(crate) template_placeholder: Option<TemplatePlaceholderMode>,
    pub(crate) template_interpolation_depth: usize,
    pub(crate) scss_interpolation_depth: usize,
    pub(crate) less_interpolation_depth: usize,
    pub(crate) sass_indent_stack: Vec<usize>,
    pub(crate) tokens: Vec<Token<'text>>,
    pub(crate) errors: Vec<ParseError>,
}

#[derive(Debug, Clone, PartialEq, Eq)]
pub struct LexResult {
    tokens: Vec<LexedToken>,
    errors: Vec<ParseError>,
    dialect: StyleDialect,
}

impl LexResult {
    pub(crate) fn new(
        tokens: Vec<LexedToken>,
        errors: Vec<ParseError>,
        dialect: StyleDialect,
    ) -> Self {
        Self {
            tokens,
            errors,
            dialect,
        }
    }

    pub fn tokens(&self) -> &[LexedToken] {
        &self.tokens
    }

    pub fn errors(&self) -> &[ParseError] {
        &self.errors
    }

    pub fn dialect(&self) -> StyleDialect {
        self.dialect
    }
}

#[derive(Debug, Clone, PartialEq, Eq)]
pub struct LexedToken {
    pub kind: SyntaxKind,
    pub range: TextRange,
    pub text: String,
}

pub(crate) fn public_token_text(text: &str) -> String {
    text.chars()
        .map(css_syntax_preprocessed_char)
        .collect::<String>()
}

pub(crate) fn is_name_start(char: char) -> bool {
    let char = css_syntax_preprocessed_char(char);
    char == '_' || char == '-' || char.is_alphabetic() || !char.is_ascii()
}

pub(crate) fn is_name_continue(char: char) -> bool {
    is_name_start(char) || char.is_ascii_digit()
}

pub(crate) fn is_non_printable_code_point(char: char) -> bool {
    let char = css_syntax_preprocessed_char(char);
    matches!(char, '\u{0000}'..='\u{0008}' | '\u{000b}' | '\u{000e}'..='\u{001f}' | '\u{007f}')
}

pub(crate) fn is_custom_property_name_text(text: &str) -> bool {
    let Some(rest) = text.strip_prefix("--") else {
        return false;
    };
    let Some(first) = rest.chars().next() else {
        return false;
    };
    first == '-' || is_name_start(first) || starts_valid_escape_text(rest)
}

pub(crate) fn is_css_at_rule_name(text: &str) -> bool {
    matches_ignore_ascii_case(
        text,
        &[
            "@charset",
            "@container",
            "@font-face",
            "@font-feature-values",
            "@function",
            "@font-palette-values",
            "@import",
            "@keyframes",
            "@layer",
            "@media",
            "@namespace",
            "@page",
            "@property",
            "@scope",
            "@starting-style",
            "@supports",
            "@counter-style",
            "@custom-media",
            "@color-profile",
            "@nest",
            "@position-try",
            "@view-transition",
            "@stylistic",
            "@styleset",
            "@character-variant",
            "@swash",
            "@ornaments",
            "@annotation",
            "@historical-forms",
            "@when",
            "@else",
        ],
    )
}

pub(crate) fn sass_token_can_end_statement(kind: SyntaxKind) -> bool {
    !matches!(
        kind,
        SyntaxKind::Whitespace
            | SyntaxKind::LineComment
            | SyntaxKind::BlockComment
            | SyntaxKind::SassIndentedNewline
            | SyntaxKind::SassIndent
            | SyntaxKind::SassDedent
            | SyntaxKind::SassOptionalSemicolon
            | SyntaxKind::Comma
            | SyntaxKind::Colon
            | SyntaxKind::DoubleColon
            | SyntaxKind::LeftBrace
            | SyntaxKind::LeftParen
            | SyntaxKind::LeftBracket
            | SyntaxKind::Plus
            | SyntaxKind::Minus
            | SyntaxKind::Star
            | SyntaxKind::Slash
            | SyntaxKind::GreaterThan
            | SyntaxKind::LessThan
            | SyntaxKind::Equals
            | SyntaxKind::Arrow
            | SyntaxKind::Pipe
            | SyntaxKind::Tilde
            | SyntaxKind::Caret
            | SyntaxKind::Ampersand
            | SyntaxKind::DoubleAmpersand
            | SyntaxKind::ColumnCombinator
            | SyntaxKind::IncludesMatch
            | SyntaxKind::DashMatch
            | SyntaxKind::PrefixMatch
            | SyntaxKind::SuffixMatch
            | SyntaxKind::SubstringMatch
            | SyntaxKind::PlusEquals
            | SyntaxKind::MinusEquals
            | SyntaxKind::SlashEquals
    )
}

pub(crate) fn text_range(start: usize, end: usize) -> TextRange {
    TextRange::new(TextSize::from(start as u32), TextSize::from(end as u32))
}

impl<'text, 'extension, E> Tokenizer<'text, 'extension, E>
where
    E: DialectExtension,
{
    pub(crate) fn new(text: &'text str, extension: &'extension E) -> Self {
        Self {
            text,
            extension,
            offset: 0,
            template_placeholder: extension.template_placeholder(),
            template_interpolation_depth: 0,
            scss_interpolation_depth: 0,
            less_interpolation_depth: 0,
            sass_indent_stack: vec![0],
            tokens: Vec::new(),
            errors: Vec::new(),
        }
    }

    pub(crate) fn tokenize(&mut self) {
        while let Some(current) = self.current_char() {
            let start = self.offset;
            match current {
                '\u{feff}' if start == 0 => self.bump_current(),
                '\r' | '\n' if self.extension.dialect() == StyleDialect::Sass => {
                    self.consume_sass_indented_newline(start)
                }
                char if char.is_whitespace() => {
                    self.consume_while(SyntaxKind::Whitespace, |c| c.is_whitespace())
                }
                '/' if self.starts_with("/*") => self.consume_block_comment(),
                '/' if self.starts_with("//") && self.extension.dialect() != StyleDialect::Css => {
                    self.consume_line_comment()
                }
                '$' if self.starts_with("${")
                    && self.template_placeholder == Some(TemplatePlaceholderMode::Brace) =>
                {
                    self.consume_template_interpolation_start(start)
                }
                '$' if self.starts_with("${")
                    && self.template_placeholder
                        == Some(TemplatePlaceholderMode::AtomicIndexed) =>
                {
                    self.consume_template_placeholder(start)
                }
                '#' if self.starts_with("#{") && self.supports_scss_interpolation() => {
                    self.consume_scss_interpolation_start(start)
                }
                '@' if self.starts_with("@{") && self.supports_less_interpolation() => {
                    self.consume_less_interpolation_start(start)
                }
                '!' if self.starts_with_ascii_keyword("!important") => {
                    self.consume_static(SyntaxKind::Important, start, "!important".len())
                }
                '<' if self.starts_with("<!--") => {
                    self.consume_static(SyntaxKind::Cdo, start, "<!--".len())
                }
                '-' if self.starts_with("-->") => {
                    self.consume_static(SyntaxKind::Cdc, start, "-->".len())
                }
                '"' | '\'' => self.consume_string(current),
                'u' | 'U' if self.starts_unicode_range() => self.consume_unicode_range(),
                '0'..='9' => self.consume_number(),
                '$' if matches!(
                    self.extension.dialect(),
                    StyleDialect::Scss | StyleDialect::Sass
                ) =>
                {
                    self.consume_prefixed_name(SyntaxKind::ScssVariable)
                }
                '@' if self.extension.dialect() == StyleDialect::Less => {
                    self.consume_less_at_name()
                }
                '@' => self.consume_at_keyword(),
                '!' => self.consume_static(SyntaxKind::Delim, start, 1),
                '.' if self.current_starts_number() => self.consume_number(),
                '.' => self.consume_static(SyntaxKind::Dot, start, 1),
                ',' => self.consume_static(SyntaxKind::Comma, start, 1),
                ':' if self.starts_with("::") => {
                    self.consume_static(SyntaxKind::DoubleColon, start, 2)
                }
                ':' => self.consume_static(SyntaxKind::Colon, start, 1),
                ';' => self.consume_static(SyntaxKind::Semicolon, start, 1),
                '{' => self.consume_static(SyntaxKind::LeftBrace, start, 1),
                '}' if self.template_interpolation_depth > 0 => {
                    self.consume_template_interpolation_end(start)
                }
                '}' if self.scss_interpolation_depth > 0 => {
                    self.consume_scss_interpolation_end(start)
                }
                '}' if self.less_interpolation_depth > 0 => {
                    self.consume_less_interpolation_end(start)
                }
                '}' => self.consume_static(SyntaxKind::RightBrace, start, 1),
                '(' => self.consume_static(SyntaxKind::LeftParen, start, 1),
                ')' => self.consume_static(SyntaxKind::RightParen, start, 1),
                '[' => self.consume_static(SyntaxKind::LeftBracket, start, 1),
                ']' => self.consume_static(SyntaxKind::RightBracket, start, 1),
                '+' if self.starts_with("+=") => {
                    self.consume_static(SyntaxKind::PlusEquals, start, 2)
                }
                '+' if self.current_starts_number() => self.consume_number(),
                '+' => self.consume_static(SyntaxKind::Plus, start, 1),
                '-' if self.starts_with("-=") => {
                    self.consume_static(SyntaxKind::MinusEquals, start, 2)
                }
                '-' if self.current_starts_number() => self.consume_number(),
                '-' if self.current_starts_ident_sequence() => self.consume_ident_like(),
                '-' => self.consume_static(SyntaxKind::Minus, start, 1),
                '*' if self.starts_with("*=") => {
                    self.consume_static(SyntaxKind::SubstringMatch, start, 2)
                }
                '*' => self.consume_static(SyntaxKind::Star, start, 1),
                '/' if self.starts_with("/=") => {
                    self.consume_static(SyntaxKind::SlashEquals, start, 2)
                }
                '/' => self.consume_static(SyntaxKind::Slash, start, 1),
                '%' if self.starts_scss_placeholder() => {
                    self.consume_prefixed_name(SyntaxKind::ScssPlaceholder)
                }
                '%' => self.consume_static(SyntaxKind::Percent, start, 1),
                '=' if self.starts_with("=>") => self.consume_static(SyntaxKind::Arrow, start, 2),
                '=' => self.consume_static(SyntaxKind::Equals, start, 1),
                '~' if self.starts_less_escaped_string() => self.consume_less_escaped_string(start),
                '~' if self.starts_with("~=") => {
                    self.consume_static(SyntaxKind::IncludesMatch, start, 2)
                }
                '~' => self.consume_static(SyntaxKind::Tilde, start, 1),
                '|' if self.starts_with("|=") => {
                    self.consume_static(SyntaxKind::DashMatch, start, 2)
                }
                '|' if self.starts_with("||") => {
                    self.consume_static(SyntaxKind::ColumnCombinator, start, 2)
                }
                '|' => self.consume_static(SyntaxKind::Pipe, start, 1),
                '^' if self.starts_with("^=") => {
                    self.consume_static(SyntaxKind::PrefixMatch, start, 2)
                }
                '^' => self.consume_static(SyntaxKind::Caret, start, 1),
                '$' if self.starts_with("$=") => {
                    self.consume_static(SyntaxKind::SuffixMatch, start, 2)
                }
                '$' if self.starts_less_property_variable() => {
                    self.consume_prefixed_name(SyntaxKind::LessPropertyVariableToken)
                }
                '&' if self.starts_with("&&") => {
                    self.consume_static(SyntaxKind::DoubleAmpersand, start, 2)
                }
                '&' => self.consume_static(SyntaxKind::Ampersand, start, 1),
                '>' => self.consume_static(SyntaxKind::GreaterThan, start, 1),
                '<' => self.consume_static(SyntaxKind::LessThan, start, 1),
                '#' if self.current_hash_starts_name() => self.consume_name_like(SyntaxKind::Hash),
                '#' => self.consume_static(SyntaxKind::Delim, start, 1),
                '\\' if self.current_starts_valid_escape() => {
                    self.consume_name_like(SyntaxKind::Ident)
                }
                char if is_name_start(char) => self.consume_ident_like(),
                char => self.consume_unexpected(char),
            }
        }
        self.consume_pending_sass_dedents();
    }
}

impl<'text, 'extension, E> Tokenizer<'text, 'extension, E>
where
    E: DialectExtension,
{
    fn consume_static(&mut self, kind: SyntaxKind, start: usize, byte_len: usize) {
        self.offset += byte_len;
        self.push(kind, start, self.offset);
    }

    fn consume_while(&mut self, kind: SyntaxKind, predicate: impl Fn(char) -> bool) {
        let start = self.offset;
        while let Some(char) = self.current_char() {
            if !predicate(char) {
                break;
            }
            self.bump_char(char);
        }
        self.push(kind, start, self.offset);
    }

    fn consume_block_comment(&mut self) {
        let start = self.offset;
        self.offset += 2;
        while self.offset < self.text.len() {
            if self.starts_with("*/") {
                self.offset += 2;
                self.push(SyntaxKind::BlockComment, start, self.offset);
                return;
            }
            match self.current_char() {
                Some(char) => self.bump_char(char),
                None => break,
            }
        }
        self.push(SyntaxKind::BlockComment, start, self.offset);
        self.error(
            ParseErrorCode::UnterminatedBlockComment,
            start,
            self.offset,
            "unterminated block comment",
        );
    }

    fn consume_line_comment(&mut self) {
        let start = self.offset;
        while let Some(char) = self.current_char() {
            if char == '\n' {
                break;
            }
            if char == '\r' {
                break;
            }
            self.bump_char(char);
        }
        self.push(SyntaxKind::LineComment, start, self.offset);
    }

    fn consume_sass_indented_newline(&mut self, start: usize) {
        self.consume_line_break();
        let indent = self.consume_sass_line_indent();
        let line_start = self.offset;
        let current_indent = self.sass_indent_stack.last().copied().unwrap_or(0);

        if indent > current_indent {
            self.push(SyntaxKind::SassIndentedNewline, start, line_start);
            self.sass_indent_stack.push(indent);
            self.push(SyntaxKind::SassIndent, line_start, line_start);
            return;
        }

        if self.previous_significant_sass_token_can_end_statement() {
            self.push(SyntaxKind::SassOptionalSemicolon, start, start);
        }
        self.push(SyntaxKind::SassIndentedNewline, start, line_start);

        while self.sass_indent_stack.len() > 1
            && self
                .sass_indent_stack
                .last()
                .is_some_and(|current| indent < *current)
        {
            self.sass_indent_stack.pop();
            self.push(SyntaxKind::SassDedent, line_start, line_start);
        }

        if self
            .sass_indent_stack
            .last()
            .is_some_and(|current| indent != *current)
        {
            self.error(
                ParseErrorCode::UnexpectedCharacter,
                line_start,
                line_start,
                "inconsistent Sass indentation",
            );
        }
    }

    fn consume_line_break(&mut self) {
        if self.starts_with("\r\n") {
            self.offset += "\r\n".len();
            return;
        }
        if let Some(char @ ('\r' | '\n')) = self.current_char() {
            self.bump_char(char);
        }
    }

    fn consume_sass_line_indent(&mut self) -> usize {
        let mut indent = 0usize;
        while let Some(char) = self.current_char() {
            match char {
                ' ' => {
                    indent += 1;
                    self.bump_char(char);
                }
                '\t' => {
                    indent += 4;
                    self.bump_char(char);
                }
                _ => break,
            }
        }
        indent
    }

    fn consume_pending_sass_dedents(&mut self) {
        if self.extension.dialect() != StyleDialect::Sass {
            return;
        }
        while self.sass_indent_stack.len() > 1 {
            self.sass_indent_stack.pop();
            self.push(SyntaxKind::SassDedent, self.offset, self.offset);
        }
    }

    fn previous_significant_sass_token_can_end_statement(&self) -> bool {
        self.tokens
            .iter()
            .rev()
            .find(|token| !token.kind.is_trivia())
            .is_some_and(|token| sass_token_can_end_statement(token.kind))
    }

    fn consume_scss_interpolation_start(&mut self, start: usize) {
        self.offset += "#{".len();
        self.scss_interpolation_depth += 1;
        self.push(SyntaxKind::ScssInterpolationStart, start, self.offset);
    }

    fn consume_template_interpolation_start(&mut self, start: usize) {
        self.offset += "${".len();
        self.template_interpolation_depth += 1;
        self.push(SyntaxKind::TemplateInterpolationStart, start, self.offset);
    }

    fn consume_scss_interpolation_end(&mut self, start: usize) {
        self.offset += '}'.len_utf8();
        self.scss_interpolation_depth = self.scss_interpolation_depth.saturating_sub(1);
        self.push(SyntaxKind::ScssInterpolationEnd, start, self.offset);
    }

    fn consume_template_interpolation_end(&mut self, start: usize) {
        self.offset += '}'.len_utf8();
        self.template_interpolation_depth = self.template_interpolation_depth.saturating_sub(1);
        self.push(SyntaxKind::TemplateInterpolationEnd, start, self.offset);
    }

    fn consume_less_interpolation_start(&mut self, start: usize) {
        self.offset += "@{".len();
        self.less_interpolation_depth += 1;
        self.push(SyntaxKind::LessInterpolationStart, start, self.offset);
    }

    fn consume_less_interpolation_end(&mut self, start: usize) {
        self.offset += '}'.len_utf8();
        self.less_interpolation_depth = self.less_interpolation_depth.saturating_sub(1);
        self.push(SyntaxKind::LessInterpolationEnd, start, self.offset);
    }

    fn consume_template_placeholder(&mut self, start: usize) {
        self.offset += "${".len();
        while let Some(char) = self.current_char() {
            self.bump_char(char);
            if char == '}' {
                break;
            }
        }
        self.push(SyntaxKind::TemplatePlaceholder, start, self.offset);
    }

    fn consume_string(&mut self, quote: char) {
        let start = self.offset;
        self.bump_char(quote);
        while let Some(char) = self.current_char() {
            self.bump_char(char);
            if matches!(char, '\n' | '\r' | '\u{000c}') {
                self.push(SyntaxKind::BadString, start, self.offset);
                self.error(
                    ParseErrorCode::UnterminatedString,
                    start,
                    self.offset,
                    "unterminated string",
                );
                return;
            }
            if char == quote {
                self.push(SyntaxKind::String, start, self.offset);
                return;
            }
            if char == '\\'
                && let Some(escaped) = self.current_char()
            {
                self.bump_char(escaped);
            }
        }
        self.push(SyntaxKind::BadString, start, self.offset);
        self.error(
            ParseErrorCode::UnterminatedString,
            start,
            self.offset,
            "unterminated string",
        );
    }

    fn consume_less_escaped_string(&mut self, start: usize) {
        self.offset += '~'.len_utf8();
        let Some(quote @ ('"' | '\'')) = self.current_char() else {
            self.push(SyntaxKind::Tilde, start, self.offset);
            return;
        };
        self.bump_char(quote);
        while let Some(char) = self.current_char() {
            self.bump_char(char);
            if matches!(char, '\n' | '\r' | '\u{000c}') {
                self.push(SyntaxKind::BadString, start, self.offset);
                self.error(
                    ParseErrorCode::UnterminatedString,
                    start,
                    self.offset,
                    "unterminated Less escaped string",
                );
                return;
            }
            if char == quote {
                self.push(SyntaxKind::LessEscapedString, start, self.offset);
                return;
            }
            if char == '\\'
                && let Some(escaped) = self.current_char()
            {
                self.bump_char(escaped);
            }
        }
        self.push(SyntaxKind::BadString, start, self.offset);
        self.error(
            ParseErrorCode::UnterminatedString,
            start,
            self.offset,
            "unterminated Less escaped string",
        );
    }

    fn consume_number(&mut self) {
        let start = self.offset;
        if matches!(self.current_char(), Some('+' | '-')) {
            self.bump_current();
        }
        self.consume_digits();
        if self.current_char() == Some('.') && self.char_after_current_is_ascii_digit() {
            self.bump_current();
            self.consume_digits();
        }
        if self.current_starts_number_exponent() {
            self.bump_current();
            if matches!(self.current_char(), Some('+' | '-')) {
                self.bump_current();
            }
            self.consume_digits();
        }
        if self.current_char() == Some('%') {
            self.offset += 1;
            self.push(SyntaxKind::Percentage, start, self.offset);
            return;
        }
        if self.current_starts_ident_sequence() {
            self.consume_name_continue_sequence();
            self.push(SyntaxKind::Dimension, start, self.offset);
            return;
        }
        self.push(SyntaxKind::Number, start, self.offset);
    }

    fn consume_unicode_range(&mut self) {
        let start = self.offset;
        self.bump_current();
        self.offset += '+'.len_utf8();
        self.consume_unicode_range_codepoints(true);
        if self.current_char() == Some('-') && self.next_char_is_hex_digit() {
            self.bump_current();
            self.consume_unicode_range_codepoints(false);
        }
        self.push(SyntaxKind::UnicodeRange, start, self.offset);
    }

    fn consume_unicode_range_codepoints(&mut self, allow_question_mark: bool) {
        let mut consumed = 0usize;
        while consumed < 6 {
            match self.current_char() {
                Some(char) if char.is_ascii_hexdigit() => {
                    self.bump_char(char);
                    consumed += 1;
                }
                Some('?') if allow_question_mark => {
                    self.bump_current();
                    consumed += 1;
                }
                _ => break,
            }
        }
    }

    fn consume_digits(&mut self) {
        while matches!(self.current_char(), Some('0'..='9')) {
            self.offset += 1;
        }
    }

    fn consume_prefixed_name(&mut self, preferred_kind: SyntaxKind) {
        let start = self.offset;
        self.bump_current();
        while matches!(self.current_char(), Some(char) if is_name_continue(char)) {
            self.bump_current();
        }
        let text = &self.text[start..self.offset];
        let kind = self
            .extension
            .classify_variable_token(text)
            .unwrap_or(preferred_kind);
        self.push(kind, start, self.offset);
    }

    fn consume_less_at_name(&mut self) {
        let start = self.offset;
        self.bump_current();
        while matches!(self.current_char(), Some(char) if is_name_continue(char)) {
            self.bump_current();
        }
        let text = &self.text[start..self.offset];
        let kind = if is_css_at_rule_name(text) {
            SyntaxKind::AtKeyword
        } else {
            self.extension
                .classify_variable_token(text)
                .unwrap_or(SyntaxKind::LessVariable)
        };
        self.push(kind, start, self.offset);
    }

    fn consume_at_keyword(&mut self) {
        let start = self.offset;
        self.bump_current();
        while matches!(self.current_char(), Some(char) if is_name_continue(char)) {
            self.bump_current();
        }
        self.push(SyntaxKind::AtKeyword, start, self.offset);
    }

    fn consume_name_like(&mut self, kind: SyntaxKind) {
        let start = self.offset;
        self.consume_name_start();
        self.consume_name_continue_sequence();
        self.push(kind, start, self.offset);
    }

    fn consume_ident_like(&mut self) {
        let start = self.offset;
        self.consume_name_continue_sequence();
        let ident = &self.text[start..self.offset];
        if matches_ignore_ascii_case(ident, &["url"])
            && self.current_char() == Some('(')
            && !self.url_starts_with_quoted_argument()
        {
            self.consume_url_token(start);
            return;
        }
        let kind = if is_custom_property_name_text(ident) {
            SyntaxKind::CustomPropertyName
        } else {
            SyntaxKind::Ident
        };
        self.push(kind, start, self.offset);
    }

    fn consume_name_start(&mut self) {
        if self.current_starts_valid_escape() {
            self.consume_name_escape();
        } else {
            self.bump_current();
        }
    }

    fn consume_name_continue_sequence(&mut self) {
        loop {
            if self.current_starts_valid_escape() {
                self.consume_name_escape();
            } else if matches!(self.current_char(), Some(char) if is_name_continue(char)) {
                self.bump_current();
            } else {
                break;
            }
        }
    }

    fn consume_name_escape(&mut self) {
        self.bump_current();
        let mut hex_digits = 0usize;
        while hex_digits < 6
            && matches!(self.current_char(), Some(char) if char.is_ascii_hexdigit())
        {
            self.bump_current();
            hex_digits += 1;
        }
        if hex_digits > 0 {
            if matches!(self.current_char(), Some(char) if char.is_whitespace()) {
                self.bump_current();
            }
        } else if self.current_char().is_some() {
            self.bump_current();
        }
    }

    fn consume_url_token(&mut self, start: usize) {
        self.bump_current();
        while matches!(self.current_char(), Some(char) if char.is_whitespace()) {
            self.bump_current();
        }
        while let Some(char) = self.current_char() {
            match char {
                ')' => {
                    self.bump_current();
                    self.push(SyntaxKind::Url, start, self.offset);
                    return;
                }
                char if char.is_whitespace() => {
                    self.bump_current();
                    while matches!(self.current_char(), Some(char) if char.is_whitespace()) {
                        self.bump_current();
                    }
                    if self.current_char() == Some(')') {
                        self.bump_current();
                        self.push(SyntaxKind::Url, start, self.offset);
                        return;
                    }
                    self.consume_bad_url(start);
                    return;
                }
                '"' | '\'' | '(' => {
                    self.consume_bad_url(start);
                    return;
                }
                '\\' if self.current_starts_valid_escape() => {
                    self.consume_name_escape();
                }
                '\\' => {
                    self.consume_bad_url(start);
                    return;
                }
                char if is_non_printable_code_point(char) => {
                    self.consume_bad_url(start);
                    return;
                }
                _ => self.bump_current(),
            }
        }
        self.push(SyntaxKind::BadUrl, start, self.offset);
        self.error(
            ParseErrorCode::UnexpectedCharacter,
            start,
            self.offset,
            "unterminated url token",
        );
    }

    fn consume_bad_url(&mut self, start: usize) {
        while let Some(char) = self.current_char() {
            if char == ')' {
                self.bump_current();
                break;
            }
            if self.current_starts_valid_escape() {
                self.consume_name_escape();
            } else {
                self.bump_current();
            }
        }
        self.push(SyntaxKind::BadUrl, start, self.offset);
        self.error(
            ParseErrorCode::UnexpectedCharacter,
            start,
            self.offset,
            "bad url token",
        );
    }

    fn url_starts_with_quoted_argument(&self) -> bool {
        let Some(mut rest) = self.text.get(self.offset + '('.len_utf8()..) else {
            return false;
        };
        rest = rest.trim_start_matches(char::is_whitespace);
        matches!(rest.chars().next(), Some('"' | '\''))
    }

    fn starts_less_property_variable(&self) -> bool {
        self.extension.dialect() == StyleDialect::Less
            && self.text[self.offset + '$'.len_utf8()..]
                .chars()
                .next()
                .is_some_and(is_name_start)
    }

    fn starts_scss_placeholder(&self) -> bool {
        matches!(
            self.extension.dialect(),
            StyleDialect::Scss | StyleDialect::Sass
        ) && self.text[self.offset + '%'.len_utf8()..]
            .chars()
            .next()
            .is_some_and(is_name_start)
    }

    fn current_hash_starts_name(&self) -> bool {
        if self.current_char() != Some('#') {
            return false;
        }
        let next_offset = self.offset + '#'.len_utf8();
        self.text[next_offset..]
            .chars()
            .next()
            .is_some_and(is_name_continue)
            || self.escape_starts_at(next_offset)
    }

    fn consume_unexpected(&mut self, char: char) {
        let start = self.offset;
        self.bump_char(char);
        self.push(SyntaxKind::Delim, start, self.offset);
        self.error(
            ParseErrorCode::UnexpectedCharacter,
            start,
            self.offset,
            "unexpected character",
        );
    }

    fn push(&mut self, kind: SyntaxKind, start: usize, end: usize) {
        self.tokens.push(Token {
            kind,
            text: &self.text[start..end],
            range: text_range(start, end),
        });
    }

    fn error(&mut self, code: ParseErrorCode, start: usize, end: usize, message: &'static str) {
        self.errors.push(ParseError {
            code,
            range: text_range(start, end),
            message,
        });
    }
}

impl<'text, 'extension, E> Tokenizer<'text, 'extension, E>
where
    E: DialectExtension,
{
    pub(crate) fn starts_with(&self, pattern: &str) -> bool {
        self.text[self.offset..].starts_with(pattern)
    }

    pub(crate) fn current_starts_valid_escape(&self) -> bool {
        self.escape_starts_at(self.offset)
    }

    pub(crate) fn current_starts_number(&self) -> bool {
        self.starts_number_at(self.offset)
    }

    pub(crate) fn current_starts_number_exponent(&self) -> bool {
        let Some('e' | 'E') = self.current_char() else {
            return false;
        };
        let exponent_offset = self.offset + 'e'.len_utf8();
        self.char_at(exponent_offset)
            .is_some_and(|char| char.is_ascii_digit())
            || (matches!(self.char_at(exponent_offset), Some('+' | '-'))
                && self.char_after_offset_is_ascii_digit(exponent_offset))
    }

    pub(crate) fn starts_number_at(&self, offset: usize) -> bool {
        let Some(first) = self.char_at(offset) else {
            return false;
        };
        let second_offset = offset + first.len_utf8();
        match first {
            '+' | '-' => {
                self.char_at(second_offset)
                    .is_some_and(|char| char.is_ascii_digit())
                    || (self.char_at(second_offset) == Some('.')
                        && self.char_after_offset_is_ascii_digit(second_offset))
            }
            '.' => self.char_after_offset_is_ascii_digit(offset),
            char => char.is_ascii_digit(),
        }
    }

    pub(crate) fn current_starts_ident_sequence(&self) -> bool {
        self.starts_ident_sequence_at(self.offset)
    }

    pub(crate) fn starts_ident_sequence_at(&self, offset: usize) -> bool {
        let Some(first) = self.char_at(offset) else {
            return false;
        };
        let second_offset = offset + first.len_utf8();
        match first {
            '-' => {
                self.char_at(second_offset)
                    .is_some_and(|char| char == '-' || is_name_start(char))
                    || self.escape_starts_at(second_offset)
            }
            '\\' => self.escape_starts_at(offset),
            char => is_name_start(char),
        }
    }

    pub(crate) fn escape_starts_at(&self, offset: usize) -> bool {
        if !self
            .text
            .get(offset..)
            .is_some_and(|remaining| remaining.starts_with('\\'))
        {
            return false;
        }
        self.text[offset + '\\'.len_utf8()..]
            .chars()
            .next()
            .is_some_and(|char| !matches!(char, '\n' | '\r' | '\u{000c}'))
    }

    pub(crate) fn char_at(&self, offset: usize) -> Option<char> {
        self.text.get(offset..)?.chars().next()
    }

    pub(crate) fn char_after_current_is_ascii_digit(&self) -> bool {
        self.char_after_offset_is_ascii_digit(self.offset)
    }

    pub(crate) fn char_after_offset_is_ascii_digit(&self, offset: usize) -> bool {
        let Some(char) = self.char_at(offset) else {
            return false;
        };
        self.char_at(offset + char.len_utf8())
            .is_some_and(|char| char.is_ascii_digit())
    }

    pub(crate) fn starts_with_ascii_keyword(&self, keyword: &str) -> bool {
        let remaining = &self.text[self.offset..];
        let Some(prefix) = remaining.get(..keyword.len()) else {
            return false;
        };
        if !matches_ignore_ascii_case(prefix, &[keyword]) {
            return false;
        }
        remaining[keyword.len()..]
            .chars()
            .next()
            .is_none_or(|char| !is_name_continue(char))
    }

    pub(crate) fn supports_scss_interpolation(&self) -> bool {
        matches!(
            self.extension.dialect(),
            StyleDialect::Scss | StyleDialect::Sass
        )
    }

    pub(crate) fn supports_less_interpolation(&self) -> bool {
        self.extension.dialect() == StyleDialect::Less
    }

    pub(crate) fn starts_less_escaped_string(&self) -> bool {
        self.extension.dialect() == StyleDialect::Less
            && (self.starts_with("~\"") || self.starts_with("~'"))
    }

    pub(crate) fn starts_unicode_range(&self) -> bool {
        let mut chars = self.text[self.offset..].chars();
        matches!(chars.next(), Some('u' | 'U'))
            && chars.next() == Some('+')
            && chars
                .next()
                .is_some_and(|char| char.is_ascii_hexdigit() || char == '?')
    }

    pub(crate) fn current_char(&self) -> Option<char> {
        self.text[self.offset..].chars().next()
    }

    pub(crate) fn next_char_is_hex_digit(&self) -> bool {
        let offset = self.offset + '-'.len_utf8();
        self.text
            .get(offset..)
            .and_then(|tail| tail.chars().next())
            .is_some_and(|char| char.is_ascii_hexdigit())
    }

    pub(crate) fn bump_current(&mut self) {
        if let Some(char) = self.current_char() {
            self.bump_char(char);
        }
    }

    pub(crate) fn bump_char(&mut self, char: char) {
        self.offset += char.len_utf8();
    }
}

fn css_syntax_preprocessed_char(char: char) -> char {
    if char == '\0' { '\u{fffd}' } else { char }
}

fn starts_valid_escape_text(text: &str) -> bool {
    text.starts_with('\\')
        && text['\\'.len_utf8()..]
            .chars()
            .next()
            .is_some_and(|char| !matches!(char, '\n' | '\r' | '\u{000c}'))
}