1use cstree::text::{TextRange, TextSize};
7use omena_syntax::{
8 StyleDialect, SyntaxKind,
9 ident::{is_css_name_continue, is_css_name_start},
10};
11
12use crate::{
13 DialectExtension, ParseError, ParseErrorCode, TemplatePlaceholderMode,
14 matches_ignore_ascii_case,
15};
16
17#[derive(Debug, Clone, Copy, PartialEq, Eq)]
18pub(crate) struct Token<'text> {
19 pub(crate) kind: SyntaxKind,
20 pub(crate) text: &'text str,
21 pub(crate) range: TextRange,
22}
23
24pub(crate) struct Tokenizer<'text, 'extension, E> {
25 pub(crate) text: &'text str,
26 pub(crate) extension: &'extension E,
27 pub(crate) offset: usize,
28 pub(crate) template_placeholder: Option<TemplatePlaceholderMode>,
29 pub(crate) template_interpolation_depth: usize,
30 pub(crate) scss_interpolation_depth: usize,
31 pub(crate) less_interpolation_depth: usize,
32 pub(crate) sass_indent_stack: Vec<usize>,
33 pub(crate) tokens: Vec<Token<'text>>,
34 pub(crate) errors: Vec<ParseError>,
35}
36
37#[derive(Debug, Clone, PartialEq, Eq)]
38pub struct LexResult {
39 tokens: Vec<LexedToken>,
40 errors: Vec<ParseError>,
41 dialect: StyleDialect,
42}
43
44impl LexResult {
45 pub(crate) fn new(
46 tokens: Vec<LexedToken>,
47 errors: Vec<ParseError>,
48 dialect: StyleDialect,
49 ) -> Self {
50 Self {
51 tokens,
52 errors,
53 dialect,
54 }
55 }
56
57 pub fn tokens(&self) -> &[LexedToken] {
58 &self.tokens
59 }
60
61 pub fn errors(&self) -> &[ParseError] {
62 &self.errors
63 }
64
65 pub fn dialect(&self) -> StyleDialect {
66 self.dialect
67 }
68}
69
70#[derive(Debug, Clone, PartialEq, Eq)]
71pub struct LexedToken {
72 pub kind: SyntaxKind,
73 pub range: TextRange,
74 pub text: String,
75}
76
77pub(crate) fn public_token_text(text: &str) -> String {
78 text.chars()
79 .map(css_syntax_preprocessed_char)
80 .collect::<String>()
81}
82
83pub(crate) fn is_name_start(char: char) -> bool {
84 is_css_name_start(css_syntax_preprocessed_char(char))
85}
86
87pub(crate) fn is_name_continue(char: char) -> bool {
88 is_css_name_continue(css_syntax_preprocessed_char(char))
89}
90
91pub(crate) fn is_non_printable_code_point(char: char) -> bool {
92 let char = css_syntax_preprocessed_char(char);
93 matches!(char, '\u{0000}'..='\u{0008}' | '\u{000b}' | '\u{000e}'..='\u{001f}' | '\u{007f}')
94}
95
96pub(crate) fn is_custom_property_name_text(text: &str) -> bool {
97 let Some(rest) = text.strip_prefix("--") else {
98 return false;
99 };
100 let Some(first) = rest.chars().next() else {
101 return false;
102 };
103 first == '-' || is_name_start(first) || starts_valid_escape_text(rest)
104}
105
106pub(crate) fn is_css_at_rule_name(text: &str) -> bool {
107 matches_ignore_ascii_case(
108 text,
109 &[
110 "@charset",
111 "@container",
112 "@font-face",
113 "@font-feature-values",
114 "@function",
115 "@font-palette-values",
116 "@import",
117 "@keyframes",
118 "@layer",
119 "@media",
120 "@namespace",
121 "@page",
122 "@property",
123 "@scope",
124 "@starting-style",
125 "@supports",
126 "@counter-style",
127 "@custom-media",
128 "@color-profile",
129 "@nest",
130 "@position-try",
131 "@view-transition",
132 "@stylistic",
133 "@styleset",
134 "@character-variant",
135 "@swash",
136 "@ornaments",
137 "@annotation",
138 "@historical-forms",
139 "@when",
140 "@else",
141 ],
142 )
143}
144
145pub(crate) fn sass_token_can_end_statement(kind: SyntaxKind) -> bool {
146 !matches!(
147 kind,
148 SyntaxKind::Whitespace
149 | SyntaxKind::LineComment
150 | SyntaxKind::BlockComment
151 | SyntaxKind::SassIndentedNewline
152 | SyntaxKind::SassIndent
153 | SyntaxKind::SassDedent
154 | SyntaxKind::SassOptionalSemicolon
155 | SyntaxKind::Comma
156 | SyntaxKind::Colon
157 | SyntaxKind::DoubleColon
158 | SyntaxKind::LeftBrace
159 | SyntaxKind::LeftParen
160 | SyntaxKind::LeftBracket
161 | SyntaxKind::Plus
162 | SyntaxKind::Minus
163 | SyntaxKind::Star
164 | SyntaxKind::Slash
165 | SyntaxKind::GreaterThan
166 | SyntaxKind::LessThan
167 | SyntaxKind::Equals
168 | SyntaxKind::Arrow
169 | SyntaxKind::Pipe
170 | SyntaxKind::Tilde
171 | SyntaxKind::Caret
172 | SyntaxKind::Ampersand
173 | SyntaxKind::DoubleAmpersand
174 | SyntaxKind::ColumnCombinator
175 | SyntaxKind::IncludesMatch
176 | SyntaxKind::DashMatch
177 | SyntaxKind::PrefixMatch
178 | SyntaxKind::SuffixMatch
179 | SyntaxKind::SubstringMatch
180 | SyntaxKind::PlusEquals
181 | SyntaxKind::MinusEquals
182 | SyntaxKind::SlashEquals
183 )
184}
185
186pub(crate) fn text_range(start: usize, end: usize) -> TextRange {
187 TextRange::new(TextSize::from(start as u32), TextSize::from(end as u32))
188}
189
190impl<'text, 'extension, E> Tokenizer<'text, 'extension, E>
191where
192 E: DialectExtension,
193{
194 pub(crate) fn new(text: &'text str, extension: &'extension E) -> Self {
195 Self {
196 text,
197 extension,
198 offset: 0,
199 template_placeholder: extension.template_placeholder(),
200 template_interpolation_depth: 0,
201 scss_interpolation_depth: 0,
202 less_interpolation_depth: 0,
203 sass_indent_stack: vec![0],
204 tokens: Vec::new(),
205 errors: Vec::new(),
206 }
207 }
208
209 pub(crate) fn tokenize(&mut self) {
210 while let Some(current) = self.current_char() {
211 let start = self.offset;
212 match current {
213 '\u{feff}' if start == 0 => self.bump_current(),
214 '\r' | '\n' if self.extension.dialect() == StyleDialect::Sass => {
215 self.consume_sass_indented_newline(start)
216 }
217 char if char.is_whitespace() => {
218 self.consume_while(SyntaxKind::Whitespace, |c| c.is_whitespace())
219 }
220 '/' if self.starts_with("/*") => self.consume_block_comment(),
221 '/' if self.starts_with("//") && self.extension.dialect() != StyleDialect::Css => {
222 self.consume_line_comment()
223 }
224 '$' if self.starts_with("${")
225 && self.template_placeholder == Some(TemplatePlaceholderMode::Brace) =>
226 {
227 self.consume_template_interpolation_start(start)
228 }
229 '$' if self.starts_with("${")
230 && self.template_placeholder
231 == Some(TemplatePlaceholderMode::AtomicIndexed) =>
232 {
233 self.consume_template_placeholder(start)
234 }
235 '#' if self.starts_with("#{") && self.supports_scss_interpolation() => {
236 self.consume_scss_interpolation_start(start)
237 }
238 '@' if self.starts_with("@{") && self.supports_less_interpolation() => {
239 self.consume_less_interpolation_start(start)
240 }
241 '!' if self.starts_with_ascii_keyword("!important") => {
242 self.consume_static(SyntaxKind::Important, start, "!important".len())
243 }
244 '<' if self.starts_with("<!--") => {
245 self.consume_static(SyntaxKind::Cdo, start, "<!--".len())
246 }
247 '-' if self.starts_with("-->") => {
248 self.consume_static(SyntaxKind::Cdc, start, "-->".len())
249 }
250 '"' | '\'' => self.consume_string(current),
251 'u' | 'U' if self.starts_unicode_range() => self.consume_unicode_range(),
252 '0'..='9' => self.consume_number(),
253 '$' if matches!(
254 self.extension.dialect(),
255 StyleDialect::Scss | StyleDialect::Sass
256 ) =>
257 {
258 self.consume_prefixed_name(SyntaxKind::ScssVariable)
259 }
260 '@' if self.extension.dialect() == StyleDialect::Less => {
261 self.consume_less_at_name()
262 }
263 '@' => self.consume_at_keyword(),
264 '!' => self.consume_static(SyntaxKind::Delim, start, 1),
265 '.' if self.current_starts_number() => self.consume_number(),
266 '.' => self.consume_static(SyntaxKind::Dot, start, 1),
267 ',' => self.consume_static(SyntaxKind::Comma, start, 1),
268 ':' if self.starts_with("::") => {
269 self.consume_static(SyntaxKind::DoubleColon, start, 2)
270 }
271 ':' => self.consume_static(SyntaxKind::Colon, start, 1),
272 ';' => self.consume_static(SyntaxKind::Semicolon, start, 1),
273 '{' => self.consume_static(SyntaxKind::LeftBrace, start, 1),
274 '}' if self.template_interpolation_depth > 0 => {
275 self.consume_template_interpolation_end(start)
276 }
277 '}' if self.scss_interpolation_depth > 0 => {
278 self.consume_scss_interpolation_end(start)
279 }
280 '}' if self.less_interpolation_depth > 0 => {
281 self.consume_less_interpolation_end(start)
282 }
283 '}' => self.consume_static(SyntaxKind::RightBrace, start, 1),
284 '(' => self.consume_static(SyntaxKind::LeftParen, start, 1),
285 ')' => self.consume_static(SyntaxKind::RightParen, start, 1),
286 '[' => self.consume_static(SyntaxKind::LeftBracket, start, 1),
287 ']' => self.consume_static(SyntaxKind::RightBracket, start, 1),
288 '+' if self.starts_with("+=") => {
289 self.consume_static(SyntaxKind::PlusEquals, start, 2)
290 }
291 '+' if self.current_starts_number() => self.consume_number(),
292 '+' => self.consume_static(SyntaxKind::Plus, start, 1),
293 '-' if self.starts_with("-=") => {
294 self.consume_static(SyntaxKind::MinusEquals, start, 2)
295 }
296 '-' if self.current_starts_number() => self.consume_number(),
297 '-' if self.current_starts_ident_sequence() => self.consume_ident_like(),
298 '-' => self.consume_static(SyntaxKind::Minus, start, 1),
299 '*' if self.starts_with("*=") => {
300 self.consume_static(SyntaxKind::SubstringMatch, start, 2)
301 }
302 '*' => self.consume_static(SyntaxKind::Star, start, 1),
303 '/' if self.starts_with("/=") => {
304 self.consume_static(SyntaxKind::SlashEquals, start, 2)
305 }
306 '/' => self.consume_static(SyntaxKind::Slash, start, 1),
307 '%' if self.starts_scss_placeholder() => {
308 self.consume_prefixed_name(SyntaxKind::ScssPlaceholder)
309 }
310 '%' => self.consume_static(SyntaxKind::Percent, start, 1),
311 '=' if self.starts_with("=>") => self.consume_static(SyntaxKind::Arrow, start, 2),
312 '=' => self.consume_static(SyntaxKind::Equals, start, 1),
313 '~' if self.starts_less_escaped_string() => self.consume_less_escaped_string(start),
314 '~' if self.starts_with("~=") => {
315 self.consume_static(SyntaxKind::IncludesMatch, start, 2)
316 }
317 '~' => self.consume_static(SyntaxKind::Tilde, start, 1),
318 '|' if self.starts_with("|=") => {
319 self.consume_static(SyntaxKind::DashMatch, start, 2)
320 }
321 '|' if self.starts_with("||") => {
322 self.consume_static(SyntaxKind::ColumnCombinator, start, 2)
323 }
324 '|' => self.consume_static(SyntaxKind::Pipe, start, 1),
325 '^' if self.starts_with("^=") => {
326 self.consume_static(SyntaxKind::PrefixMatch, start, 2)
327 }
328 '^' => self.consume_static(SyntaxKind::Caret, start, 1),
329 '$' if self.starts_with("$=") => {
330 self.consume_static(SyntaxKind::SuffixMatch, start, 2)
331 }
332 '$' if self.starts_less_property_variable() => {
333 self.consume_prefixed_name(SyntaxKind::LessPropertyVariableToken)
334 }
335 '&' if self.starts_with("&&") => {
336 self.consume_static(SyntaxKind::DoubleAmpersand, start, 2)
337 }
338 '&' => self.consume_static(SyntaxKind::Ampersand, start, 1),
339 '>' => self.consume_static(SyntaxKind::GreaterThan, start, 1),
340 '<' => self.consume_static(SyntaxKind::LessThan, start, 1),
341 '#' if self.current_hash_starts_name() => self.consume_name_like(SyntaxKind::Hash),
342 '#' => self.consume_static(SyntaxKind::Delim, start, 1),
343 '\\' if self.current_starts_valid_escape() => {
344 self.consume_name_like(SyntaxKind::Ident)
345 }
346 char if is_name_start(char) => self.consume_ident_like(),
347 char => self.consume_unexpected(char),
348 }
349 }
350 self.consume_pending_sass_dedents();
351 }
352}
353
354impl<'text, 'extension, E> Tokenizer<'text, 'extension, E>
355where
356 E: DialectExtension,
357{
358 fn consume_static(&mut self, kind: SyntaxKind, start: usize, byte_len: usize) {
359 self.offset += byte_len;
360 self.push(kind, start, self.offset);
361 }
362
363 fn consume_while(&mut self, kind: SyntaxKind, predicate: impl Fn(char) -> bool) {
364 let start = self.offset;
365 while let Some(char) = self.current_char() {
366 if !predicate(char) {
367 break;
368 }
369 self.bump_char(char);
370 }
371 self.push(kind, start, self.offset);
372 }
373
374 fn consume_block_comment(&mut self) {
375 let start = self.offset;
376 self.offset += 2;
377 while self.offset < self.text.len() {
378 if self.starts_with("*/") {
379 self.offset += 2;
380 self.push(SyntaxKind::BlockComment, start, self.offset);
381 return;
382 }
383 match self.current_char() {
384 Some(char) => self.bump_char(char),
385 None => break,
386 }
387 }
388 self.push(SyntaxKind::BlockComment, start, self.offset);
389 self.error(
390 ParseErrorCode::UnterminatedBlockComment,
391 start,
392 self.offset,
393 "unterminated block comment",
394 );
395 }
396
397 fn consume_line_comment(&mut self) {
398 let start = self.offset;
399 while let Some(char) = self.current_char() {
400 if char == '\n' {
401 break;
402 }
403 if char == '\r' {
404 break;
405 }
406 self.bump_char(char);
407 }
408 self.push(SyntaxKind::LineComment, start, self.offset);
409 }
410
411 fn consume_sass_indented_newline(&mut self, start: usize) {
412 self.consume_line_break();
413 let indent = self.consume_sass_line_indent();
414 let line_start = self.offset;
415 let current_indent = self.sass_indent_stack.last().copied().unwrap_or(0);
416
417 if indent > current_indent {
418 self.push(SyntaxKind::SassIndentedNewline, start, line_start);
419 self.sass_indent_stack.push(indent);
420 self.push(SyntaxKind::SassIndent, line_start, line_start);
421 return;
422 }
423
424 if self.previous_significant_sass_token_can_end_statement() {
425 self.push(SyntaxKind::SassOptionalSemicolon, start, start);
426 }
427 self.push(SyntaxKind::SassIndentedNewline, start, line_start);
428
429 while self.sass_indent_stack.len() > 1
430 && self
431 .sass_indent_stack
432 .last()
433 .is_some_and(|current| indent < *current)
434 {
435 self.sass_indent_stack.pop();
436 self.push(SyntaxKind::SassDedent, line_start, line_start);
437 }
438
439 if self
440 .sass_indent_stack
441 .last()
442 .is_some_and(|current| indent != *current)
443 {
444 self.error(
445 ParseErrorCode::UnexpectedCharacter,
446 line_start,
447 line_start,
448 "inconsistent Sass indentation",
449 );
450 }
451 }
452
453 fn consume_line_break(&mut self) {
454 if self.starts_with("\r\n") {
455 self.offset += "\r\n".len();
456 return;
457 }
458 if let Some(char @ ('\r' | '\n')) = self.current_char() {
459 self.bump_char(char);
460 }
461 }
462
463 fn consume_sass_line_indent(&mut self) -> usize {
464 let mut indent = 0usize;
465 while let Some(char) = self.current_char() {
466 match char {
467 ' ' => {
468 indent += 1;
469 self.bump_char(char);
470 }
471 '\t' => {
472 indent += 4;
473 self.bump_char(char);
474 }
475 _ => break,
476 }
477 }
478 indent
479 }
480
481 fn consume_pending_sass_dedents(&mut self) {
482 if self.extension.dialect() != StyleDialect::Sass {
483 return;
484 }
485 while self.sass_indent_stack.len() > 1 {
486 self.sass_indent_stack.pop();
487 self.push(SyntaxKind::SassDedent, self.offset, self.offset);
488 }
489 }
490
491 fn previous_significant_sass_token_can_end_statement(&self) -> bool {
492 self.tokens
493 .iter()
494 .rev()
495 .find(|token| !token.kind.is_trivia())
496 .is_some_and(|token| sass_token_can_end_statement(token.kind))
497 }
498
499 fn consume_scss_interpolation_start(&mut self, start: usize) {
500 self.offset += "#{".len();
501 self.scss_interpolation_depth += 1;
502 self.push(SyntaxKind::ScssInterpolationStart, start, self.offset);
503 }
504
505 fn consume_template_interpolation_start(&mut self, start: usize) {
506 self.offset += "${".len();
507 self.template_interpolation_depth += 1;
508 self.push(SyntaxKind::TemplateInterpolationStart, start, self.offset);
509 }
510
511 fn consume_scss_interpolation_end(&mut self, start: usize) {
512 self.offset += '}'.len_utf8();
513 self.scss_interpolation_depth = self.scss_interpolation_depth.saturating_sub(1);
514 self.push(SyntaxKind::ScssInterpolationEnd, start, self.offset);
515 }
516
517 fn consume_template_interpolation_end(&mut self, start: usize) {
518 self.offset += '}'.len_utf8();
519 self.template_interpolation_depth = self.template_interpolation_depth.saturating_sub(1);
520 self.push(SyntaxKind::TemplateInterpolationEnd, start, self.offset);
521 }
522
523 fn consume_less_interpolation_start(&mut self, start: usize) {
524 self.offset += "@{".len();
525 self.less_interpolation_depth += 1;
526 self.push(SyntaxKind::LessInterpolationStart, start, self.offset);
527 }
528
529 fn consume_less_interpolation_end(&mut self, start: usize) {
530 self.offset += '}'.len_utf8();
531 self.less_interpolation_depth = self.less_interpolation_depth.saturating_sub(1);
532 self.push(SyntaxKind::LessInterpolationEnd, start, self.offset);
533 }
534
535 fn consume_template_placeholder(&mut self, start: usize) {
536 self.offset += "${".len();
537 while let Some(char) = self.current_char() {
538 self.bump_char(char);
539 if char == '}' {
540 break;
541 }
542 }
543 self.push(SyntaxKind::TemplatePlaceholder, start, self.offset);
544 }
545
546 fn consume_string(&mut self, quote: char) {
547 let start = self.offset;
548 self.bump_char(quote);
549 while let Some(char) = self.current_char() {
550 self.bump_char(char);
551 if matches!(char, '\n' | '\r' | '\u{000c}') {
552 self.push(SyntaxKind::BadString, start, self.offset);
553 self.error(
554 ParseErrorCode::UnterminatedString,
555 start,
556 self.offset,
557 "unterminated string",
558 );
559 return;
560 }
561 if char == quote {
562 self.push(SyntaxKind::String, start, self.offset);
563 return;
564 }
565 if char == '\\'
566 && let Some(escaped) = self.current_char()
567 {
568 self.bump_char(escaped);
569 }
570 }
571 self.push(SyntaxKind::BadString, start, self.offset);
572 self.error(
573 ParseErrorCode::UnterminatedString,
574 start,
575 self.offset,
576 "unterminated string",
577 );
578 }
579
580 fn consume_less_escaped_string(&mut self, start: usize) {
581 self.offset += '~'.len_utf8();
582 let Some(quote @ ('"' | '\'')) = self.current_char() else {
583 self.push(SyntaxKind::Tilde, start, self.offset);
584 return;
585 };
586 self.bump_char(quote);
587 while let Some(char) = self.current_char() {
588 self.bump_char(char);
589 if matches!(char, '\n' | '\r' | '\u{000c}') {
590 self.push(SyntaxKind::BadString, start, self.offset);
591 self.error(
592 ParseErrorCode::UnterminatedString,
593 start,
594 self.offset,
595 "unterminated Less escaped string",
596 );
597 return;
598 }
599 if char == quote {
600 self.push(SyntaxKind::LessEscapedString, start, self.offset);
601 return;
602 }
603 if char == '\\'
604 && let Some(escaped) = self.current_char()
605 {
606 self.bump_char(escaped);
607 }
608 }
609 self.push(SyntaxKind::BadString, start, self.offset);
610 self.error(
611 ParseErrorCode::UnterminatedString,
612 start,
613 self.offset,
614 "unterminated Less escaped string",
615 );
616 }
617
618 fn consume_number(&mut self) {
619 let start = self.offset;
620 if matches!(self.current_char(), Some('+' | '-')) {
621 self.bump_current();
622 }
623 self.consume_digits();
624 if self.current_char() == Some('.') && self.char_after_current_is_ascii_digit() {
625 self.bump_current();
626 self.consume_digits();
627 }
628 if self.current_starts_number_exponent() {
629 self.bump_current();
630 if matches!(self.current_char(), Some('+' | '-')) {
631 self.bump_current();
632 }
633 self.consume_digits();
634 }
635 if self.current_char() == Some('%') {
636 self.offset += 1;
637 self.push(SyntaxKind::Percentage, start, self.offset);
638 return;
639 }
640 if self.current_starts_ident_sequence() {
641 self.consume_name_continue_sequence();
642 self.push(SyntaxKind::Dimension, start, self.offset);
643 return;
644 }
645 self.push(SyntaxKind::Number, start, self.offset);
646 }
647
648 fn consume_unicode_range(&mut self) {
649 let start = self.offset;
650 self.bump_current();
651 self.offset += '+'.len_utf8();
652 self.consume_unicode_range_codepoints(true);
653 if self.current_char() == Some('-') && self.next_char_is_hex_digit() {
654 self.bump_current();
655 self.consume_unicode_range_codepoints(false);
656 }
657 self.push(SyntaxKind::UnicodeRange, start, self.offset);
658 }
659
660 fn consume_unicode_range_codepoints(&mut self, allow_question_mark: bool) {
661 let mut consumed = 0usize;
662 while consumed < 6 {
663 match self.current_char() {
664 Some(char) if char.is_ascii_hexdigit() => {
665 self.bump_char(char);
666 consumed += 1;
667 }
668 Some('?') if allow_question_mark => {
669 self.bump_current();
670 consumed += 1;
671 }
672 _ => break,
673 }
674 }
675 }
676
677 fn consume_digits(&mut self) {
678 while matches!(self.current_char(), Some('0'..='9')) {
679 self.offset += 1;
680 }
681 }
682
683 fn consume_prefixed_name(&mut self, preferred_kind: SyntaxKind) {
684 let start = self.offset;
685 self.bump_current();
686 while matches!(self.current_char(), Some(char) if is_name_continue(char)) {
687 self.bump_current();
688 }
689 let text = &self.text[start..self.offset];
690 let kind = self
691 .extension
692 .classify_variable_token(text)
693 .unwrap_or(preferred_kind);
694 self.push(kind, start, self.offset);
695 }
696
697 fn consume_less_at_name(&mut self) {
698 let start = self.offset;
699 self.bump_current();
700 while matches!(self.current_char(), Some(char) if is_name_continue(char)) {
701 self.bump_current();
702 }
703 let text = &self.text[start..self.offset];
704 let kind = if is_css_at_rule_name(text) {
705 SyntaxKind::AtKeyword
706 } else {
707 self.extension
708 .classify_variable_token(text)
709 .unwrap_or(SyntaxKind::LessVariable)
710 };
711 self.push(kind, start, self.offset);
712 }
713
714 fn consume_at_keyword(&mut self) {
715 let start = self.offset;
716 self.bump_current();
717 while matches!(self.current_char(), Some(char) if is_name_continue(char)) {
718 self.bump_current();
719 }
720 self.push(SyntaxKind::AtKeyword, start, self.offset);
721 }
722
723 fn consume_name_like(&mut self, kind: SyntaxKind) {
724 let start = self.offset;
725 self.consume_name_start();
726 self.consume_name_continue_sequence();
727 self.push(kind, start, self.offset);
728 }
729
730 fn consume_ident_like(&mut self) {
731 let start = self.offset;
732 self.consume_name_continue_sequence();
733 let ident = &self.text[start..self.offset];
734 if matches_ignore_ascii_case(ident, &["url"])
735 && self.current_char() == Some('(')
736 && !self.url_starts_with_quoted_argument()
737 {
738 self.consume_url_token(start);
739 return;
740 }
741 let kind = if is_custom_property_name_text(ident) {
742 SyntaxKind::CustomPropertyName
743 } else {
744 SyntaxKind::Ident
745 };
746 self.push(kind, start, self.offset);
747 }
748
749 fn consume_name_start(&mut self) {
750 if self.current_starts_valid_escape() {
751 self.consume_name_escape();
752 } else {
753 self.bump_current();
754 }
755 }
756
757 fn consume_name_continue_sequence(&mut self) {
758 loop {
759 if self.current_starts_valid_escape() {
760 self.consume_name_escape();
761 } else if matches!(self.current_char(), Some(char) if is_name_continue(char)) {
762 self.bump_current();
763 } else {
764 break;
765 }
766 }
767 }
768
769 fn consume_name_escape(&mut self) {
770 self.bump_current();
771 let mut hex_digits = 0usize;
772 while hex_digits < 6
773 && matches!(self.current_char(), Some(char) if char.is_ascii_hexdigit())
774 {
775 self.bump_current();
776 hex_digits += 1;
777 }
778 if hex_digits > 0 {
779 if matches!(self.current_char(), Some(char) if char.is_whitespace()) {
780 self.bump_current();
781 }
782 } else if self.current_char().is_some() {
783 self.bump_current();
784 }
785 }
786
787 fn consume_url_token(&mut self, start: usize) {
788 self.bump_current();
789 while matches!(self.current_char(), Some(char) if char.is_whitespace()) {
790 self.bump_current();
791 }
792 while let Some(char) = self.current_char() {
793 match char {
794 ')' => {
795 self.bump_current();
796 self.push(SyntaxKind::Url, start, self.offset);
797 return;
798 }
799 char if char.is_whitespace() => {
800 self.bump_current();
801 while matches!(self.current_char(), Some(char) if char.is_whitespace()) {
802 self.bump_current();
803 }
804 if self.current_char() == Some(')') {
805 self.bump_current();
806 self.push(SyntaxKind::Url, start, self.offset);
807 return;
808 }
809 self.consume_bad_url(start);
810 return;
811 }
812 '"' | '\'' | '(' => {
813 self.consume_bad_url(start);
814 return;
815 }
816 '\\' if self.current_starts_valid_escape() => {
817 self.consume_name_escape();
818 }
819 '\\' => {
820 self.consume_bad_url(start);
821 return;
822 }
823 char if is_non_printable_code_point(char) => {
824 self.consume_bad_url(start);
825 return;
826 }
827 _ => self.bump_current(),
828 }
829 }
830 self.push(SyntaxKind::BadUrl, start, self.offset);
831 self.error(
832 ParseErrorCode::UnexpectedCharacter,
833 start,
834 self.offset,
835 "unterminated url token",
836 );
837 }
838
839 fn consume_bad_url(&mut self, start: usize) {
840 while let Some(char) = self.current_char() {
841 if char == ')' {
842 self.bump_current();
843 break;
844 }
845 if self.current_starts_valid_escape() {
846 self.consume_name_escape();
847 } else {
848 self.bump_current();
849 }
850 }
851 self.push(SyntaxKind::BadUrl, start, self.offset);
852 self.error(
853 ParseErrorCode::UnexpectedCharacter,
854 start,
855 self.offset,
856 "bad url token",
857 );
858 }
859
860 fn url_starts_with_quoted_argument(&self) -> bool {
861 let Some(mut rest) = self.text.get(self.offset + '('.len_utf8()..) else {
862 return false;
863 };
864 rest = rest.trim_start_matches(char::is_whitespace);
865 matches!(rest.chars().next(), Some('"' | '\''))
866 }
867
868 fn starts_less_property_variable(&self) -> bool {
869 self.extension.dialect() == StyleDialect::Less
870 && self.text[self.offset + '$'.len_utf8()..]
871 .chars()
872 .next()
873 .is_some_and(is_name_start)
874 }
875
876 fn starts_scss_placeholder(&self) -> bool {
877 matches!(
878 self.extension.dialect(),
879 StyleDialect::Scss | StyleDialect::Sass
880 ) && self.text[self.offset + '%'.len_utf8()..]
881 .chars()
882 .next()
883 .is_some_and(is_name_start)
884 }
885
886 fn current_hash_starts_name(&self) -> bool {
887 if self.current_char() != Some('#') {
888 return false;
889 }
890 let next_offset = self.offset + '#'.len_utf8();
891 self.text[next_offset..]
892 .chars()
893 .next()
894 .is_some_and(is_name_continue)
895 || self.escape_starts_at(next_offset)
896 }
897
898 fn consume_unexpected(&mut self, char: char) {
899 let start = self.offset;
900 self.bump_char(char);
901 self.push(SyntaxKind::Delim, start, self.offset);
902 self.error(
903 ParseErrorCode::UnexpectedCharacter,
904 start,
905 self.offset,
906 "unexpected character",
907 );
908 }
909
910 fn push(&mut self, kind: SyntaxKind, start: usize, end: usize) {
911 self.tokens.push(Token {
912 kind,
913 text: &self.text[start..end],
914 range: text_range(start, end),
915 });
916 }
917
918 fn error(&mut self, code: ParseErrorCode, start: usize, end: usize, message: &'static str) {
919 self.errors.push(ParseError {
920 code,
921 range: text_range(start, end),
922 message,
923 });
924 }
925}
926
927impl<'text, 'extension, E> Tokenizer<'text, 'extension, E>
928where
929 E: DialectExtension,
930{
931 pub(crate) fn starts_with(&self, pattern: &str) -> bool {
932 self.text[self.offset..].starts_with(pattern)
933 }
934
935 pub(crate) fn current_starts_valid_escape(&self) -> bool {
936 self.escape_starts_at(self.offset)
937 }
938
939 pub(crate) fn current_starts_number(&self) -> bool {
940 self.starts_number_at(self.offset)
941 }
942
943 pub(crate) fn current_starts_number_exponent(&self) -> bool {
944 let Some('e' | 'E') = self.current_char() else {
945 return false;
946 };
947 let exponent_offset = self.offset + 'e'.len_utf8();
948 self.char_at(exponent_offset)
949 .is_some_and(|char| char.is_ascii_digit())
950 || (matches!(self.char_at(exponent_offset), Some('+' | '-'))
951 && self.char_after_offset_is_ascii_digit(exponent_offset))
952 }
953
954 pub(crate) fn starts_number_at(&self, offset: usize) -> bool {
955 let Some(first) = self.char_at(offset) else {
956 return false;
957 };
958 let second_offset = offset + first.len_utf8();
959 match first {
960 '+' | '-' => {
961 self.char_at(second_offset)
962 .is_some_and(|char| char.is_ascii_digit())
963 || (self.char_at(second_offset) == Some('.')
964 && self.char_after_offset_is_ascii_digit(second_offset))
965 }
966 '.' => self.char_after_offset_is_ascii_digit(offset),
967 char => char.is_ascii_digit(),
968 }
969 }
970
971 pub(crate) fn current_starts_ident_sequence(&self) -> bool {
972 self.starts_ident_sequence_at(self.offset)
973 }
974
975 pub(crate) fn starts_ident_sequence_at(&self, offset: usize) -> bool {
976 let Some(first) = self.char_at(offset) else {
977 return false;
978 };
979 let second_offset = offset + first.len_utf8();
980 match first {
981 '-' => {
982 self.char_at(second_offset)
983 .is_some_and(|char| char == '-' || is_name_start(char))
984 || self.escape_starts_at(second_offset)
985 }
986 '\\' => self.escape_starts_at(offset),
987 char => is_name_start(char),
988 }
989 }
990
991 pub(crate) fn escape_starts_at(&self, offset: usize) -> bool {
992 if !self
993 .text
994 .get(offset..)
995 .is_some_and(|remaining| remaining.starts_with('\\'))
996 {
997 return false;
998 }
999 self.text[offset + '\\'.len_utf8()..]
1000 .chars()
1001 .next()
1002 .is_some_and(|char| !matches!(char, '\n' | '\r' | '\u{000c}'))
1003 }
1004
1005 pub(crate) fn char_at(&self, offset: usize) -> Option<char> {
1006 self.text.get(offset..)?.chars().next()
1007 }
1008
1009 pub(crate) fn char_after_current_is_ascii_digit(&self) -> bool {
1010 self.char_after_offset_is_ascii_digit(self.offset)
1011 }
1012
1013 pub(crate) fn char_after_offset_is_ascii_digit(&self, offset: usize) -> bool {
1014 let Some(char) = self.char_at(offset) else {
1015 return false;
1016 };
1017 self.char_at(offset + char.len_utf8())
1018 .is_some_and(|char| char.is_ascii_digit())
1019 }
1020
1021 pub(crate) fn starts_with_ascii_keyword(&self, keyword: &str) -> bool {
1022 let remaining = &self.text[self.offset..];
1023 let Some(prefix) = remaining.get(..keyword.len()) else {
1024 return false;
1025 };
1026 if !matches_ignore_ascii_case(prefix, &[keyword]) {
1027 return false;
1028 }
1029 remaining[keyword.len()..]
1030 .chars()
1031 .next()
1032 .is_none_or(|char| !is_name_continue(char))
1033 }
1034
1035 pub(crate) fn supports_scss_interpolation(&self) -> bool {
1036 matches!(
1037 self.extension.dialect(),
1038 StyleDialect::Scss | StyleDialect::Sass
1039 )
1040 }
1041
1042 pub(crate) fn supports_less_interpolation(&self) -> bool {
1043 self.extension.dialect() == StyleDialect::Less
1044 }
1045
1046 pub(crate) fn starts_less_escaped_string(&self) -> bool {
1047 self.extension.dialect() == StyleDialect::Less
1048 && (self.starts_with("~\"") || self.starts_with("~'"))
1049 }
1050
1051 pub(crate) fn starts_unicode_range(&self) -> bool {
1052 let mut chars = self.text[self.offset..].chars();
1053 matches!(chars.next(), Some('u' | 'U'))
1054 && chars.next() == Some('+')
1055 && chars
1056 .next()
1057 .is_some_and(|char| char.is_ascii_hexdigit() || char == '?')
1058 }
1059
1060 pub(crate) fn current_char(&self) -> Option<char> {
1061 self.text[self.offset..].chars().next()
1062 }
1063
1064 pub(crate) fn next_char_is_hex_digit(&self) -> bool {
1065 let offset = self.offset + '-'.len_utf8();
1066 self.text
1067 .get(offset..)
1068 .and_then(|tail| tail.chars().next())
1069 .is_some_and(|char| char.is_ascii_hexdigit())
1070 }
1071
1072 pub(crate) fn bump_current(&mut self) {
1073 if let Some(char) = self.current_char() {
1074 self.bump_char(char);
1075 }
1076 }
1077
1078 pub(crate) fn bump_char(&mut self, char: char) {
1079 self.offset += char.len_utf8();
1080 }
1081}
1082
1083fn css_syntax_preprocessed_char(char: char) -> char {
1084 if char == '\0' { '\u{fffd}' } else { char }
1085}
1086
1087fn starts_valid_escape_text(text: &str) -> bool {
1088 text.starts_with('\\')
1089 && text['\\'.len_utf8()..]
1090 .chars()
1091 .next()
1092 .is_some_and(|char| !matches!(char, '\n' | '\r' | '\u{000c}'))
1093}