tabnas_css/lex.rs
1/* Copyright (c) 2025 Richard Rodger, MIT License */
2
3//! The lexer: the single `cssToken` matcher the plugin installs on the
4//! engine, and the scanners it reads spans with.
5//!
6//! CSS is context-sensitive — the same characters can begin a selector, a
7//! property or a value — so the matcher owns the hard tokenisation and the
8//! grammar only assembles typed nodes from what it emits. Which token comes
9//! out depends on the rule that is active when the character is read: the
10//! engine reads a token only when the alternate being tried needs one, under
11//! the rule trying it, and passes that rule to the matcher.
12//!
13//! See `ts/src/css.ts` for the canonical commentary; the three ports are kept
14//! in step, and this module follows `buildCssTokenMatcher` there line for
15//! line.
16
17use std::fmt;
18use std::sync::Arc;
19
20use tabnas::{
21 Context, Lexer, Rule, TabnasError, Tin as EngineTin, Token, Value, TIN_TX, TIN_VL, TIN_ZZ,
22};
23
24use crate::plugin::{object, put, text};
25
26/// A token kind.
27///
28/// `Ob`/`Cb`/`Cl`/`Ca` are the fixed punctuation the canonical port borrows
29/// from the engine (`{` `}` `:` and `;`, the last remapped onto jsonic's
30/// member separator). The rest are this grammar's own: a custom name is
31/// required for a comment NODE because the engine's builtin comment tin is in
32/// the parser's ignore set, so emitting it would silently drop the node.
33#[derive(Clone, Copy, Debug, PartialEq, Eq)]
34pub enum Tin {
35 /// `{` — start of a block.
36 Ob,
37 /// `}` — end of a block.
38 Cb,
39 /// `:` — declaration separator.
40 Cl,
41 /// `;` — declaration terminator.
42 Ca,
43 /// A selector, keyframe value, or property name.
44 Tx,
45 /// `,` — selector-group separator.
46 Gc,
47 /// A declaration value (raw text).
48 Vl,
49 /// A comment, at a statement position.
50 Cc,
51 /// An at-rule with a rules body (`@media`, `@supports`, …).
52 Atr,
53 /// An at-rule with a declarations body (`@font-face`, `@page`, …).
54 Atd,
55 /// `@keyframes`.
56 Atk,
57 /// A statement at-rule (`@import`, `@charset`, …).
58 Ats,
59 /// End of input.
60 Zz,
61}
62
63impl Tin {
64 /// The token's grammar name, as `css-grammar.jsonic` writes it.
65 pub fn name(self) -> &'static str {
66 match self {
67 Tin::Ob => "#OB",
68 Tin::Cb => "#CB",
69 Tin::Cl => "#CL",
70 Tin::Ca => "#CA",
71 Tin::Tx => "#TX",
72 Tin::Gc => "#GC",
73 Tin::Vl => "#VL",
74 Tin::Cc => "#CC",
75 Tin::Atr => "#ATR",
76 Tin::Atd => "#ATD",
77 Tin::Atk => "#ATK",
78 Tin::Ats => "#ATS",
79 Tin::Zz => "#ZZ",
80 }
81 }
82
83 /// A human description, for diagnostics and diagram legends.
84 pub fn describe(self) -> &'static str {
85 match self {
86 Tin::Ob => "{ — start of a block",
87 Tin::Cb => "} — end of a block",
88 Tin::Cl => ": — declaration separator",
89 Tin::Ca => "; — declaration terminator",
90 Tin::Tx => "a selector, keyframe value, or property name",
91 Tin::Gc => ", — selector-group separator",
92 Tin::Vl => "a declaration value (raw text)",
93 Tin::Cc => "a comment",
94 Tin::Atr => "an at-rule with a rules body (@media, @supports, …)",
95 Tin::Atd => "an at-rule with a declarations body (@font-face, @page)",
96 Tin::Atk => "@keyframes",
97 Tin::Ats => "a statement at-rule (@import, @charset, …)",
98 Tin::Zz => "end of input",
99 }
100 }
101}
102
103/// A parse failure.
104///
105/// `code` is the contract: the shared fixtures pin `ERROR:<code>` and compare
106/// it exactly. [`Css::parse`](crate::Css::parse) raises only the two codes the
107/// canonical port inherits from the engine — `unterminated_comment` and
108/// `unexpected` — and this crate declares none of its own, matching
109/// `tabnas.plugin.json` (`errorCodes: []`). The engine's own tree form can
110/// also stop with `cancel`, at [`TREE_RULE_DEPTH`](crate::TREE_RULE_DEPTH).
111#[derive(Clone, Debug, PartialEq, Eq)]
112pub struct Error {
113 /// The error code, compared exactly by the conformance fixtures.
114 pub code: String,
115 /// A human-readable explanation.
116 pub message: String,
117 /// 1-based line where the failure was found.
118 pub line: usize,
119 /// 1-based column (UTF-16 code units) where the failure was found.
120 pub column: usize,
121}
122
123impl fmt::Display for Error {
124 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
125 write!(
126 f,
127 "[css/{}]: {} (line {}, column {})",
128 self.code, self.message, self.line, self.column
129 )
130 }
131}
132
133impl std::error::Error for Error {}
134
135/// The scanners' "there is no valid end here" result: an unclosed
136/// `/* ... */`.
137///
138/// A distinct value rather than an index, because the caller's response is a
139/// REJECTION, not a shorter span. Running the scan to end of source instead is
140/// how an unclosed comment in an at-rule prelude quietly became part of the
141/// prelude: `@import/*red` parsed as an import of `/*red`.
142const UNTERMINATED: usize = usize::MAX;
143
144/// What [`scan_to_brace_or_end`] found first.
145#[derive(Clone, Copy, PartialEq, Eq, Debug)]
146enum Kind {
147 /// A top-level `{` — so the run just scanned is a selector or a prelude.
148 Selector,
149 /// A top-level `;`/`}` or end of input — so it is a property or params.
150 Decl,
151}
152
153/// Rule names at which a `/* */` comment is captured as a NODE: the
154/// statement / declaration / keyframe LIST readers, which lex the first token
155/// of each item.
156///
157/// The item builders (`statement`/`decl`/`keyframe`) are deliberately absent.
158/// They reuse the token the list reader already produced, so a comment seen
159/// mid-construct — between a property and its `:`, say — is read under a
160/// builder, declined here, and skipped as insignificant whitespace.
161///
162/// The block wrappers are present because their empty-block `#OB #CB`
163/// lookahead lexes the first body token, so a comment right after `{` is
164/// captured there.
165const COMMENT_NODE_RULES: [&str; 6] = [
166 "items",
167 "decls",
168 "kfitems",
169 "declbody",
170 "rulesbody",
171 "kfbody",
172];
173
174fn is_comment_node_rule(name: &str) -> bool {
175 COMMENT_NODE_RULES.contains(&name)
176}
177
178/// At-keywords whose body is a declaration list rather than a rules list.
179const DECLS_KW: [&str; 7] = [
180 "font-face",
181 "page",
182 "viewport",
183 "-ms-viewport",
184 "counter-style",
185 "property",
186 "font-palette-values",
187];
188
189/// `^(-[a-z]+-)?keyframes$`, without a regex engine.
190fn is_keyframes_kw(kw: &str) -> bool {
191 if kw == "keyframes" {
192 return true;
193 }
194 match vendor_prefix(kw) {
195 Some(v) => &kw[v.len()..] == "keyframes",
196 None => false,
197 }
198}
199
200// The heads of the at-rule nodes: `makeAtRules`, `makeAtDecls`,
201// `makeKeyframes` and `makeAtStmt` in the canonical port, each without the
202// body array the grammar's constructor adds. Key order is JavaScript's: a
203// key assigned twice keeps its first place.
204
205/// The head of a block at-rule whose body is a list of rules (`@media`,
206/// `@supports`, `@document`, `@host`, and any other with a rules body).
207pub(crate) fn at_rules_head(kw: &str, prelude: &str) -> Value {
208 let mut node = object();
209 match kw {
210 "host" => put(&mut node, "type", text("host")),
211 _ if "document" == kw || kw.ends_with("-document") => {
212 // `vendor` is always present on a document node, the empty
213 // string when the at-keyword carries no prefix.
214 put(&mut node, "type", text("document"));
215 put(&mut node, "document", text(prelude));
216 put(&mut node, "vendor", text(vendor_prefix(kw).unwrap_or("")));
217 }
218 _ => {
219 put(&mut node, "type", text(kw));
220 put(&mut node, kw, text(prelude));
221 }
222 }
223 node
224}
225
226/// The head of a block at-rule whose body is declarations (`@font-face`,
227/// `@page`, and any other with a declarations body). A `@page` prelude is a
228/// selector group: `@page toc, index:blank`.
229pub(crate) fn at_decls_head(kw: &str, prelude: &str) -> Value {
230 let mut node = object();
231 put(&mut node, "type", text(kw));
232 if "page" == kw {
233 let selectors = split_selectors(prelude)
234 .into_iter()
235 .map(Value::String)
236 .collect();
237 put(&mut node, "selectors", Value::array(selectors));
238 }
239 node
240}
241
242/// The head of a `@keyframes` block, possibly vendor-prefixed.
243pub(crate) fn keyframes_head(kw: &str, name: &str) -> Value {
244 let mut node = object();
245 put(&mut node, "type", text("keyframes"));
246 put(&mut node, "name", text(name));
247 if let Some(vendor) = vendor_prefix(kw) {
248 put(&mut node, "vendor", text(vendor));
249 }
250 node
251}
252
253/// A statement at-rule (`@import`, `@charset`, `@namespace`, …), whole: the
254/// at-keyword is the node type and the field carrying its params.
255///
256/// `@custom-media --name <query>` splits its params into a name and a media
257/// query, as `/^(--\S+)\s*([\s\S]*)$/` does in the canonical port: `--` and
258/// at least one non-space character, then the rest, trimmed.
259pub(crate) fn at_stmt_head(kw: &str, params: &str) -> Value {
260 let mut node = object();
261 if "custom-media" == kw && params.starts_with("--") {
262 let name_end = params.find(es_is_whitespace).unwrap_or(params.len());
263 if 2 < name_end {
264 put(&mut node, "type", text("custom-media"));
265 put(&mut node, "name", text(¶ms[..name_end]));
266 put(&mut node, "media", text(es_trim(¶ms[name_end..])));
267 return node;
268 }
269 }
270 put(&mut node, "type", text(kw));
271 put(&mut node, kw, text(params));
272 node
273}
274
275/// `^(-[a-z]+-)`: a vendor prefix on an at-keyword, e.g. `-webkit-keyframes`.
276pub fn vendor_prefix(kw: &str) -> Option<&str> {
277 let b = kw.as_bytes();
278 if b.first() != Some(&b'-') {
279 return None;
280 }
281 let mut i = 1;
282 while i < b.len() && b[i].is_ascii_lowercase() {
283 i += 1;
284 }
285 if 1 < i && b.get(i) == Some(&b'-') {
286 Some(&kw[..i + 1])
287 } else {
288 None
289 }
290}
291
292/// Whether `c` is whitespace as ECMAScript defines it, which is NOT what
293/// [`char::is_whitespace`] answers.
294///
295/// The canonical port trims with JavaScript's `String.prototype.trim`, whose
296/// set is ECMAScript WhiteSpace plus LineTerminator. Rust's set is the Unicode
297/// `White_Space` property. They differ by exactly two code points, and both
298/// differences change the AST:
299///
300/// - **U+FEFF** (zero-width no-break space, the byte-order mark) is
301/// ECMAScript whitespace and is not Unicode `White_Space`. A stylesheet that
302/// opens with a byte-order mark yields the selector `a` in the canonical
303/// port and would yield `\u{feff}a` under [`str::trim`].
304/// - **U+0085** (next line) is Unicode `White_Space` and is not ECMAScript
305/// whitespace. `@media x \u{85}{…}` keeps that character in its prelude in
306/// the canonical port and would lose it under [`str::trim`].
307///
308/// Everything else in either set is in both: the ASCII controls, U+0020, the
309/// `Zs` category, and the two line separators U+2028 and U+2029.
310pub fn es_is_whitespace(c: char) -> bool {
311 ('\u{feff}' == c) || (c.is_whitespace() && '\u{85}' != c)
312}
313
314/// Trim as JavaScript's `String.prototype.trim` does. See
315/// [`es_is_whitespace`] for why this is not [`str::trim`].
316pub fn es_trim(s: &str) -> &str {
317 s.trim_matches(es_is_whitespace)
318}
319
320/// The column width of `s`: its length in UTF-16 code units, which is what a
321/// JavaScript string index counts.
322pub(crate) fn col_width(s: &str) -> usize {
323 s.chars().map(char::len_utf16).sum()
324}
325
326/// Slice `src` by byte offsets, clamping to something sliceable.
327///
328/// This is the Rust half of the Go port's `clampLen`, and it exists for the
329/// same reason: the scanners deliberately overshoot. An `i += 2` over a `\`
330/// escape, an unterminated string or an unterminated comment can step past the
331/// end of the source, and JavaScript CLAMPS an out-of-range slice bound where
332/// Go panics — so the Go port clamps to reproduce the JavaScript result rather
333/// than invent a new one.
334///
335/// Rust panics on one bound more than Go does: an index inside a multi-byte
336/// character. Every index these scanners RETURN is at an ASCII character or at
337/// the end of the source, because each one stops at `{`, `}`, `;`, `,`, `*/`,
338/// a quote or end of input, so the fallback below is unreachable in practice —
339/// it is here so that a future scanner change degrades to a shorter span
340/// instead of a panic in a caller's parse.
341fn safe_slice(src: &str, start: usize, end: usize) -> &str {
342 let mut start = start.min(src.len());
343 let mut end = end.clamp(start, src.len());
344 while 0 < start && !src.is_char_boundary(start) {
345 start -= 1;
346 }
347 while start < end && !src.is_char_boundary(end) {
348 end -= 1;
349 }
350 &src[start..end]
351}
352
353/// The index of the `*` that opens the `*/` closing the comment at `i`, or
354/// `None` if there is no closing `*/`.
355fn find_comment_end(src: &[u8], i: usize) -> Option<usize> {
356 let mut j = i + 2;
357 while j + 1 < src.len() {
358 if src[j] == b'*' && src[j + 1] == b'/' {
359 return Some(j);
360 }
361 j += 1;
362 }
363 None
364}
365
366/// The index after a closed `/* ... */`, or [`UNTERMINATED`].
367///
368/// It does NOT run to end of source. An unclosed `/* ... */` is an error, not
369/// a comment to the end of the file — which is what both the engine's builtin
370/// comment matcher and reworkcss do, and what `test/spec/reworkcss.tsv` pins.
371fn skip_comment(src: &[u8], i: usize) -> usize {
372 match find_comment_end(src, i) {
373 // `e + 1` is the `/`, so `e + 2` is always within the source.
374 Some(e) => e + 2,
375 None => UNTERMINATED,
376 }
377}
378
379/// Skip a quoted string; returns the index after the closing quote.
380fn skip_string(src: &[u8], mut i: usize) -> usize {
381 let quote = src[i];
382 i += 1;
383 while i < src.len() {
384 if src[i] == b'\\' {
385 i += 2;
386 continue;
387 }
388 if src[i] == quote {
389 return i + 1;
390 }
391 i += 1;
392 }
393 // A trailing backslash steps i ONE PAST the end. That overshoot is not a
394 // defect to clamp away here: the canonical port returns it too, and its
395 // column arithmetic counts it. See [`css_token`] and [`safe_slice`].
396 i
397}
398
399/// Scan a key prelude: where it ends, and whether it is a selector (a
400/// top-level `{` comes first) or a declaration (a top-level `;`/`}` or end of
401/// input comes first). Strings, `()`, `[]` and comments are skipped.
402fn scan_to_brace_or_end(src: &[u8], i: usize) -> (Kind, usize) {
403 let mut scan = Scan { i, depth: 0 };
404 loop {
405 if let Some(found) = scan.step(src) {
406 return found;
407 }
408 }
409}
410
411/// [`scan_to_brace_or_end`] one step at a time. The position and the
412/// bracket depth are its whole state, so two scans in the same state go on
413/// to the same answer: what [`BraceScans`] relies on.
414#[derive(Clone, Copy, Debug)]
415struct Scan {
416 i: usize,
417 depth: usize,
418}
419
420impl Scan {
421 /// Take one step: past a string, a comment, an escape or one byte.
422 /// `Some` is the scan's answer.
423 fn step(&mut self, src: &[u8]) -> Option<(Kind, usize)> {
424 let Some(&c) = src.get(self.i) else {
425 // i may be one past the end; see skip_string.
426 return Some((Kind::Decl, self.i));
427 };
428 if c == b'"' || c == b'\'' {
429 self.i = skip_string(src, self.i);
430 return None;
431 }
432 if c == b'/' && src.get(self.i + 1) == Some(&b'*') {
433 let n = skip_comment(src, self.i);
434 if UNTERMINATED == n {
435 return Some((Kind::Decl, UNTERMINATED));
436 }
437 self.i = n;
438 return None;
439 }
440 // A CSS escape (`\(`, `\'`, `\3A `, …) hides the next character from
441 // the structural scan.
442 if c == b'\\' {
443 self.i += 2;
444 return None;
445 }
446 if c == b'(' || c == b'[' {
447 self.depth += 1;
448 } else if c == b')' || c == b']' {
449 self.depth = self.depth.saturating_sub(1);
450 } else if 0 == self.depth {
451 if c == b'{' {
452 return Some((Kind::Selector, self.i));
453 }
454 if c == b';' || c == b'}' {
455 return Some((Kind::Decl, self.i));
456 }
457 }
458 self.i += 1;
459 None
460 }
461}
462
463/// [`scan_to_brace_or_end`] for every token of a parse, in linear time.
464///
465/// Each selector of a group, and each value of a keyframe-selector group,
466/// is classified by scanning to the group's `{`, so a group of `k` items
467/// scanned from each item's start costs `k` times the group's length: a
468/// 200 KB list of selectors took 9 s. The scan from an item's start is the
469/// scan from the group's first item, resumed there, whenever that scan
470/// passes the item's start at bracket depth 0 (the state [`Scan`] carries,
471/// outside any string, comment or escape, which it steps over whole). So
472/// the answer is kept with a cursor that follows the tokens forward, and a
473/// later start the cursor reaches in that state takes the kept answer. Any
474/// other start scans afresh and becomes the one kept. The cursor only moves
475/// forward, so a parse's scanning is linear in its length.
476#[derive(Clone, Copy, Debug)]
477struct BraceScans {
478 from: usize,
479 found: (Kind, usize),
480 cursor: Scan,
481}
482
483/// The `ctx.u` key [`BraceScans`] is kept under, for the parse.
484pub(crate) const BRACE_SCAN: &str = "tabnas-css/brace-scan";
485
486impl BraceScans {
487 /// The answer for a token starting at `s_i`, keeping the state under
488 /// [`BRACE_SCAN`] with one map lookup: the matcher asks on every `#TX`,
489 /// and a flat stylesheet is mostly misses.
490 fn scan(ctx: &mut Context, src: &[u8], s_i: usize) -> (Kind, usize) {
491 let Some(Value::Array(slot)) = ctx.u.get_mut(BRACE_SCAN) else {
492 let found = scan_to_brace_or_end(src, s_i);
493 let kept = BraceScans::fresh(s_i, found);
494 ctx.u
495 .insert(BRACE_SCAN.to_string(), Value::array(kept.fields().to_vec()));
496 return found;
497 };
498 let fields = Arc::make_mut(slot);
499 if let Some(mut kept) = BraceScans::decode(fields) {
500 if kept.from <= s_i && s_i < kept.found.1 && kept.cursor.i <= s_i {
501 while kept.cursor.i < s_i && kept.cursor.step(src).is_none() {}
502 let reached = kept.cursor.i == s_i && 0 == kept.cursor.depth;
503 kept.encode(fields);
504 if reached {
505 return kept.found;
506 }
507 }
508 }
509 // An unclosed comment's answer is kept too: a scan from a later start
510 // the cursor reaches meets the same `/*`, and the engine's recovery
511 // asks again at the very start it failed at, up to its skip budget.
512 let found = scan_to_brace_or_end(src, s_i);
513 BraceScans::fresh(s_i, found).encode(fields);
514 found
515 }
516
517 fn fresh(from: usize, found: (Kind, usize)) -> BraceScans {
518 BraceScans {
519 from,
520 found,
521 cursor: Scan { i: from, depth: 0 },
522 }
523 }
524
525 fn decode(fields: &[Value]) -> Option<BraceScans> {
526 let at = |n: usize| match fields.get(n) {
527 Some(Value::Number(v)) => Some(*v as usize),
528 _ => None,
529 };
530 let kind = if 0 == at(1)? {
531 Kind::Selector
532 } else {
533 Kind::Decl
534 };
535 Some(BraceScans {
536 from: at(0)?,
537 found: (kind, at(2)?),
538 cursor: Scan {
539 i: at(3)?,
540 depth: at(4)?,
541 },
542 })
543 }
544
545 fn numbers(&self) -> [f64; 5] {
546 let kind = match self.found.0 {
547 Kind::Selector => 0,
548 Kind::Decl => 1,
549 };
550 [
551 self.from,
552 kind,
553 self.found.1,
554 self.cursor.i,
555 self.cursor.depth,
556 ]
557 .map(|n| n as f64)
558 }
559
560 fn fields(&self) -> [Value; 5] {
561 self.numbers().map(Value::Number)
562 }
563
564 /// Write this state over `fields`, number by number when the shape fits.
565 fn encode(&self, fields: &mut Vec<Value>) {
566 let numbers = self.numbers();
567 if numbers.len() == fields.len() {
568 for (field, n) in fields.iter_mut().zip(numbers) {
569 match field {
570 Value::Number(slot) => *slot = n,
571 other => *other = Value::Number(n),
572 }
573 }
574 } else {
575 *fields = self.fields().to_vec();
576 }
577 }
578}
579
580/// Scan a single selector: to the next top-level `,` (a group separator) or
581/// `{`, or end of input. Strings, `()`, `[]` and comments are skipped.
582fn scan_selector_end(src: &[u8], mut i: usize) -> usize {
583 let mut depth = 0usize;
584 while i < src.len() {
585 let c = src[i];
586 if c == b'"' || c == b'\'' {
587 i = skip_string(src, i);
588 continue;
589 }
590 if c == b'/' && src.get(i + 1) == Some(&b'*') {
591 let n = skip_comment(src, i);
592 if UNTERMINATED == n {
593 return UNTERMINATED;
594 }
595 i = n;
596 continue;
597 }
598 if c == b'\\' {
599 i += 2;
600 continue;
601 }
602 if c == b'(' || c == b'[' {
603 depth += 1;
604 } else if c == b')' || c == b']' {
605 depth = depth.saturating_sub(1);
606 } else if 0 == depth && (c == b',' || c == b'{') {
607 return i;
608 }
609 i += 1;
610 }
611 // i may be one past the end; see skip_string.
612 i
613}
614
615/// Scan a declaration value or at-rule params: to the next top-level `;` or
616/// `}`, or end of input. Strings, `()`, `[]` and comments are skipped.
617fn scan_value_end(src: &[u8], mut i: usize) -> usize {
618 let mut depth = 0usize;
619 while i < src.len() {
620 let c = src[i];
621 if c == b'"' || c == b'\'' {
622 i = skip_string(src, i);
623 continue;
624 }
625 if c == b'/' && src.get(i + 1) == Some(&b'*') {
626 let n = skip_comment(src, i);
627 if UNTERMINATED == n {
628 return UNTERMINATED;
629 }
630 i = n;
631 continue;
632 }
633 if c == b'\\' {
634 i += 2;
635 continue;
636 }
637 if c == b'(' || c == b'[' {
638 depth += 1;
639 } else if c == b')' || c == b']' {
640 depth = depth.saturating_sub(1);
641 } else if 0 == depth && (c == b';' || c == b'}') {
642 return i;
643 }
644 i += 1;
645 }
646 // i may be one past the end; see skip_string.
647 i
648}
649
650/// Remove `/* ... */` comments from a selector or value run, leaving quoted
651/// strings (which may contain `/*`) untouched.
652pub fn strip_comments(s: &str) -> String {
653 if !s.contains("/*") {
654 return s.to_string();
655 }
656 let src = s.as_bytes();
657 let mut out: Vec<u8> = Vec::with_capacity(src.len());
658 let mut i = 0;
659 while i < src.len() {
660 let c = src[i];
661 if c == b'"' || c == b'\'' {
662 let j = skip_string(src, i);
663 out.extend_from_slice(&src[i..j.min(src.len())]);
664 i = j;
665 continue;
666 }
667 if c == b'/' && src.get(i + 1) == Some(&b'*') {
668 let n = skip_comment(src, i);
669 if UNTERMINATED == n {
670 // Unreachable once the callers reject an unclosed comment,
671 // but a sentinel assigned to i would spin. The rest of the
672 // span is comment text; drop it.
673 break;
674 }
675 i = n;
676 continue;
677 }
678 out.push(c);
679 i += 1;
680 }
681 String::from_utf8(out).unwrap_or_else(|e| String::from_utf8_lossy(e.as_bytes()).into_owned())
682}
683
684/// A CSS property name character.
685///
686/// Beyond the identifier characters this admits the legacy IE hack prefixes
687/// `*`, `#`, `/` (as in `//prop`) and a literal backslash, matching the
688/// reworkcss property pattern `\*?[-#\/\*\\\w]+(\[[0-9a-z_-]+\])?` — real
689/// stylesheets rely on them.
690fn is_prop_char(c: u8) -> bool {
691 c.is_ascii_alphanumeric()
692 || c == b'-'
693 || c == b'_'
694 || c == b'#'
695 || c == b'*'
696 || c == b'/'
697 || c == b'\\'
698}
699
700/// A `[0-9a-z_-]` character, the only content allowed in a property name's
701/// bracket suffix (e.g. `opacity[sqrt]`).
702fn is_prop_suffix_char(c: u8) -> bool {
703 c.is_ascii_digit() || c.is_ascii_lowercase() || c == b'-' || c == b'_'
704}
705
706/// Scan a property name: the run of property characters plus an optional
707/// `[...]` suffix. Returns the index after the name (`== i` if there is none).
708fn scan_prop_end(src: &[u8], i: usize) -> usize {
709 let mut e_i = i;
710 while e_i < src.len() && is_prop_char(src[e_i]) {
711 e_i += 1;
712 }
713 if e_i == i {
714 return i;
715 }
716 if src.get(e_i) == Some(&b'[') {
717 let mut b_i = e_i + 1;
718 while b_i < src.len() && is_prop_suffix_char(src[b_i]) {
719 b_i += 1;
720 }
721 if e_i + 1 < b_i && src.get(b_i) == Some(&b']') {
722 return b_i + 1;
723 }
724 }
725 e_i
726}
727
728/// An at-keyword character: letters, digits and `-`.
729fn is_at_char(c: u8) -> bool {
730 c.is_ascii_alphanumeric() || c == b'-'
731}
732
733/// Split a selector-group prelude on its top-level commas (commas inside
734/// strings, `()` or `[]` are part of a single selector), stripping comments
735/// and trimming each selector. An empty prelude yields an empty list.
736pub fn split_selectors(prelude: &str) -> Vec<String> {
737 let src = prelude.as_bytes();
738 let mut out = Vec::new();
739 let mut i = 0;
740 while i <= src.len() {
741 let mut end = scan_selector_end(src, i);
742 if UNTERMINATED == end {
743 // Unreachable in practice: the prelude was scanned (and an
744 // unclosed comment rejected) before it got here. Treated as "the
745 // rest" so a sentinel can never index or spin.
746 end = src.len();
747 }
748 let one = es_trim(&strip_comments(safe_slice(prelude, i, end))).to_string();
749 if !one.is_empty() {
750 out.push(one);
751 }
752 if end >= src.len() {
753 break;
754 }
755 i = end + 1;
756 }
757 out
758}
759
760// ---------------------------------------------------------------------------
761// The `cssToken` matcher, as the engine runs it.
762
763/// The token number a matcher gives a token the engine is to resolve by
764/// NAME, as it lexes: the six this grammar names itself (`#CC`, `#GC` and
765/// the four at-rule tokens).
766///
767/// The engine numbers a name when it is first registered, and
768/// [`Tabnas::merge`](tabnas::Tabnas::merge) renumbers every custom name in
769/// the merged instance without running the plugin again, so a number
770/// captured at install names another token there. The canonical port's
771/// `lex.token('#CC', …)` resolves the name on every call for the same
772/// reason. `#TX` and `#VL` are the engine's own, with fixed numbers.
773const BY_NAME: EngineTin = -1;
774
775/// Where a scan ran past the end of the source, and by how much: the
776/// `ctx.u` key [`css_token`] records it under and [`lex_subscriber`] reads.
777pub(crate) const OVERSHOOT: &str = "tabnas-css/overshoot";
778
779/// The most of the source an `unterminated_comment` diagnostic shows. The
780/// span runs from the scan's start to the end of the source, which for a
781/// large stylesheet would be most of it; the message is informative, not the
782/// contract, so it is cut here.
783const BAD_SPAN: usize = 32;
784
785/// What [`plan_token`] decided for the characters at the cursor.
786enum Plan {
787 /// Not this matcher's: the engine's builtin matchers take it.
788 Decline,
789 /// An unclosed `/* ... */` behind the cursor: `unterminated_comment`.
790 Bad,
791 /// A token, and the byte offset just past it, which may be ONE past the
792 /// end of the source (see [`css_token`]).
793 Emit {
794 name: &'static str,
795 tin: EngineTin,
796 val: Value,
797 raw: String,
798 end: usize,
799 },
800}
801
802/// Decide the token at byte `s_i` of `src`, read under the rule `rule`.
803///
804/// `buildCssTokenMatcher` in `ts/src/css.ts`, step for step: whitespace is
805/// declined (the engine's space and line matchers take it), a comment is a
806/// node only at a list position, a value runs to the next top-level `;` or
807/// `}` under `declval`, a top-level `,` separates a selector group, `@`
808/// starts an at-rule, `{` `}` `;` are the grammar's, and anything else is a
809/// selector or a property name by `{`-before-`;` lookahead.
810fn plan_token(
811 src: &str,
812 s_i: usize,
813 rule: &str,
814 lower: bool,
815 brace: &mut dyn FnMut(usize) -> (Kind, usize),
816) -> Plan {
817 let bytes = src.as_bytes();
818 let Some(&c) = bytes.get(s_i) else {
819 return Plan::Decline;
820 };
821 if c == b' ' || c == b'\t' || c == b'\r' || c == b'\n' {
822 return Plan::Decline;
823 }
824 let emit = |name, tin, val: String, raw: &str, end| Plan::Emit {
825 name,
826 tin,
827 val: Value::String(val),
828 raw: raw.to_string(),
829 end,
830 };
831
832 // Comments: a node at a list position, otherwise declined, and then
833 // skipped by the engine's builtin comment matcher.
834 if c == b'/' && bytes.get(s_i + 1) == Some(&b'*') {
835 if !is_comment_node_rule(rule) {
836 return Plan::Decline;
837 }
838 return match find_comment_end(bytes, s_i) {
839 None => Plan::Bad,
840 Some(e) => emit(
841 "#CC",
842 BY_NAME,
843 safe_slice(src, s_i + 2, e).to_string(),
844 safe_slice(src, s_i, e + 2),
845 e + 2,
846 ),
847 };
848 }
849
850 // Value position: a declaration value up to the next top-level `;`/`}`,
851 // comments stripped and surrounding space trimmed.
852 if "declval" == rule {
853 if c == b'{' || c == b'}' || c == b';' || c == b':' {
854 return Plan::Decline;
855 }
856 let end = scan_value_end(bytes, s_i);
857 if UNTERMINATED == end {
858 return Plan::Bad;
859 }
860 let raw = safe_slice(src, s_i, end);
861 let val = es_trim(&strip_comments(raw)).to_string();
862 return emit("#VL", TIN_VL, val, raw, end);
863 }
864
865 // A top-level selector-group comma.
866 if c == b',' {
867 return emit("#GC", BY_NAME, ",".to_string(), ",", s_i + 1);
868 }
869
870 // An at-rule: block or statement by `{`-before-`;` lookahead, and a
871 // block's body by keyword. The token's value is the node's head: its
872 // type and the fields read from the keyword and the prelude (block) or
873 // params (statement), so that a reader of the rule passes (the
874 // transducer part, `alchemy/transduce.alc`) sees the node as the walk
875 // builds it, a pass carrying a token's value and never its `use` bag.
876 if c == b'@' {
877 let mut k_end = s_i + 1;
878 while k_end < bytes.len() && is_at_char(bytes[k_end]) {
879 k_end += 1;
880 }
881 let kw = safe_slice(src, s_i + 1, k_end).to_string();
882 let (kind, index) = brace(s_i);
883 if UNTERMINATED == index {
884 return Plan::Bad;
885 }
886 if Kind::Selector == kind {
887 // At-rule preludes KEEP their comments (they are only trimmed),
888 // matching reworkcss.
889 let prelude = es_trim(safe_slice(src, k_end, index)).to_string();
890 let (name, tin) = if is_keyframes_kw(&kw) {
891 ("#ATK", BY_NAME)
892 } else if DECLS_KW.contains(&kw.as_str()) {
893 ("#ATD", BY_NAME)
894 } else {
895 ("#ATR", BY_NAME)
896 };
897 let head = match name {
898 "#ATK" => keyframes_head(&kw, &prelude),
899 "#ATD" => at_decls_head(&kw, &prelude),
900 _ => at_rules_head(&kw, &prelude),
901 };
902 let raw = safe_slice(src, s_i, index);
903 return Plan::Emit {
904 name,
905 tin,
906 val: head,
907 raw: raw.to_string(),
908 end: index,
909 };
910 }
911 // A statement at-rule's params run to the next top-level `;`/`}`. A
912 // `;` is consumed (it ends the statement); a `}` or the end of the
913 // source is left for the enclosing rule.
914 let p_end = scan_value_end(bytes, k_end);
915 if UNTERMINATED == p_end {
916 return Plan::Bad;
917 }
918 let params = es_trim(safe_slice(src, k_end, p_end)).to_string();
919 let end = if bytes.get(p_end) == Some(&b';') {
920 p_end + 1
921 } else {
922 p_end
923 };
924 let raw = safe_slice(src, s_i, end);
925 return Plan::Emit {
926 name: "#ATS",
927 tin: BY_NAME,
928 val: at_stmt_head(&kw, ¶ms),
929 raw: raw.to_string(),
930 end,
931 };
932 }
933
934 // Other fixed punctuation belongs to the grammar. `:` does not: a run
935 // that starts with one is scanned below, as in the canonical port.
936 if c == b'{' || c == b'}' || c == b';' {
937 return Plan::Decline;
938 }
939
940 // A selector or a property name, by `{`-before-`;` lookahead.
941 let (kind, index) = brace(s_i);
942 if UNTERMINATED == index {
943 return Plan::Bad;
944 }
945 if Kind::Selector == kind {
946 // One selector of a (possible) group: up to the next top-level `,`
947 // or `{`, comments stripped and surrounding space trimmed.
948 let end = scan_selector_end(bytes, s_i);
949 if UNTERMINATED == end {
950 return Plan::Bad;
951 }
952 let raw = safe_slice(src, s_i, end);
953 let val = es_trim(&strip_comments(raw)).to_string();
954 return emit("#TX", TIN_TX, val, raw, end);
955 }
956
957 // A property name: the run of property characters up to `:`,
958 // whitespace, `;` or `}`. `/` and `*` are property characters (the
959 // `*prop` and `//prop` IE hacks), so a trailing `/*...*/` hack comment
960 // lands inside the scanned name; comments are stripped out of it, as
961 // reworkcss does.
962 let e_i = scan_prop_end(bytes, s_i);
963 if e_i == s_i {
964 return Plan::Decline;
965 }
966 let raw = safe_slice(src, s_i, e_i);
967 let mut prop = strip_comments(raw);
968 if prop.is_empty() {
969 return Plan::Decline;
970 }
971 if lower {
972 prop = prop.to_lowercase();
973 }
974 emit("#TX", TIN_TX, prop, raw, e_i)
975}
976
977/// The `cssToken` matcher, registered as `@css-token` and named by the
978/// grammar's options at order 1e5, ahead of every builtin matcher.
979///
980/// Two things here are deliberate and easy to "fix" wrongly:
981///
982/// - The end of a token may be ONE PAST the source. A `\` at the last
983/// character is an escape whose escaped character is not there, and the
984/// scanners' `i += 2` steps off the end. The canonical port advances its
985/// column by that unclamped width and takes the CLAMPED span for the
986/// token's text, so `@host\` yields a node ending at column 7 inside a
987/// stylesheet ending at column 8, one past a source six characters long.
988/// The engine will not move its cursor past the end, so the overshoot is
989/// recorded under [`OVERSHOOT`] and [`lex_subscriber`] adds it to the
990/// end-of-source token, the only token that can follow it.
991/// - An unclosed comment is reported WITHOUT advancing, so the diagnostic
992/// points at the start of the scan, as the canonical port's does.
993///
994/// A matcher must never advance and then decline: the engine captured its
995/// dispatch state before the custom matchers ran.
996pub(crate) fn css_token(
997 lexer: &mut Lexer<'_>,
998 rule: &mut Rule,
999 ctx: &mut Context,
1000 lower: bool,
1001) -> Option<Token> {
1002 let point = lexer.point();
1003 let s_i = point.site.si;
1004 let (plan, chars, overshoot) = {
1005 let src = lexer.source();
1006 let bytes = src.as_bytes();
1007 let mut brace = |at: usize| BraceScans::scan(ctx, bytes, at);
1008 let plan = plan_token(src, s_i, rule.name.as_str(), lower, &mut brace);
1009 let (chars, overshoot) = match &plan {
1010 Plan::Decline => return None,
1011 Plan::Bad => (
1012 safe_slice(src, s_i, src.len())
1013 .chars()
1014 .take(BAD_SPAN)
1015 .count(),
1016 0,
1017 ),
1018 Plan::Emit { end, .. } => (
1019 safe_slice(src, s_i, (*end).min(src.len())).chars().count(),
1020 end.saturating_sub(src.len()),
1021 ),
1022 };
1023 (plan, chars, overshoot)
1024 };
1025 match plan {
1026 Plan::Decline => None,
1027 Plan::Bad => Some(lexer.bad_span(
1028 "unterminated_comment",
1029 point.site.pos,
1030 point.site.pos + chars,
1031 )),
1032 Plan::Emit {
1033 name,
1034 tin,
1035 val,
1036 raw,
1037 ..
1038 } => {
1039 lexer.advance_chars(chars);
1040 if 0 < overshoot {
1041 ctx.u
1042 .insert(OVERSHOOT.to_string(), Value::Number(overshoot as f64));
1043 }
1044 Some(lexer.token(name, tin, val, raw, point))
1045 }
1046 }
1047}
1048
1049/// The plugin's lex subscriber: it sees every token the engine fetches.
1050///
1051/// - The end-of-source token takes the column overshoot [`css_token`]
1052/// recorded, so the stylesheet's end and an error at the end of the source
1053/// land where the canonical port puts them.
1054/// - It once dropped the unconsumed lookahead when a bad token arrived
1055/// behind a good one, so that an unclosed comment hidden from the
1056/// property scan was the error reported rather than `unexpected` at the
1057/// property. tabnas/parser#274 made the engine raise a fetched bad token
1058/// at once, as the canonical does, and record and skip it under
1059/// recovery, so that work is the engine's now. `test/spec/comments.tsv`
1060/// and `tests/plugin.rs` pin the answers in every mode.
1061pub(crate) fn lex_subscriber(token: &mut Token, _rule: &mut Rule, ctx: &mut Context) {
1062 if TIN_ZZ == token.tin {
1063 if let Some(Value::Number(n)) = ctx.u.get(OVERSHOOT) {
1064 token.site.ci += *n as usize;
1065 }
1066 }
1067}
1068
1069/// The number of astral characters (those two UTF-16 code units wide) in
1070/// scalars `[from, to)` of `src`.
1071pub(crate) fn astral_between(src: &str, from: usize, to: usize) -> usize {
1072 src.chars()
1073 .skip(from)
1074 .take(to.saturating_sub(from))
1075 .filter(|c| 2 == c.len_utf16())
1076 .count()
1077}
1078
1079/// An engine error as this crate's, with the column in UTF-16 code units.
1080///
1081/// The engine counts a column in Unicode scalars since the last reset (a
1082/// `\n` in any token, or the end of a run of line characters); the canonical
1083/// port counts UTF-16 code units since the same points. The two differ by
1084/// the astral characters between the reset and the error, which is what is
1085/// added here. The message is the engine's, and is not the contract.
1086impl From<TabnasError> for Error {
1087 fn from(error: TabnasError) -> Error {
1088 let from = error.pos.saturating_sub(error.col.saturating_sub(1));
1089 let wide = astral_between(&error.full_source, from, error.pos);
1090 Error {
1091 code: error.code,
1092 message: error.detail,
1093 line: error.row,
1094 column: error.col + wide,
1095 }
1096 }
1097}