Skip to main content

tabnas_css/
lex.rs

1/* Copyright (c) 2025 Richard Rodger, MIT License */
2
3//! The lexer: the single `cssToken` matcher the plugin installs on the
4//! engine, and the scanners it reads spans with.
5//!
6//! CSS is context-sensitive — the same characters can begin a selector, a
7//! property or a value — so the matcher owns the hard tokenisation and the
8//! grammar only assembles typed nodes from what it emits. Which token comes
9//! out depends on the rule that is active when the character is read: the
10//! engine reads a token only when the alternate being tried needs one, under
11//! the rule trying it, and passes that rule to the matcher.
12//!
13//! See `ts/src/css.ts` for the canonical commentary; the three ports are kept
14//! in step, and this module follows `buildCssTokenMatcher` there line for
15//! line.
16
17use std::fmt;
18use std::sync::Arc;
19
20use tabnas::{
21    Context, Lexer, Rule, TabnasError, Tin as EngineTin, Token, Value, TIN_TX, TIN_VL, TIN_ZZ,
22};
23
24use crate::plugin::{object, put, text};
25
26/// A token kind.
27///
28/// `Ob`/`Cb`/`Cl`/`Ca` are the fixed punctuation the canonical port borrows
29/// from the engine (`{` `}` `:` and `;`, the last remapped onto jsonic's
30/// member separator). The rest are this grammar's own: a custom name is
31/// required for a comment NODE because the engine's builtin comment tin is in
32/// the parser's ignore set, so emitting it would silently drop the node.
33#[derive(Clone, Copy, Debug, PartialEq, Eq)]
34pub enum Tin {
35    /// `{` — start of a block.
36    Ob,
37    /// `}` — end of a block.
38    Cb,
39    /// `:` — declaration separator.
40    Cl,
41    /// `;` — declaration terminator.
42    Ca,
43    /// A selector, keyframe value, or property name.
44    Tx,
45    /// `,` — selector-group separator.
46    Gc,
47    /// A declaration value (raw text).
48    Vl,
49    /// A comment, at a statement position.
50    Cc,
51    /// An at-rule with a rules body (`@media`, `@supports`, …).
52    Atr,
53    /// An at-rule with a declarations body (`@font-face`, `@page`, …).
54    Atd,
55    /// `@keyframes`.
56    Atk,
57    /// A statement at-rule (`@import`, `@charset`, …).
58    Ats,
59    /// End of input.
60    Zz,
61}
62
63impl Tin {
64    /// The token's grammar name, as `css-grammar.jsonic` writes it.
65    pub fn name(self) -> &'static str {
66        match self {
67            Tin::Ob => "#OB",
68            Tin::Cb => "#CB",
69            Tin::Cl => "#CL",
70            Tin::Ca => "#CA",
71            Tin::Tx => "#TX",
72            Tin::Gc => "#GC",
73            Tin::Vl => "#VL",
74            Tin::Cc => "#CC",
75            Tin::Atr => "#ATR",
76            Tin::Atd => "#ATD",
77            Tin::Atk => "#ATK",
78            Tin::Ats => "#ATS",
79            Tin::Zz => "#ZZ",
80        }
81    }
82
83    /// A human description, for diagnostics and diagram legends.
84    pub fn describe(self) -> &'static str {
85        match self {
86            Tin::Ob => "{ — start of a block",
87            Tin::Cb => "} — end of a block",
88            Tin::Cl => ": — declaration separator",
89            Tin::Ca => "; — declaration terminator",
90            Tin::Tx => "a selector, keyframe value, or property name",
91            Tin::Gc => ", — selector-group separator",
92            Tin::Vl => "a declaration value (raw text)",
93            Tin::Cc => "a comment",
94            Tin::Atr => "an at-rule with a rules body (@media, @supports, …)",
95            Tin::Atd => "an at-rule with a declarations body (@font-face, @page)",
96            Tin::Atk => "@keyframes",
97            Tin::Ats => "a statement at-rule (@import, @charset, …)",
98            Tin::Zz => "end of input",
99        }
100    }
101}
102
103/// A parse failure.
104///
105/// `code` is the contract: the shared fixtures pin `ERROR:<code>` and compare
106/// it exactly. [`Css::parse`](crate::Css::parse) raises only the two codes the
107/// canonical port inherits from the engine — `unterminated_comment` and
108/// `unexpected` — and this crate declares none of its own, matching
109/// `tabnas.plugin.json` (`errorCodes: []`). The engine's own tree form can
110/// also stop with `cancel`, at [`TREE_RULE_DEPTH`](crate::TREE_RULE_DEPTH).
111#[derive(Clone, Debug, PartialEq, Eq)]
112pub struct Error {
113    /// The error code, compared exactly by the conformance fixtures.
114    pub code: String,
115    /// A human-readable explanation.
116    pub message: String,
117    /// 1-based line where the failure was found.
118    pub line: usize,
119    /// 1-based column (UTF-16 code units) where the failure was found.
120    pub column: usize,
121}
122
123impl fmt::Display for Error {
124    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
125        write!(
126            f,
127            "[css/{}]: {} (line {}, column {})",
128            self.code, self.message, self.line, self.column
129        )
130    }
131}
132
133impl std::error::Error for Error {}
134
135/// The scanners' "there is no valid end here" result: an unclosed
136/// `/* ... */`.
137///
138/// A distinct value rather than an index, because the caller's response is a
139/// REJECTION, not a shorter span. Running the scan to end of source instead is
140/// how an unclosed comment in an at-rule prelude quietly became part of the
141/// prelude: `@import/*red` parsed as an import of `/*red`.
142const UNTERMINATED: usize = usize::MAX;
143
144/// What [`scan_to_brace_or_end`] found first.
145#[derive(Clone, Copy, PartialEq, Eq, Debug)]
146enum Kind {
147    /// A top-level `{` — so the run just scanned is a selector or a prelude.
148    Selector,
149    /// A top-level `;`/`}` or end of input — so it is a property or params.
150    Decl,
151}
152
153/// Rule names at which a `/* */` comment is captured as a NODE: the
154/// statement / declaration / keyframe LIST readers, which lex the first token
155/// of each item.
156///
157/// The item builders (`statement`/`decl`/`keyframe`) are deliberately absent.
158/// They reuse the token the list reader already produced, so a comment seen
159/// mid-construct — between a property and its `:`, say — is read under a
160/// builder, declined here, and skipped as insignificant whitespace.
161///
162/// The block wrappers are present because their empty-block `#OB #CB`
163/// lookahead lexes the first body token, so a comment right after `{` is
164/// captured there.
165const COMMENT_NODE_RULES: [&str; 6] = [
166    "items",
167    "decls",
168    "kfitems",
169    "declbody",
170    "rulesbody",
171    "kfbody",
172];
173
174fn is_comment_node_rule(name: &str) -> bool {
175    COMMENT_NODE_RULES.contains(&name)
176}
177
178/// At-keywords whose body is a declaration list rather than a rules list.
179const DECLS_KW: [&str; 7] = [
180    "font-face",
181    "page",
182    "viewport",
183    "-ms-viewport",
184    "counter-style",
185    "property",
186    "font-palette-values",
187];
188
189/// `^(-[a-z]+-)?keyframes$`, without a regex engine.
190fn is_keyframes_kw(kw: &str) -> bool {
191    if kw == "keyframes" {
192        return true;
193    }
194    match vendor_prefix(kw) {
195        Some(v) => &kw[v.len()..] == "keyframes",
196        None => false,
197    }
198}
199
200// The heads of the at-rule nodes: `makeAtRules`, `makeAtDecls`,
201// `makeKeyframes` and `makeAtStmt` in the canonical port, each without the
202// body array the grammar's constructor adds. Key order is JavaScript's: a
203// key assigned twice keeps its first place.
204
205/// The head of a block at-rule whose body is a list of rules (`@media`,
206/// `@supports`, `@document`, `@host`, and any other with a rules body).
207pub(crate) fn at_rules_head(kw: &str, prelude: &str) -> Value {
208    let mut node = object();
209    match kw {
210        "host" => put(&mut node, "type", text("host")),
211        _ if "document" == kw || kw.ends_with("-document") => {
212            // `vendor` is always present on a document node, the empty
213            // string when the at-keyword carries no prefix.
214            put(&mut node, "type", text("document"));
215            put(&mut node, "document", text(prelude));
216            put(&mut node, "vendor", text(vendor_prefix(kw).unwrap_or("")));
217        }
218        _ => {
219            put(&mut node, "type", text(kw));
220            put(&mut node, kw, text(prelude));
221        }
222    }
223    node
224}
225
226/// The head of a block at-rule whose body is declarations (`@font-face`,
227/// `@page`, and any other with a declarations body). A `@page` prelude is a
228/// selector group: `@page toc, index:blank`.
229pub(crate) fn at_decls_head(kw: &str, prelude: &str) -> Value {
230    let mut node = object();
231    put(&mut node, "type", text(kw));
232    if "page" == kw {
233        let selectors = split_selectors(prelude)
234            .into_iter()
235            .map(Value::String)
236            .collect();
237        put(&mut node, "selectors", Value::array(selectors));
238    }
239    node
240}
241
242/// The head of a `@keyframes` block, possibly vendor-prefixed.
243pub(crate) fn keyframes_head(kw: &str, name: &str) -> Value {
244    let mut node = object();
245    put(&mut node, "type", text("keyframes"));
246    put(&mut node, "name", text(name));
247    if let Some(vendor) = vendor_prefix(kw) {
248        put(&mut node, "vendor", text(vendor));
249    }
250    node
251}
252
253/// A statement at-rule (`@import`, `@charset`, `@namespace`, …), whole: the
254/// at-keyword is the node type and the field carrying its params.
255///
256/// `@custom-media --name <query>` splits its params into a name and a media
257/// query, as `/^(--\S+)\s*([\s\S]*)$/` does in the canonical port: `--` and
258/// at least one non-space character, then the rest, trimmed.
259pub(crate) fn at_stmt_head(kw: &str, params: &str) -> Value {
260    let mut node = object();
261    if "custom-media" == kw && params.starts_with("--") {
262        let name_end = params.find(es_is_whitespace).unwrap_or(params.len());
263        if 2 < name_end {
264            put(&mut node, "type", text("custom-media"));
265            put(&mut node, "name", text(&params[..name_end]));
266            put(&mut node, "media", text(es_trim(&params[name_end..])));
267            return node;
268        }
269    }
270    put(&mut node, "type", text(kw));
271    put(&mut node, kw, text(params));
272    node
273}
274
275/// `^(-[a-z]+-)`: a vendor prefix on an at-keyword, e.g. `-webkit-keyframes`.
276pub fn vendor_prefix(kw: &str) -> Option<&str> {
277    let b = kw.as_bytes();
278    if b.first() != Some(&b'-') {
279        return None;
280    }
281    let mut i = 1;
282    while i < b.len() && b[i].is_ascii_lowercase() {
283        i += 1;
284    }
285    if 1 < i && b.get(i) == Some(&b'-') {
286        Some(&kw[..i + 1])
287    } else {
288        None
289    }
290}
291
292/// Whether `c` is whitespace as ECMAScript defines it, which is NOT what
293/// [`char::is_whitespace`] answers.
294///
295/// The canonical port trims with JavaScript's `String.prototype.trim`, whose
296/// set is ECMAScript WhiteSpace plus LineTerminator. Rust's set is the Unicode
297/// `White_Space` property. They differ by exactly two code points, and both
298/// differences change the AST:
299///
300/// - **U+FEFF** (zero-width no-break space, the byte-order mark) is
301///   ECMAScript whitespace and is not Unicode `White_Space`. A stylesheet that
302///   opens with a byte-order mark yields the selector `a` in the canonical
303///   port and would yield `\u{feff}a` under [`str::trim`].
304/// - **U+0085** (next line) is Unicode `White_Space` and is not ECMAScript
305///   whitespace. `@media x \u{85}{…}` keeps that character in its prelude in
306///   the canonical port and would lose it under [`str::trim`].
307///
308/// Everything else in either set is in both: the ASCII controls, U+0020, the
309/// `Zs` category, and the two line separators U+2028 and U+2029.
310pub fn es_is_whitespace(c: char) -> bool {
311    ('\u{feff}' == c) || (c.is_whitespace() && '\u{85}' != c)
312}
313
314/// Trim as JavaScript's `String.prototype.trim` does. See
315/// [`es_is_whitespace`] for why this is not [`str::trim`].
316pub fn es_trim(s: &str) -> &str {
317    s.trim_matches(es_is_whitespace)
318}
319
320/// The column width of `s`: its length in UTF-16 code units, which is what a
321/// JavaScript string index counts.
322pub(crate) fn col_width(s: &str) -> usize {
323    s.chars().map(char::len_utf16).sum()
324}
325
326/// Slice `src` by byte offsets, clamping to something sliceable.
327///
328/// This is the Rust half of the Go port's `clampLen`, and it exists for the
329/// same reason: the scanners deliberately overshoot. An `i += 2` over a `\`
330/// escape, an unterminated string or an unterminated comment can step past the
331/// end of the source, and JavaScript CLAMPS an out-of-range slice bound where
332/// Go panics — so the Go port clamps to reproduce the JavaScript result rather
333/// than invent a new one.
334///
335/// Rust panics on one bound more than Go does: an index inside a multi-byte
336/// character. Every index these scanners RETURN is at an ASCII character or at
337/// the end of the source, because each one stops at `{`, `}`, `;`, `,`, `*/`,
338/// a quote or end of input, so the fallback below is unreachable in practice —
339/// it is here so that a future scanner change degrades to a shorter span
340/// instead of a panic in a caller's parse.
341fn safe_slice(src: &str, start: usize, end: usize) -> &str {
342    let mut start = start.min(src.len());
343    let mut end = end.clamp(start, src.len());
344    while 0 < start && !src.is_char_boundary(start) {
345        start -= 1;
346    }
347    while start < end && !src.is_char_boundary(end) {
348        end -= 1;
349    }
350    &src[start..end]
351}
352
353/// The index of the `*` that opens the `*/` closing the comment at `i`, or
354/// `None` if there is no closing `*/`.
355fn find_comment_end(src: &[u8], i: usize) -> Option<usize> {
356    let mut j = i + 2;
357    while j + 1 < src.len() {
358        if src[j] == b'*' && src[j + 1] == b'/' {
359            return Some(j);
360        }
361        j += 1;
362    }
363    None
364}
365
366/// The index after a closed `/* ... */`, or [`UNTERMINATED`].
367///
368/// It does NOT run to end of source. An unclosed `/* ... */` is an error, not
369/// a comment to the end of the file — which is what both the engine's builtin
370/// comment matcher and reworkcss do, and what `test/spec/reworkcss.tsv` pins.
371fn skip_comment(src: &[u8], i: usize) -> usize {
372    match find_comment_end(src, i) {
373        // `e + 1` is the `/`, so `e + 2` is always within the source.
374        Some(e) => e + 2,
375        None => UNTERMINATED,
376    }
377}
378
379/// Skip a quoted string; returns the index after the closing quote.
380fn skip_string(src: &[u8], mut i: usize) -> usize {
381    let quote = src[i];
382    i += 1;
383    while i < src.len() {
384        if src[i] == b'\\' {
385            i += 2;
386            continue;
387        }
388        if src[i] == quote {
389            return i + 1;
390        }
391        i += 1;
392    }
393    // A trailing backslash steps i ONE PAST the end. That overshoot is not a
394    // defect to clamp away here: the canonical port returns it too, and its
395    // column arithmetic counts it. See [`css_token`] and [`safe_slice`].
396    i
397}
398
399/// Scan a key prelude: where it ends, and whether it is a selector (a
400/// top-level `{` comes first) or a declaration (a top-level `;`/`}` or end of
401/// input comes first). Strings, `()`, `[]` and comments are skipped.
402fn scan_to_brace_or_end(src: &[u8], i: usize) -> (Kind, usize) {
403    let mut scan = Scan { i, depth: 0 };
404    loop {
405        if let Some(found) = scan.step(src) {
406            return found;
407        }
408    }
409}
410
411/// [`scan_to_brace_or_end`] one step at a time. The position and the
412/// bracket depth are its whole state, so two scans in the same state go on
413/// to the same answer: what [`BraceScans`] relies on.
414#[derive(Clone, Copy, Debug)]
415struct Scan {
416    i: usize,
417    depth: usize,
418}
419
420impl Scan {
421    /// Take one step: past a string, a comment, an escape or one byte.
422    /// `Some` is the scan's answer.
423    fn step(&mut self, src: &[u8]) -> Option<(Kind, usize)> {
424        let Some(&c) = src.get(self.i) else {
425            // i may be one past the end; see skip_string.
426            return Some((Kind::Decl, self.i));
427        };
428        if c == b'"' || c == b'\'' {
429            self.i = skip_string(src, self.i);
430            return None;
431        }
432        if c == b'/' && src.get(self.i + 1) == Some(&b'*') {
433            let n = skip_comment(src, self.i);
434            if UNTERMINATED == n {
435                return Some((Kind::Decl, UNTERMINATED));
436            }
437            self.i = n;
438            return None;
439        }
440        // A CSS escape (`\(`, `\'`, `\3A `, …) hides the next character from
441        // the structural scan.
442        if c == b'\\' {
443            self.i += 2;
444            return None;
445        }
446        if c == b'(' || c == b'[' {
447            self.depth += 1;
448        } else if c == b')' || c == b']' {
449            self.depth = self.depth.saturating_sub(1);
450        } else if 0 == self.depth {
451            if c == b'{' {
452                return Some((Kind::Selector, self.i));
453            }
454            if c == b';' || c == b'}' {
455                return Some((Kind::Decl, self.i));
456            }
457        }
458        self.i += 1;
459        None
460    }
461}
462
463/// [`scan_to_brace_or_end`] for every token of a parse, in linear time.
464///
465/// Each selector of a group, and each value of a keyframe-selector group,
466/// is classified by scanning to the group's `{`, so a group of `k` items
467/// scanned from each item's start costs `k` times the group's length: a
468/// 200 KB list of selectors took 9 s. The scan from an item's start is the
469/// scan from the group's first item, resumed there, whenever that scan
470/// passes the item's start at bracket depth 0 (the state [`Scan`] carries,
471/// outside any string, comment or escape, which it steps over whole). So
472/// the answer is kept with a cursor that follows the tokens forward, and a
473/// later start the cursor reaches in that state takes the kept answer. Any
474/// other start scans afresh and becomes the one kept. The cursor only moves
475/// forward, so a parse's scanning is linear in its length.
476#[derive(Clone, Copy, Debug)]
477struct BraceScans {
478    from: usize,
479    found: (Kind, usize),
480    cursor: Scan,
481}
482
483/// The `ctx.u` key [`BraceScans`] is kept under, for the parse.
484pub(crate) const BRACE_SCAN: &str = "tabnas-css/brace-scan";
485
486impl BraceScans {
487    /// The answer for a token starting at `s_i`, keeping the state under
488    /// [`BRACE_SCAN`] with one map lookup: the matcher asks on every `#TX`,
489    /// and a flat stylesheet is mostly misses.
490    fn scan(ctx: &mut Context, src: &[u8], s_i: usize) -> (Kind, usize) {
491        let Some(Value::Array(slot)) = ctx.u.get_mut(BRACE_SCAN) else {
492            let found = scan_to_brace_or_end(src, s_i);
493            let kept = BraceScans::fresh(s_i, found);
494            ctx.u
495                .insert(BRACE_SCAN.to_string(), Value::array(kept.fields().to_vec()));
496            return found;
497        };
498        let fields = Arc::make_mut(slot);
499        if let Some(mut kept) = BraceScans::decode(fields) {
500            if kept.from <= s_i && s_i < kept.found.1 && kept.cursor.i <= s_i {
501                while kept.cursor.i < s_i && kept.cursor.step(src).is_none() {}
502                let reached = kept.cursor.i == s_i && 0 == kept.cursor.depth;
503                kept.encode(fields);
504                if reached {
505                    return kept.found;
506                }
507            }
508        }
509        // An unclosed comment's answer is kept too: a scan from a later start
510        // the cursor reaches meets the same `/*`, and the engine's recovery
511        // asks again at the very start it failed at, up to its skip budget.
512        let found = scan_to_brace_or_end(src, s_i);
513        BraceScans::fresh(s_i, found).encode(fields);
514        found
515    }
516
517    fn fresh(from: usize, found: (Kind, usize)) -> BraceScans {
518        BraceScans {
519            from,
520            found,
521            cursor: Scan { i: from, depth: 0 },
522        }
523    }
524
525    fn decode(fields: &[Value]) -> Option<BraceScans> {
526        let at = |n: usize| match fields.get(n) {
527            Some(Value::Number(v)) => Some(*v as usize),
528            _ => None,
529        };
530        let kind = if 0 == at(1)? {
531            Kind::Selector
532        } else {
533            Kind::Decl
534        };
535        Some(BraceScans {
536            from: at(0)?,
537            found: (kind, at(2)?),
538            cursor: Scan {
539                i: at(3)?,
540                depth: at(4)?,
541            },
542        })
543    }
544
545    fn numbers(&self) -> [f64; 5] {
546        let kind = match self.found.0 {
547            Kind::Selector => 0,
548            Kind::Decl => 1,
549        };
550        [
551            self.from,
552            kind,
553            self.found.1,
554            self.cursor.i,
555            self.cursor.depth,
556        ]
557        .map(|n| n as f64)
558    }
559
560    fn fields(&self) -> [Value; 5] {
561        self.numbers().map(Value::Number)
562    }
563
564    /// Write this state over `fields`, number by number when the shape fits.
565    fn encode(&self, fields: &mut Vec<Value>) {
566        let numbers = self.numbers();
567        if numbers.len() == fields.len() {
568            for (field, n) in fields.iter_mut().zip(numbers) {
569                match field {
570                    Value::Number(slot) => *slot = n,
571                    other => *other = Value::Number(n),
572                }
573            }
574        } else {
575            *fields = self.fields().to_vec();
576        }
577    }
578}
579
580/// Scan a single selector: to the next top-level `,` (a group separator) or
581/// `{`, or end of input. Strings, `()`, `[]` and comments are skipped.
582fn scan_selector_end(src: &[u8], mut i: usize) -> usize {
583    let mut depth = 0usize;
584    while i < src.len() {
585        let c = src[i];
586        if c == b'"' || c == b'\'' {
587            i = skip_string(src, i);
588            continue;
589        }
590        if c == b'/' && src.get(i + 1) == Some(&b'*') {
591            let n = skip_comment(src, i);
592            if UNTERMINATED == n {
593                return UNTERMINATED;
594            }
595            i = n;
596            continue;
597        }
598        if c == b'\\' {
599            i += 2;
600            continue;
601        }
602        if c == b'(' || c == b'[' {
603            depth += 1;
604        } else if c == b')' || c == b']' {
605            depth = depth.saturating_sub(1);
606        } else if 0 == depth && (c == b',' || c == b'{') {
607            return i;
608        }
609        i += 1;
610    }
611    // i may be one past the end; see skip_string.
612    i
613}
614
615/// Scan a declaration value or at-rule params: to the next top-level `;` or
616/// `}`, or end of input. Strings, `()`, `[]` and comments are skipped.
617fn scan_value_end(src: &[u8], mut i: usize) -> usize {
618    let mut depth = 0usize;
619    while i < src.len() {
620        let c = src[i];
621        if c == b'"' || c == b'\'' {
622            i = skip_string(src, i);
623            continue;
624        }
625        if c == b'/' && src.get(i + 1) == Some(&b'*') {
626            let n = skip_comment(src, i);
627            if UNTERMINATED == n {
628                return UNTERMINATED;
629            }
630            i = n;
631            continue;
632        }
633        if c == b'\\' {
634            i += 2;
635            continue;
636        }
637        if c == b'(' || c == b'[' {
638            depth += 1;
639        } else if c == b')' || c == b']' {
640            depth = depth.saturating_sub(1);
641        } else if 0 == depth && (c == b';' || c == b'}') {
642            return i;
643        }
644        i += 1;
645    }
646    // i may be one past the end; see skip_string.
647    i
648}
649
650/// Remove `/* ... */` comments from a selector or value run, leaving quoted
651/// strings (which may contain `/*`) untouched.
652pub fn strip_comments(s: &str) -> String {
653    if !s.contains("/*") {
654        return s.to_string();
655    }
656    let src = s.as_bytes();
657    let mut out: Vec<u8> = Vec::with_capacity(src.len());
658    let mut i = 0;
659    while i < src.len() {
660        let c = src[i];
661        if c == b'"' || c == b'\'' {
662            let j = skip_string(src, i);
663            out.extend_from_slice(&src[i..j.min(src.len())]);
664            i = j;
665            continue;
666        }
667        if c == b'/' && src.get(i + 1) == Some(&b'*') {
668            let n = skip_comment(src, i);
669            if UNTERMINATED == n {
670                // Unreachable once the callers reject an unclosed comment,
671                // but a sentinel assigned to i would spin. The rest of the
672                // span is comment text; drop it.
673                break;
674            }
675            i = n;
676            continue;
677        }
678        out.push(c);
679        i += 1;
680    }
681    String::from_utf8(out).unwrap_or_else(|e| String::from_utf8_lossy(e.as_bytes()).into_owned())
682}
683
684/// A CSS property name character.
685///
686/// Beyond the identifier characters this admits the legacy IE hack prefixes
687/// `*`, `#`, `/` (as in `//prop`) and a literal backslash, matching the
688/// reworkcss property pattern `\*?[-#\/\*\\\w]+(\[[0-9a-z_-]+\])?` — real
689/// stylesheets rely on them.
690fn is_prop_char(c: u8) -> bool {
691    c.is_ascii_alphanumeric()
692        || c == b'-'
693        || c == b'_'
694        || c == b'#'
695        || c == b'*'
696        || c == b'/'
697        || c == b'\\'
698}
699
700/// A `[0-9a-z_-]` character, the only content allowed in a property name's
701/// bracket suffix (e.g. `opacity[sqrt]`).
702fn is_prop_suffix_char(c: u8) -> bool {
703    c.is_ascii_digit() || c.is_ascii_lowercase() || c == b'-' || c == b'_'
704}
705
706/// Scan a property name: the run of property characters plus an optional
707/// `[...]` suffix. Returns the index after the name (`== i` if there is none).
708fn scan_prop_end(src: &[u8], i: usize) -> usize {
709    let mut e_i = i;
710    while e_i < src.len() && is_prop_char(src[e_i]) {
711        e_i += 1;
712    }
713    if e_i == i {
714        return i;
715    }
716    if src.get(e_i) == Some(&b'[') {
717        let mut b_i = e_i + 1;
718        while b_i < src.len() && is_prop_suffix_char(src[b_i]) {
719            b_i += 1;
720        }
721        if e_i + 1 < b_i && src.get(b_i) == Some(&b']') {
722            return b_i + 1;
723        }
724    }
725    e_i
726}
727
728/// An at-keyword character: letters, digits and `-`.
729fn is_at_char(c: u8) -> bool {
730    c.is_ascii_alphanumeric() || c == b'-'
731}
732
733/// Split a selector-group prelude on its top-level commas (commas inside
734/// strings, `()` or `[]` are part of a single selector), stripping comments
735/// and trimming each selector. An empty prelude yields an empty list.
736pub fn split_selectors(prelude: &str) -> Vec<String> {
737    let src = prelude.as_bytes();
738    let mut out = Vec::new();
739    let mut i = 0;
740    while i <= src.len() {
741        let mut end = scan_selector_end(src, i);
742        if UNTERMINATED == end {
743            // Unreachable in practice: the prelude was scanned (and an
744            // unclosed comment rejected) before it got here. Treated as "the
745            // rest" so a sentinel can never index or spin.
746            end = src.len();
747        }
748        let one = es_trim(&strip_comments(safe_slice(prelude, i, end))).to_string();
749        if !one.is_empty() {
750            out.push(one);
751        }
752        if end >= src.len() {
753            break;
754        }
755        i = end + 1;
756    }
757    out
758}
759
760// ---------------------------------------------------------------------------
761// The `cssToken` matcher, as the engine runs it.
762
763/// The token number a matcher gives a token the engine is to resolve by
764/// NAME, as it lexes: the six this grammar names itself (`#CC`, `#GC` and
765/// the four at-rule tokens).
766///
767/// The engine numbers a name when it is first registered, and
768/// [`Tabnas::merge`](tabnas::Tabnas::merge) renumbers every custom name in
769/// the merged instance without running the plugin again, so a number
770/// captured at install names another token there. The canonical port's
771/// `lex.token('#CC', …)` resolves the name on every call for the same
772/// reason. `#TX` and `#VL` are the engine's own, with fixed numbers.
773const BY_NAME: EngineTin = -1;
774
775/// Where a scan ran past the end of the source, and by how much: the
776/// `ctx.u` key [`css_token`] records it under and [`lex_subscriber`] reads.
777pub(crate) const OVERSHOOT: &str = "tabnas-css/overshoot";
778
779/// The most of the source an `unterminated_comment` diagnostic shows. The
780/// span runs from the scan's start to the end of the source, which for a
781/// large stylesheet would be most of it; the message is informative, not the
782/// contract, so it is cut here.
783const BAD_SPAN: usize = 32;
784
785/// What [`plan_token`] decided for the characters at the cursor.
786enum Plan {
787    /// Not this matcher's: the engine's builtin matchers take it.
788    Decline,
789    /// An unclosed `/* ... */` behind the cursor: `unterminated_comment`.
790    Bad,
791    /// A token, and the byte offset just past it, which may be ONE past the
792    /// end of the source (see [`css_token`]).
793    Emit {
794        name: &'static str,
795        tin: EngineTin,
796        val: Value,
797        raw: String,
798        end: usize,
799    },
800}
801
802/// Decide the token at byte `s_i` of `src`, read under the rule `rule`.
803///
804/// `buildCssTokenMatcher` in `ts/src/css.ts`, step for step: whitespace is
805/// declined (the engine's space and line matchers take it), a comment is a
806/// node only at a list position, a value runs to the next top-level `;` or
807/// `}` under `declval`, a top-level `,` separates a selector group, `@`
808/// starts an at-rule, `{` `}` `;` are the grammar's, and anything else is a
809/// selector or a property name by `{`-before-`;` lookahead.
810fn plan_token(
811    src: &str,
812    s_i: usize,
813    rule: &str,
814    lower: bool,
815    brace: &mut dyn FnMut(usize) -> (Kind, usize),
816) -> Plan {
817    let bytes = src.as_bytes();
818    let Some(&c) = bytes.get(s_i) else {
819        return Plan::Decline;
820    };
821    if c == b' ' || c == b'\t' || c == b'\r' || c == b'\n' {
822        return Plan::Decline;
823    }
824    let emit = |name, tin, val: String, raw: &str, end| Plan::Emit {
825        name,
826        tin,
827        val: Value::String(val),
828        raw: raw.to_string(),
829        end,
830    };
831
832    // Comments: a node at a list position, otherwise declined, and then
833    // skipped by the engine's builtin comment matcher.
834    if c == b'/' && bytes.get(s_i + 1) == Some(&b'*') {
835        if !is_comment_node_rule(rule) {
836            return Plan::Decline;
837        }
838        return match find_comment_end(bytes, s_i) {
839            None => Plan::Bad,
840            Some(e) => emit(
841                "#CC",
842                BY_NAME,
843                safe_slice(src, s_i + 2, e).to_string(),
844                safe_slice(src, s_i, e + 2),
845                e + 2,
846            ),
847        };
848    }
849
850    // Value position: a declaration value up to the next top-level `;`/`}`,
851    // comments stripped and surrounding space trimmed.
852    if "declval" == rule {
853        if c == b'{' || c == b'}' || c == b';' || c == b':' {
854            return Plan::Decline;
855        }
856        let end = scan_value_end(bytes, s_i);
857        if UNTERMINATED == end {
858            return Plan::Bad;
859        }
860        let raw = safe_slice(src, s_i, end);
861        let val = es_trim(&strip_comments(raw)).to_string();
862        return emit("#VL", TIN_VL, val, raw, end);
863    }
864
865    // A top-level selector-group comma.
866    if c == b',' {
867        return emit("#GC", BY_NAME, ",".to_string(), ",", s_i + 1);
868    }
869
870    // An at-rule: block or statement by `{`-before-`;` lookahead, and a
871    // block's body by keyword. The token's value is the node's head: its
872    // type and the fields read from the keyword and the prelude (block) or
873    // params (statement), so that a reader of the rule passes (the
874    // transducer part, `alchemy/transduce.alc`) sees the node as the walk
875    // builds it, a pass carrying a token's value and never its `use` bag.
876    if c == b'@' {
877        let mut k_end = s_i + 1;
878        while k_end < bytes.len() && is_at_char(bytes[k_end]) {
879            k_end += 1;
880        }
881        let kw = safe_slice(src, s_i + 1, k_end).to_string();
882        let (kind, index) = brace(s_i);
883        if UNTERMINATED == index {
884            return Plan::Bad;
885        }
886        if Kind::Selector == kind {
887            // At-rule preludes KEEP their comments (they are only trimmed),
888            // matching reworkcss.
889            let prelude = es_trim(safe_slice(src, k_end, index)).to_string();
890            let (name, tin) = if is_keyframes_kw(&kw) {
891                ("#ATK", BY_NAME)
892            } else if DECLS_KW.contains(&kw.as_str()) {
893                ("#ATD", BY_NAME)
894            } else {
895                ("#ATR", BY_NAME)
896            };
897            let head = match name {
898                "#ATK" => keyframes_head(&kw, &prelude),
899                "#ATD" => at_decls_head(&kw, &prelude),
900                _ => at_rules_head(&kw, &prelude),
901            };
902            let raw = safe_slice(src, s_i, index);
903            return Plan::Emit {
904                name,
905                tin,
906                val: head,
907                raw: raw.to_string(),
908                end: index,
909            };
910        }
911        // A statement at-rule's params run to the next top-level `;`/`}`. A
912        // `;` is consumed (it ends the statement); a `}` or the end of the
913        // source is left for the enclosing rule.
914        let p_end = scan_value_end(bytes, k_end);
915        if UNTERMINATED == p_end {
916            return Plan::Bad;
917        }
918        let params = es_trim(safe_slice(src, k_end, p_end)).to_string();
919        let end = if bytes.get(p_end) == Some(&b';') {
920            p_end + 1
921        } else {
922            p_end
923        };
924        let raw = safe_slice(src, s_i, end);
925        return Plan::Emit {
926            name: "#ATS",
927            tin: BY_NAME,
928            val: at_stmt_head(&kw, &params),
929            raw: raw.to_string(),
930            end,
931        };
932    }
933
934    // Other fixed punctuation belongs to the grammar. `:` does not: a run
935    // that starts with one is scanned below, as in the canonical port.
936    if c == b'{' || c == b'}' || c == b';' {
937        return Plan::Decline;
938    }
939
940    // A selector or a property name, by `{`-before-`;` lookahead.
941    let (kind, index) = brace(s_i);
942    if UNTERMINATED == index {
943        return Plan::Bad;
944    }
945    if Kind::Selector == kind {
946        // One selector of a (possible) group: up to the next top-level `,`
947        // or `{`, comments stripped and surrounding space trimmed.
948        let end = scan_selector_end(bytes, s_i);
949        if UNTERMINATED == end {
950            return Plan::Bad;
951        }
952        let raw = safe_slice(src, s_i, end);
953        let val = es_trim(&strip_comments(raw)).to_string();
954        return emit("#TX", TIN_TX, val, raw, end);
955    }
956
957    // A property name: the run of property characters up to `:`,
958    // whitespace, `;` or `}`. `/` and `*` are property characters (the
959    // `*prop` and `//prop` IE hacks), so a trailing `/*...*/` hack comment
960    // lands inside the scanned name; comments are stripped out of it, as
961    // reworkcss does.
962    let e_i = scan_prop_end(bytes, s_i);
963    if e_i == s_i {
964        return Plan::Decline;
965    }
966    let raw = safe_slice(src, s_i, e_i);
967    let mut prop = strip_comments(raw);
968    if prop.is_empty() {
969        return Plan::Decline;
970    }
971    if lower {
972        prop = prop.to_lowercase();
973    }
974    emit("#TX", TIN_TX, prop, raw, e_i)
975}
976
977/// The `cssToken` matcher, registered as `@css-token` and named by the
978/// grammar's options at order 1e5, ahead of every builtin matcher.
979///
980/// Two things here are deliberate and easy to "fix" wrongly:
981///
982/// - The end of a token may be ONE PAST the source. A `\` at the last
983///   character is an escape whose escaped character is not there, and the
984///   scanners' `i += 2` steps off the end. The canonical port advances its
985///   column by that unclamped width and takes the CLAMPED span for the
986///   token's text, so `@host\` yields a node ending at column 7 inside a
987///   stylesheet ending at column 8, one past a source six characters long.
988///   The engine will not move its cursor past the end, so the overshoot is
989///   recorded under [`OVERSHOOT`] and [`lex_subscriber`] adds it to the
990///   end-of-source token, the only token that can follow it.
991/// - An unclosed comment is reported WITHOUT advancing, so the diagnostic
992///   points at the start of the scan, as the canonical port's does.
993///
994/// A matcher must never advance and then decline: the engine captured its
995/// dispatch state before the custom matchers ran.
996pub(crate) fn css_token(
997    lexer: &mut Lexer<'_>,
998    rule: &mut Rule,
999    ctx: &mut Context,
1000    lower: bool,
1001) -> Option<Token> {
1002    let point = lexer.point();
1003    let s_i = point.site.si;
1004    let (plan, chars, overshoot) = {
1005        let src = lexer.source();
1006        let bytes = src.as_bytes();
1007        let mut brace = |at: usize| BraceScans::scan(ctx, bytes, at);
1008        let plan = plan_token(src, s_i, rule.name.as_str(), lower, &mut brace);
1009        let (chars, overshoot) = match &plan {
1010            Plan::Decline => return None,
1011            Plan::Bad => (
1012                safe_slice(src, s_i, src.len())
1013                    .chars()
1014                    .take(BAD_SPAN)
1015                    .count(),
1016                0,
1017            ),
1018            Plan::Emit { end, .. } => (
1019                safe_slice(src, s_i, (*end).min(src.len())).chars().count(),
1020                end.saturating_sub(src.len()),
1021            ),
1022        };
1023        (plan, chars, overshoot)
1024    };
1025    match plan {
1026        Plan::Decline => None,
1027        Plan::Bad => Some(lexer.bad_span(
1028            "unterminated_comment",
1029            point.site.pos,
1030            point.site.pos + chars,
1031        )),
1032        Plan::Emit {
1033            name,
1034            tin,
1035            val,
1036            raw,
1037            ..
1038        } => {
1039            lexer.advance_chars(chars);
1040            if 0 < overshoot {
1041                ctx.u
1042                    .insert(OVERSHOOT.to_string(), Value::Number(overshoot as f64));
1043            }
1044            Some(lexer.token(name, tin, val, raw, point))
1045        }
1046    }
1047}
1048
1049/// The plugin's lex subscriber: it sees every token the engine fetches.
1050///
1051/// - The end-of-source token takes the column overshoot [`css_token`]
1052///   recorded, so the stylesheet's end and an error at the end of the source
1053///   land where the canonical port puts them.
1054/// - It once dropped the unconsumed lookahead when a bad token arrived
1055///   behind a good one, so that an unclosed comment hidden from the
1056///   property scan was the error reported rather than `unexpected` at the
1057///   property. tabnas/parser#274 made the engine raise a fetched bad token
1058///   at once, as the canonical does, and record and skip it under
1059///   recovery, so that work is the engine's now. `test/spec/comments.tsv`
1060///   and `tests/plugin.rs` pin the answers in every mode.
1061pub(crate) fn lex_subscriber(token: &mut Token, _rule: &mut Rule, ctx: &mut Context) {
1062    if TIN_ZZ == token.tin {
1063        if let Some(Value::Number(n)) = ctx.u.get(OVERSHOOT) {
1064            token.site.ci += *n as usize;
1065        }
1066    }
1067}
1068
1069/// The number of astral characters (those two UTF-16 code units wide) in
1070/// scalars `[from, to)` of `src`.
1071pub(crate) fn astral_between(src: &str, from: usize, to: usize) -> usize {
1072    src.chars()
1073        .skip(from)
1074        .take(to.saturating_sub(from))
1075        .filter(|c| 2 == c.len_utf16())
1076        .count()
1077}
1078
1079/// An engine error as this crate's, with the column in UTF-16 code units.
1080///
1081/// The engine counts a column in Unicode scalars since the last reset (a
1082/// `\n` in any token, or the end of a run of line characters); the canonical
1083/// port counts UTF-16 code units since the same points. The two differ by
1084/// the astral characters between the reset and the error, which is what is
1085/// added here. The message is the engine's, and is not the contract.
1086impl From<TabnasError> for Error {
1087    fn from(error: TabnasError) -> Error {
1088        let from = error.pos.saturating_sub(error.col.saturating_sub(1));
1089        let wide = astral_between(&error.full_source, from, error.pos);
1090        Error {
1091            code: error.code,
1092            message: error.detail,
1093            line: error.row,
1094            column: error.col + wide,
1095        }
1096    }
1097}