Skip to main content

tabnas_css/
lex.rs

1/* Copyright (c) 2025 Richard Rodger, MIT License */
2
3//! The lexer: the single `cssToken` matcher the plugin installs on the
4//! engine, and the scanners it reads spans with.
5//!
6//! CSS is context-sensitive — the same characters can begin a selector, a
7//! property or a value — so the matcher owns the hard tokenisation and the
8//! grammar only assembles typed nodes from what it emits. Which token comes
9//! out depends on the rule that is active when the character is read: the
10//! engine reads a token only when the alternate being tried needs one, under
11//! the rule trying it, and passes that rule to the matcher.
12//!
13//! See `ts/src/css.ts` for the canonical commentary; the three ports are kept
14//! in step, and this module follows `buildCssTokenMatcher` there line for
15//! line.
16
17use std::fmt;
18use std::sync::Arc;
19
20use tabnas::{
21    Context, Lexer, Rule, TabnasError, Tin as EngineTin, Token, Value, TIN_TX, TIN_VL, TIN_ZZ,
22};
23
24use crate::plugin::{object, put, text};
25
26/// A token kind.
27///
28/// `Ob`/`Cb`/`Cl`/`Ca` are the fixed punctuation the canonical port borrows
29/// from the engine (`{` `}` `:` and `;`, the last remapped onto jsonic's
30/// member separator). The rest are this grammar's own: a custom name is
31/// required for a comment NODE because the engine's builtin comment tin is in
32/// the parser's ignore set, so emitting it would silently drop the node.
33#[derive(Clone, Copy, Debug, PartialEq, Eq)]
34pub enum Tin {
35    /// `{` — start of a block.
36    Ob,
37    /// `}` — end of a block.
38    Cb,
39    /// `:` — declaration separator.
40    Cl,
41    /// `;` — declaration terminator.
42    Ca,
43    /// A selector, keyframe value, or property name.
44    Tx,
45    /// `,` — selector-group separator.
46    Gc,
47    /// A declaration value (raw text).
48    Vl,
49    /// A comment, at a statement position.
50    Cc,
51    /// An at-rule with a rules body (`@media`, `@supports`, …).
52    Atr,
53    /// An at-rule with a declarations body (`@font-face`, `@page`, …).
54    Atd,
55    /// `@keyframes`.
56    Atk,
57    /// A statement at-rule (`@import`, `@charset`, …).
58    Ats,
59    /// End of input.
60    Zz,
61}
62
63impl Tin {
64    /// The token's grammar name, as `css-grammar.jsonic` writes it.
65    pub fn name(self) -> &'static str {
66        match self {
67            Tin::Ob => "#OB",
68            Tin::Cb => "#CB",
69            Tin::Cl => "#CL",
70            Tin::Ca => "#CA",
71            Tin::Tx => "#TX",
72            Tin::Gc => "#GC",
73            Tin::Vl => "#VL",
74            Tin::Cc => "#CC",
75            Tin::Atr => "#ATR",
76            Tin::Atd => "#ATD",
77            Tin::Atk => "#ATK",
78            Tin::Ats => "#ATS",
79            Tin::Zz => "#ZZ",
80        }
81    }
82
83    /// A human description, for diagnostics and diagram legends.
84    pub fn describe(self) -> &'static str {
85        match self {
86            Tin::Ob => "{ — start of a block",
87            Tin::Cb => "} — end of a block",
88            Tin::Cl => ": — declaration separator",
89            Tin::Ca => "; — declaration terminator",
90            Tin::Tx => "a selector, keyframe value, or property name",
91            Tin::Gc => ", — selector-group separator",
92            Tin::Vl => "a declaration value (raw text)",
93            Tin::Cc => "a comment",
94            Tin::Atr => "an at-rule with a rules body (@media, @supports, …)",
95            Tin::Atd => "an at-rule with a declarations body (@font-face, @page)",
96            Tin::Atk => "@keyframes",
97            Tin::Ats => "a statement at-rule (@import, @charset, …)",
98            Tin::Zz => "end of input",
99        }
100    }
101}
102
103/// A parse failure.
104///
105/// `code` is the contract: the shared fixtures pin `ERROR:<code>` and compare
106/// it exactly. [`Css::parse`](crate::Css::parse) raises only the two codes the
107/// canonical port inherits from the engine — `unterminated_comment` and
108/// `unexpected` — and this crate declares none of its own, matching
109/// `tabnas.plugin.json` (`errorCodes: []`). The engine's own tree form can
110/// also stop with `cancel`, at [`TREE_RULE_DEPTH`](crate::TREE_RULE_DEPTH).
111#[derive(Clone, Debug, PartialEq, Eq)]
112pub struct Error {
113    /// The error code, compared exactly by the conformance fixtures.
114    pub code: String,
115    /// A human-readable explanation.
116    pub message: String,
117    /// 1-based line where the failure was found.
118    pub line: usize,
119    /// 1-based column (UTF-16 code units) where the failure was found.
120    pub column: usize,
121}
122
123impl fmt::Display for Error {
124    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
125        write!(
126            f,
127            "[css/{}]: {} (line {}, column {})",
128            self.code, self.message, self.line, self.column
129        )
130    }
131}
132
133impl std::error::Error for Error {}
134
135/// The scanners' "there is no valid end here" result: an unclosed
136/// `/* ... */`.
137///
138/// A distinct value rather than an index, because the caller's response is a
139/// REJECTION, not a shorter span. Running the scan to end of source instead is
140/// how an unclosed comment in an at-rule prelude quietly became part of the
141/// prelude: `@import/*red` parsed as an import of `/*red`.
142const UNTERMINATED: usize = usize::MAX;
143
144/// What [`scan_to_brace_or_end`] found first.
145#[derive(Clone, Copy, PartialEq, Eq, Debug)]
146enum Kind {
147    /// A top-level `{` — so the run just scanned is a selector or a prelude.
148    Selector,
149    /// A top-level `;`/`}` or end of input — so it is a property or params.
150    Decl,
151}
152
153/// Rule names at which a `/* */` comment is captured as a NODE: the
154/// statement / declaration / keyframe LIST readers, which lex the first token
155/// of each item.
156///
157/// The item builders (`statement`/`decl`/`keyframe`) are deliberately absent.
158/// They reuse the token the list reader already produced, so a comment seen
159/// mid-construct — between a property and its `:`, say — is read under a
160/// builder, declined here, and skipped as insignificant whitespace.
161///
162/// The block wrappers are present because their empty-block `#OB #CB`
163/// lookahead lexes the first body token, so a comment right after `{` is
164/// captured there.
165const COMMENT_NODE_RULES: [&str; 6] = [
166    "items",
167    "decls",
168    "kfitems",
169    "declbody",
170    "rulesbody",
171    "kfbody",
172];
173
174fn is_comment_node_rule(name: &str) -> bool {
175    COMMENT_NODE_RULES.contains(&name)
176}
177
178/// At-keywords whose body is a declaration list rather than a rules list.
179const DECLS_KW: [&str; 7] = [
180    "font-face",
181    "page",
182    "viewport",
183    "-ms-viewport",
184    "counter-style",
185    "property",
186    "font-palette-values",
187];
188
189/// `^(-[a-z]+-)?keyframes$`, without a regex engine.
190fn is_keyframes_kw(kw: &str) -> bool {
191    if kw == "keyframes" {
192        return true;
193    }
194    match vendor_prefix(kw) {
195        Some(v) => &kw[v.len()..] == "keyframes",
196        None => false,
197    }
198}
199
200// The heads of the at-rule nodes: `makeAtRules`, `makeAtDecls`,
201// `makeKeyframes` and `makeAtStmt` in the canonical port, each without the
202// body array the grammar's constructor adds. Key order is JavaScript's: a
203// key assigned twice keeps its first place.
204
205/// The head of a block at-rule whose body is a list of rules (`@media`,
206/// `@supports`, `@document`, `@host`, and any other with a rules body).
207pub(crate) fn at_rules_head(kw: &str, prelude: &str) -> Value {
208    let mut node = object();
209    match kw {
210        "host" => put(&mut node, "type", text("host")),
211        _ if "document" == kw || kw.ends_with("-document") => {
212            // `vendor` is always present on a document node, the empty
213            // string when the at-keyword carries no prefix.
214            put(&mut node, "type", text("document"));
215            put(&mut node, "document", text(prelude));
216            put(&mut node, "vendor", text(vendor_prefix(kw).unwrap_or("")));
217        }
218        _ => {
219            put(&mut node, "type", text(kw));
220            put(&mut node, kw, text(prelude));
221        }
222    }
223    node
224}
225
226/// The head of a block at-rule whose body is declarations (`@font-face`,
227/// `@page`, and any other with a declarations body). A `@page` prelude is a
228/// selector group: `@page toc, index:blank`.
229pub(crate) fn at_decls_head(kw: &str, prelude: &str) -> Value {
230    let mut node = object();
231    put(&mut node, "type", text(kw));
232    if "page" == kw {
233        let selectors = split_selectors(prelude)
234            .into_iter()
235            .map(Value::String)
236            .collect();
237        put(&mut node, "selectors", Value::array(selectors));
238    }
239    node
240}
241
242/// The head of a `@keyframes` block, possibly vendor-prefixed.
243pub(crate) fn keyframes_head(kw: &str, name: &str) -> Value {
244    let mut node = object();
245    put(&mut node, "type", text("keyframes"));
246    put(&mut node, "name", text(name));
247    if let Some(vendor) = vendor_prefix(kw) {
248        put(&mut node, "vendor", text(vendor));
249    }
250    node
251}
252
253/// A statement at-rule (`@import`, `@charset`, `@namespace`, …), whole: the
254/// at-keyword is the node type and the field carrying its params.
255///
256/// `@custom-media --name <query>` splits its params into a name and a media
257/// query, as `/^(--\S+)\s*([\s\S]*)$/` does in the canonical port: `--` and
258/// at least one non-space character, then the rest, trimmed.
259pub(crate) fn at_stmt_head(kw: &str, params: &str) -> Value {
260    let mut node = object();
261    if "custom-media" == kw && params.starts_with("--") {
262        let name_end = params.find(es_is_whitespace).unwrap_or(params.len());
263        if 2 < name_end {
264            put(&mut node, "type", text("custom-media"));
265            put(&mut node, "name", text(&params[..name_end]));
266            put(&mut node, "media", text(es_trim(&params[name_end..])));
267            return node;
268        }
269    }
270    put(&mut node, "type", text(kw));
271    put(&mut node, kw, text(params));
272    node
273}
274
275/// `^(-[a-z]+-)`: a vendor prefix on an at-keyword, e.g. `-webkit-keyframes`.
276pub fn vendor_prefix(kw: &str) -> Option<&str> {
277    let b = kw.as_bytes();
278    if b.first() != Some(&b'-') {
279        return None;
280    }
281    let mut i = 1;
282    while i < b.len() && b[i].is_ascii_lowercase() {
283        i += 1;
284    }
285    if 1 < i && b.get(i) == Some(&b'-') {
286        Some(&kw[..i + 1])
287    } else {
288        None
289    }
290}
291
292/// Whether `c` is whitespace as ECMAScript defines it, which is NOT what
293/// [`char::is_whitespace`] answers.
294///
295/// The canonical port trims with JavaScript's `String.prototype.trim`, whose
296/// set is ECMAScript WhiteSpace plus LineTerminator. Rust's set is the Unicode
297/// `White_Space` property. They differ by exactly two code points, and both
298/// differences change the AST:
299///
300/// - **U+FEFF** (zero-width no-break space, the byte-order mark) is
301///   ECMAScript whitespace and is not Unicode `White_Space`. A stylesheet that
302///   opens with a byte-order mark yields the selector `a` in the canonical
303///   port and would yield `\u{feff}a` under [`str::trim`].
304/// - **U+0085** (next line) is Unicode `White_Space` and is not ECMAScript
305///   whitespace. `@media x \u{85}{…}` keeps that character in its prelude in
306///   the canonical port and would lose it under [`str::trim`].
307///
308/// Everything else in either set is in both: the ASCII controls, U+0020, the
309/// `Zs` category, and the two line separators U+2028 and U+2029.
310pub fn es_is_whitespace(c: char) -> bool {
311    ('\u{feff}' == c) || (c.is_whitespace() && '\u{85}' != c)
312}
313
314/// Trim as JavaScript's `String.prototype.trim` does. See
315/// [`es_is_whitespace`] for why this is not [`str::trim`].
316pub fn es_trim(s: &str) -> &str {
317    s.trim_matches(es_is_whitespace)
318}
319
320/// The column width of `s`: its length in UTF-16 code units, which is what a
321/// JavaScript string index counts.
322pub(crate) fn col_width(s: &str) -> usize {
323    s.chars().map(char::len_utf16).sum()
324}
325
326/// Slice `src` by byte offsets, clamping to something sliceable.
327///
328/// This is the Rust half of the Go port's `clampLen`, and it exists for the
329/// same reason: the scanners deliberately overshoot. An `i += 2` over a `\`
330/// escape, an unterminated string or an unterminated comment can step past the
331/// end of the source, and JavaScript CLAMPS an out-of-range slice bound where
332/// Go panics — so the Go port clamps to reproduce the JavaScript result rather
333/// than invent a new one.
334///
335/// Rust panics on one bound more than Go does: an index inside a multi-byte
336/// character. Every index these scanners RETURN is at an ASCII character or at
337/// the end of the source, because each one stops at `{`, `}`, `;`, `,`, `*/`,
338/// a quote or end of input, so the fallback below is unreachable in practice —
339/// it is here so that a future scanner change degrades to a shorter span
340/// instead of a panic in a caller's parse.
341fn safe_slice(src: &str, start: usize, end: usize) -> &str {
342    let mut start = start.min(src.len());
343    let mut end = end.clamp(start, src.len());
344    while 0 < start && !src.is_char_boundary(start) {
345        start -= 1;
346    }
347    while start < end && !src.is_char_boundary(end) {
348        end -= 1;
349    }
350    &src[start..end]
351}
352
353/// The index of the `*` that opens the `*/` closing the comment at `i`, or
354/// `None` if there is no closing `*/`.
355fn find_comment_end(src: &[u8], i: usize) -> Option<usize> {
356    let mut j = i + 2;
357    while j + 1 < src.len() {
358        if src[j] == b'*' && src[j + 1] == b'/' {
359            return Some(j);
360        }
361        j += 1;
362    }
363    None
364}
365
366/// The index after a closed `/* ... */`, or [`UNTERMINATED`].
367///
368/// It does NOT run to end of source. An unclosed `/* ... */` is an error, not
369/// a comment to the end of the file — which is what both the engine's builtin
370/// comment matcher and reworkcss do, and what `test/spec/reworkcss.tsv` pins.
371fn skip_comment(src: &[u8], i: usize) -> usize {
372    match find_comment_end(src, i) {
373        // `e + 1` is the `/`, so `e + 2` is always within the source.
374        Some(e) => e + 2,
375        None => UNTERMINATED,
376    }
377}
378
379/// Skip a quoted string; returns the index after the closing quote.
380fn skip_string(src: &[u8], mut i: usize) -> usize {
381    let quote = src[i];
382    i += 1;
383    while i < src.len() {
384        if src[i] == b'\\' {
385            i += 2;
386            continue;
387        }
388        if src[i] == quote {
389            return i + 1;
390        }
391        i += 1;
392    }
393    // A trailing backslash steps i ONE PAST the end. That overshoot is not a
394    // defect to clamp away here: the canonical port returns it too, and its
395    // column arithmetic counts it. See [`css_token`] and [`safe_slice`].
396    i
397}
398
399/// Scan a key prelude: where it ends, and whether it is a selector (a
400/// top-level `{` comes first) or a declaration (a top-level `;`/`}` or end of
401/// input comes first). Strings, `()`, `[]` and comments are skipped.
402fn scan_to_brace_or_end(src: &[u8], i: usize) -> (Kind, usize) {
403    let mut scan = Scan { i, depth: 0 };
404    loop {
405        if let Some(found) = scan.step(src, 0) {
406            return found;
407        }
408    }
409}
410
411/// [`scan_to_brace_or_end`] one step at a time. The position and the
412/// bracket depth are its whole state, so two scans in the same state go on
413/// to the same answer: what [`BraceScans`] relies on.
414#[derive(Clone, Copy, Debug)]
415struct Scan {
416    i: usize,
417    depth: usize,
418}
419
420impl Scan {
421    /// Take one step: past a string, a comment, an escape or one byte.
422    /// `Some` is the scan's answer. `src` is the text the lexer holds, and
423    /// `base` the source's byte offset of its first byte: the scan's
424    /// positions are the source's, as the kept ones in [`BraceScans`] are.
425    fn step(&mut self, src: &[u8], base: usize) -> Option<(Kind, usize)> {
426        let at = self.i - base;
427        let Some(&c) = src.get(at) else {
428            // i may be one past the end; see skip_string.
429            return Some((Kind::Decl, self.i));
430        };
431        if c == b'"' || c == b'\'' {
432            self.i = base + skip_string(src, at);
433            return None;
434        }
435        if c == b'/' && src.get(at + 1) == Some(&b'*') {
436            let n = skip_comment(src, at);
437            if UNTERMINATED == n {
438                return Some((Kind::Decl, UNTERMINATED));
439            }
440            self.i = base + n;
441            return None;
442        }
443        // A CSS escape (`\(`, `\'`, `\3A `, …) hides the next character from
444        // the structural scan.
445        if c == b'\\' {
446            self.i += 2;
447            return None;
448        }
449        if c == b'(' || c == b'[' {
450            self.depth += 1;
451        } else if c == b')' || c == b']' {
452            self.depth = self.depth.saturating_sub(1);
453        } else if 0 == self.depth {
454            if c == b'{' {
455                return Some((Kind::Selector, self.i));
456            }
457            if c == b';' || c == b'}' {
458                return Some((Kind::Decl, self.i));
459            }
460        }
461        self.i += 1;
462        None
463    }
464}
465
466/// [`scan_to_brace_or_end`] for every token of a parse, in linear time.
467///
468/// Each selector of a group, and each value of a keyframe-selector group,
469/// is classified by scanning to the group's `{`, so a group of `k` items
470/// scanned from each item's start costs `k` times the group's length: a
471/// 200 KB list of selectors took 9 s. The scan from an item's start is the
472/// scan from the group's first item, resumed there, whenever that scan
473/// passes the item's start at bracket depth 0 (the state [`Scan`] carries,
474/// outside any string, comment or escape, which it steps over whole). So
475/// the answer is kept with a cursor that follows the tokens forward, and a
476/// later start the cursor reaches in that state takes the kept answer. Any
477/// other start scans afresh and becomes the one kept. The cursor only moves
478/// forward, so a parse's scanning is linear in its length.
479///
480/// The kept positions are the source's, and the text is read where the
481/// lexer holds it: in a parse from a reader, the window, which may have
482/// dropped the text behind the cursor. A kept cursor whose text the window
483/// no longer holds is passed over, and the start scans afresh, which costs
484/// time and never changes an answer. An answer the scan found only by
485/// reading to the end of the window is not kept while the source may have
486/// more: the matcher fills the window and asks again.
487#[derive(Clone, Copy, Debug)]
488struct BraceScans {
489    from: usize,
490    found: (Kind, usize),
491    cursor: Scan,
492}
493
494/// The `ctx.u` key [`BraceScans`] is kept under, for the parse.
495pub(crate) const BRACE_SCAN: &str = "tabnas-css/brace-scan";
496
497impl BraceScans {
498    /// The answer for a token starting at byte `at` of `src`, the text the
499    /// lexer holds, whose first byte is the source's byte `base`: its kind,
500    /// and where it ended in `src` (or [`UNTERMINATED`]). The state is kept
501    /// under [`BRACE_SCAN`] with one map lookup: the matcher asks on every
502    /// `#TX`, and a flat stylesheet is mostly misses. Unless `exhausted`,
503    /// the end of `src` may not be the source's, and an answer found there
504    /// is not kept.
505    fn scan(
506        ctx: &mut Context,
507        src: &[u8],
508        base: usize,
509        at: usize,
510        exhausted: bool,
511    ) -> (Kind, usize) {
512        let s_i = base + at;
513        let held = |found: (Kind, usize)| match found.1 {
514            UNTERMINATED => found,
515            end => (found.0, end - base),
516        };
517        let Some(Value::Array(slot)) = ctx.u.get_mut(BRACE_SCAN) else {
518            let found = BraceScans::fresh_scan(src, base, at);
519            if exhausted || !BraceScans::at_end(src, base, found) {
520                let kept = BraceScans::fresh(s_i, found);
521                ctx.u
522                    .insert(BRACE_SCAN.to_string(), Value::array(kept.fields().to_vec()));
523            }
524            return held(found);
525        };
526        let fields = Arc::make_mut(slot);
527        if let Some(mut kept) = BraceScans::decode(fields) {
528            if kept.from <= s_i
529                && s_i < kept.found.1
530                && base <= kept.cursor.i
531                && kept.cursor.i <= s_i
532            {
533                while kept.cursor.i < s_i && kept.cursor.step(src, base).is_none() {}
534                let reached = kept.cursor.i == s_i && 0 == kept.cursor.depth;
535                kept.encode(fields);
536                if reached {
537                    return held(kept.found);
538                }
539            }
540        }
541        // An unclosed comment's answer is kept too: a scan from a later start
542        // the cursor reaches meets the same `/*`, and the engine's recovery
543        // asks again at the very start it failed at, up to its skip budget.
544        let found = BraceScans::fresh_scan(src, base, at);
545        if exhausted || !BraceScans::at_end(src, base, found) {
546            BraceScans::fresh(s_i, found).encode(fields);
547        }
548        held(found)
549    }
550
551    /// [`scan_to_brace_or_end`] from byte `at` of `src`, its answer at the
552    /// source's position.
553    fn fresh_scan(src: &[u8], base: usize, at: usize) -> (Kind, usize) {
554        match scan_to_brace_or_end(src, at) {
555            (kind, UNTERMINATED) => (kind, UNTERMINATED),
556            (kind, end) => (kind, base + end),
557        }
558    }
559
560    /// Whether `found`, at the source's position, was read to the end of
561    /// `src`: an unclosed comment, or the end of the text, which a `\`
562    /// escape can overshoot by one.
563    fn at_end(src: &[u8], base: usize, found: (Kind, usize)) -> bool {
564        UNTERMINATED == found.1 || base + src.len() <= found.1
565    }
566
567    fn fresh(from: usize, found: (Kind, usize)) -> BraceScans {
568        BraceScans {
569            from,
570            found,
571            cursor: Scan { i: from, depth: 0 },
572        }
573    }
574
575    fn decode(fields: &[Value]) -> Option<BraceScans> {
576        let at = |n: usize| match fields.get(n) {
577            Some(Value::Number(v)) => Some(*v as usize),
578            _ => None,
579        };
580        let kind = if 0 == at(1)? {
581            Kind::Selector
582        } else {
583            Kind::Decl
584        };
585        Some(BraceScans {
586            from: at(0)?,
587            found: (kind, at(2)?),
588            cursor: Scan {
589                i: at(3)?,
590                depth: at(4)?,
591            },
592        })
593    }
594
595    fn numbers(&self) -> [f64; 5] {
596        let kind = match self.found.0 {
597            Kind::Selector => 0,
598            Kind::Decl => 1,
599        };
600        [
601            self.from,
602            kind,
603            self.found.1,
604            self.cursor.i,
605            self.cursor.depth,
606        ]
607        .map(|n| n as f64)
608    }
609
610    fn fields(&self) -> [Value; 5] {
611        self.numbers().map(Value::Number)
612    }
613
614    /// Write this state over `fields`, number by number when the shape fits.
615    fn encode(&self, fields: &mut Vec<Value>) {
616        let numbers = self.numbers();
617        if numbers.len() == fields.len() {
618            for (field, n) in fields.iter_mut().zip(numbers) {
619                match field {
620                    Value::Number(slot) => *slot = n,
621                    other => *other = Value::Number(n),
622                }
623            }
624        } else {
625            *fields = self.fields().to_vec();
626        }
627    }
628}
629
630/// Scan a single selector: to the next top-level `,` (a group separator) or
631/// `{`, or end of input. Strings, `()`, `[]` and comments are skipped.
632fn scan_selector_end(src: &[u8], mut i: usize) -> usize {
633    let mut depth = 0usize;
634    while i < src.len() {
635        let c = src[i];
636        if c == b'"' || c == b'\'' {
637            i = skip_string(src, i);
638            continue;
639        }
640        if c == b'/' && src.get(i + 1) == Some(&b'*') {
641            let n = skip_comment(src, i);
642            if UNTERMINATED == n {
643                return UNTERMINATED;
644            }
645            i = n;
646            continue;
647        }
648        if c == b'\\' {
649            i += 2;
650            continue;
651        }
652        if c == b'(' || c == b'[' {
653            depth += 1;
654        } else if c == b')' || c == b']' {
655            depth = depth.saturating_sub(1);
656        } else if 0 == depth && (c == b',' || c == b'{') {
657            return i;
658        }
659        i += 1;
660    }
661    // i may be one past the end; see skip_string.
662    i
663}
664
665/// Scan a declaration value or at-rule params: to the next top-level `;` or
666/// `}`, or end of input. Strings, `()`, `[]` and comments are skipped.
667fn scan_value_end(src: &[u8], mut i: usize) -> usize {
668    let mut depth = 0usize;
669    while i < src.len() {
670        let c = src[i];
671        if c == b'"' || c == b'\'' {
672            i = skip_string(src, i);
673            continue;
674        }
675        if c == b'/' && src.get(i + 1) == Some(&b'*') {
676            let n = skip_comment(src, i);
677            if UNTERMINATED == n {
678                return UNTERMINATED;
679            }
680            i = n;
681            continue;
682        }
683        if c == b'\\' {
684            i += 2;
685            continue;
686        }
687        if c == b'(' || c == b'[' {
688            depth += 1;
689        } else if c == b')' || c == b']' {
690            depth = depth.saturating_sub(1);
691        } else if 0 == depth && (c == b';' || c == b'}') {
692            return i;
693        }
694        i += 1;
695    }
696    // i may be one past the end; see skip_string.
697    i
698}
699
700/// Remove `/* ... */` comments from a selector or value run, leaving quoted
701/// strings (which may contain `/*`) untouched.
702pub fn strip_comments(s: &str) -> String {
703    if !s.contains("/*") {
704        return s.to_string();
705    }
706    let src = s.as_bytes();
707    let mut out: Vec<u8> = Vec::with_capacity(src.len());
708    let mut i = 0;
709    while i < src.len() {
710        let c = src[i];
711        if c == b'"' || c == b'\'' {
712            let j = skip_string(src, i);
713            out.extend_from_slice(&src[i..j.min(src.len())]);
714            i = j;
715            continue;
716        }
717        if c == b'/' && src.get(i + 1) == Some(&b'*') {
718            let n = skip_comment(src, i);
719            if UNTERMINATED == n {
720                // Unreachable once the callers reject an unclosed comment,
721                // but a sentinel assigned to i would spin. The rest of the
722                // span is comment text; drop it.
723                break;
724            }
725            i = n;
726            continue;
727        }
728        out.push(c);
729        i += 1;
730    }
731    String::from_utf8(out).unwrap_or_else(|e| String::from_utf8_lossy(e.as_bytes()).into_owned())
732}
733
734/// A CSS property name character.
735///
736/// Beyond the identifier characters this admits the legacy IE hack prefixes
737/// `*`, `#`, `/` (as in `//prop`) and a literal backslash, matching the
738/// reworkcss property pattern `\*?[-#\/\*\\\w]+(\[[0-9a-z_-]+\])?` — real
739/// stylesheets rely on them.
740fn is_prop_char(c: u8) -> bool {
741    c.is_ascii_alphanumeric()
742        || c == b'-'
743        || c == b'_'
744        || c == b'#'
745        || c == b'*'
746        || c == b'/'
747        || c == b'\\'
748}
749
750/// A `[0-9a-z_-]` character, the only content allowed in a property name's
751/// bracket suffix (e.g. `opacity[sqrt]`).
752fn is_prop_suffix_char(c: u8) -> bool {
753    c.is_ascii_digit() || c.is_ascii_lowercase() || c == b'-' || c == b'_'
754}
755
756/// Scan a property name: the run of property characters plus an optional
757/// `[...]` suffix. Returns the index after the name (`== i` if there is none).
758fn scan_prop_end(src: &[u8], i: usize) -> usize {
759    let mut e_i = i;
760    while e_i < src.len() && is_prop_char(src[e_i]) {
761        e_i += 1;
762    }
763    if e_i == i {
764        return i;
765    }
766    if src.get(e_i) == Some(&b'[') {
767        let mut b_i = e_i + 1;
768        while b_i < src.len() && is_prop_suffix_char(src[b_i]) {
769            b_i += 1;
770        }
771        if e_i + 1 < b_i && src.get(b_i) == Some(&b']') {
772            return b_i + 1;
773        }
774    }
775    e_i
776}
777
778/// An at-keyword character: letters, digits and `-`.
779fn is_at_char(c: u8) -> bool {
780    c.is_ascii_alphanumeric() || c == b'-'
781}
782
783/// Split a selector-group prelude on its top-level commas (commas inside
784/// strings, `()` or `[]` are part of a single selector), stripping comments
785/// and trimming each selector. An empty prelude yields an empty list.
786pub fn split_selectors(prelude: &str) -> Vec<String> {
787    let src = prelude.as_bytes();
788    let mut out = Vec::new();
789    let mut i = 0;
790    while i <= src.len() {
791        let mut end = scan_selector_end(src, i);
792        if UNTERMINATED == end {
793            // Unreachable in practice: the prelude was scanned (and an
794            // unclosed comment rejected) before it got here. Treated as "the
795            // rest" so a sentinel can never index or spin.
796            end = src.len();
797        }
798        let one = es_trim(&strip_comments(safe_slice(prelude, i, end))).to_string();
799        if !one.is_empty() {
800            out.push(one);
801        }
802        if end >= src.len() {
803            break;
804        }
805        i = end + 1;
806    }
807    out
808}
809
810// ---------------------------------------------------------------------------
811// The `cssToken` matcher, as the engine runs it.
812
813/// The token number a matcher gives a token the engine is to resolve by
814/// NAME, as it lexes: the six this grammar names itself (`#CC`, `#GC` and
815/// the four at-rule tokens).
816///
817/// The engine numbers a name when it is first registered, and
818/// [`Tabnas::merge`](tabnas::Tabnas::merge) renumbers every custom name in
819/// the merged instance without running the plugin again, so a number
820/// captured at install names another token there. The canonical port's
821/// `lex.token('#CC', …)` resolves the name on every call for the same
822/// reason. `#TX` and `#VL` are the engine's own, with fixed numbers.
823const BY_NAME: EngineTin = -1;
824
825/// Where a scan ran past the end of the source, and by how much: the
826/// `ctx.u` key [`css_token`] records it under and [`lex_subscriber`] reads.
827pub(crate) const OVERSHOOT: &str = "tabnas-css/overshoot";
828
829/// The most of the source an `unterminated_comment` diagnostic shows. The
830/// span runs from the scan's start to the end of the source, which for a
831/// large stylesheet would be most of it; the message is informative, not the
832/// contract, so it is cut here.
833const BAD_SPAN: usize = 32;
834
835/// What [`plan_token`] decided for the characters at the cursor.
836enum Plan {
837    /// Not this matcher's: the engine's builtin matchers take it.
838    Decline,
839    /// An unclosed `/* ... */` behind the cursor: `unterminated_comment`.
840    Bad,
841    /// A token, and the byte offset just past it, which may be ONE past the
842    /// end of the source (see [`css_token`]).
843    Emit {
844        name: &'static str,
845        tin: EngineTin,
846        val: Value,
847        raw: String,
848        end: usize,
849    },
850    /// A scan read to the end of the text the lexer holds while the source
851    /// may have more: [`css_token`] fills the window and plans again.
852    Edge,
853}
854
855/// Decide the token at byte `s_i` of `src`, read under the rule `rule`.
856///
857/// `src` is the text the lexer holds: the whole source, or in a parse from
858/// a reader the window. Unless `exhausted`, a scan that reads to the end of
859/// `src` answers [`Plan::Edge`]; with `exhausted`, the end of `src` is the
860/// end of the source.
861///
862/// `buildCssTokenMatcher` in `ts/src/css.ts`, step for step: whitespace is
863/// declined (the engine's space and line matchers take it), a comment is a
864/// node only at a list position, a value runs to the next top-level `;` or
865/// `}` under `declval`, a top-level `,` separates a selector group, `@`
866/// starts an at-rule, `{` `}` `;` are the grammar's, and anything else is a
867/// selector or a property name by `{`-before-`;` lookahead.
868fn plan_token(
869    src: &str,
870    s_i: usize,
871    rule: &str,
872    lower: bool,
873    exhausted: bool,
874    brace: &mut dyn FnMut(usize) -> (Kind, usize),
875) -> Plan {
876    let bytes = src.as_bytes();
877    // Whether a scan that stopped at `i` read to the end of `src`: it ran
878    // off the end, which a `\` escape can overshoot by one, or met an
879    // unclosed comment, whose `*/` may lie past the end.
880    let edge = |i: usize| !exhausted && (UNTERMINATED == i || bytes.len() <= i);
881    let Some(&c) = bytes.get(s_i) else {
882        return if exhausted { Plan::Decline } else { Plan::Edge };
883    };
884    if c == b' ' || c == b'\t' || c == b'\r' || c == b'\n' {
885        return Plan::Decline;
886    }
887    let emit = |name, tin, val: String, raw: &str, end| Plan::Emit {
888        name,
889        tin,
890        val: Value::String(val),
891        raw: raw.to_string(),
892        end,
893    };
894
895    // Comments: a node at a list position, otherwise declined, and then
896    // skipped by the engine's builtin comment matcher. Whether a `/` opens
897    // one depends on the byte after it.
898    if c == b'/' && edge(s_i + 1) {
899        return Plan::Edge;
900    }
901    if c == b'/' && bytes.get(s_i + 1) == Some(&b'*') {
902        if !is_comment_node_rule(rule) {
903            return Plan::Decline;
904        }
905        return match find_comment_end(bytes, s_i) {
906            None if !exhausted => Plan::Edge,
907            None => Plan::Bad,
908            Some(e) => emit(
909                "#CC",
910                BY_NAME,
911                safe_slice(src, s_i + 2, e).to_string(),
912                safe_slice(src, s_i, e + 2),
913                e + 2,
914            ),
915        };
916    }
917
918    // Value position: a declaration value up to the next top-level `;`/`}`,
919    // comments stripped and surrounding space trimmed.
920    if "declval" == rule {
921        if c == b'{' || c == b'}' || c == b';' || c == b':' {
922            return Plan::Decline;
923        }
924        let end = scan_value_end(bytes, s_i);
925        if edge(end) {
926            return Plan::Edge;
927        }
928        if UNTERMINATED == end {
929            return Plan::Bad;
930        }
931        let raw = safe_slice(src, s_i, end);
932        let val = es_trim(&strip_comments(raw)).to_string();
933        return emit("#VL", TIN_VL, val, raw, end);
934    }
935
936    // A top-level selector-group comma.
937    if c == b',' {
938        return emit("#GC", BY_NAME, ",".to_string(), ",", s_i + 1);
939    }
940
941    // An at-rule: block or statement by `{`-before-`;` lookahead, and a
942    // block's body by keyword. The token's value is the node's head: its
943    // type and the fields read from the keyword and the prelude (block) or
944    // params (statement), so that a reader of the rule passes (the
945    // transducer part, `alchemy/transduce.alc`) sees the node as the walk
946    // builds it, a pass carrying a token's value and never its `use` bag.
947    if c == b'@' {
948        let mut k_end = s_i + 1;
949        while k_end < bytes.len() && is_at_char(bytes[k_end]) {
950            k_end += 1;
951        }
952        let kw = safe_slice(src, s_i + 1, k_end).to_string();
953        // The lookahead reads from the `@` on, through the keyword, so where
954        // it stops inside `src` the keyword stopped inside it too, and so
955        // does the params scan below, which stops where it stops.
956        let (kind, index) = brace(s_i);
957        if edge(index) {
958            return Plan::Edge;
959        }
960        if UNTERMINATED == index {
961            return Plan::Bad;
962        }
963        if Kind::Selector == kind {
964            // At-rule preludes KEEP their comments (they are only trimmed),
965            // matching reworkcss.
966            let prelude = es_trim(safe_slice(src, k_end, index)).to_string();
967            let (name, tin) = if is_keyframes_kw(&kw) {
968                ("#ATK", BY_NAME)
969            } else if DECLS_KW.contains(&kw.as_str()) {
970                ("#ATD", BY_NAME)
971            } else {
972                ("#ATR", BY_NAME)
973            };
974            let head = match name {
975                "#ATK" => keyframes_head(&kw, &prelude),
976                "#ATD" => at_decls_head(&kw, &prelude),
977                _ => at_rules_head(&kw, &prelude),
978            };
979            let raw = safe_slice(src, s_i, index);
980            return Plan::Emit {
981                name,
982                tin,
983                val: head,
984                raw: raw.to_string(),
985                end: index,
986            };
987        }
988        // A statement at-rule's params run to the next top-level `;`/`}`. A
989        // `;` is consumed (it ends the statement); a `}` or the end of the
990        // source is left for the enclosing rule.
991        let p_end = scan_value_end(bytes, k_end);
992        if UNTERMINATED == p_end {
993            return Plan::Bad;
994        }
995        let params = es_trim(safe_slice(src, k_end, p_end)).to_string();
996        let end = if bytes.get(p_end) == Some(&b';') {
997            p_end + 1
998        } else {
999            p_end
1000        };
1001        let raw = safe_slice(src, s_i, end);
1002        return Plan::Emit {
1003            name: "#ATS",
1004            tin: BY_NAME,
1005            val: at_stmt_head(&kw, &params),
1006            raw: raw.to_string(),
1007            end,
1008        };
1009    }
1010
1011    // Other fixed punctuation belongs to the grammar. `:` does not: a run
1012    // that starts with one is scanned below, as in the canonical port.
1013    if c == b'{' || c == b'}' || c == b';' {
1014        return Plan::Decline;
1015    }
1016
1017    // A selector or a property name, by `{`-before-`;` lookahead.
1018    let (kind, index) = brace(s_i);
1019    if edge(index) {
1020        return Plan::Edge;
1021    }
1022    if UNTERMINATED == index {
1023        return Plan::Bad;
1024    }
1025    // Both scans below stop at or before what ended the lookahead, which
1026    // skipped what they skip and stopped at a `{`, `;` or `}`, none of which
1027    // they read past, so they read only text the lookahead read.
1028    if Kind::Selector == kind {
1029        // One selector of a (possible) group: up to the next top-level `,`
1030        // or `{`, comments stripped and surrounding space trimmed.
1031        let end = scan_selector_end(bytes, s_i);
1032        if UNTERMINATED == end {
1033            return Plan::Bad;
1034        }
1035        let raw = safe_slice(src, s_i, end);
1036        let val = es_trim(&strip_comments(raw)).to_string();
1037        return emit("#TX", TIN_TX, val, raw, end);
1038    }
1039
1040    // A property name: the run of property characters up to `:`,
1041    // whitespace, `;` or `}`. `/` and `*` are property characters (the
1042    // `*prop` and `//prop` IE hacks), so a trailing `/*...*/` hack comment
1043    // lands inside the scanned name; comments are stripped out of it, as
1044    // reworkcss does.
1045    let e_i = scan_prop_end(bytes, s_i);
1046    if e_i == s_i {
1047        return Plan::Decline;
1048    }
1049    let raw = safe_slice(src, s_i, e_i);
1050    let mut prop = strip_comments(raw);
1051    if prop.is_empty() {
1052        return Plan::Decline;
1053    }
1054    if lower {
1055        prop = prop.to_lowercase();
1056    }
1057    emit("#TX", TIN_TX, prop, raw, e_i)
1058}
1059
1060/// The `cssToken` matcher, registered as `@css-token` and named by the
1061/// grammar's options at order 1e5, ahead of every builtin matcher.
1062///
1063/// The token is planned over the text the lexer holds: the whole source,
1064/// or in a parse from a reader (`lex.window`, admin ADR-30) the window,
1065/// which keeps the text from the oldest token the parser holds to what it
1066/// has read, and which [`Lexer::window`] gives from the cursor on. Each
1067/// scan reads forward from the cursor to what ends it, so one that stops
1068/// inside the window gives what the whole text gives. One that reaches the
1069/// window's end may have stopped short: the plan is then [`Plan::Edge`],
1070/// and the matcher asks the window for twice what it holds past the cursor
1071/// ([`Lexer::fill`]) and plans again over the grown text. Where the window
1072/// does not grow, the source ends there, and the plan is made with the end
1073/// of the window taken for the end of the source. A long token so costs its
1074/// length to read, not its square. In a whole text `fill` only answers, and
1075/// the first plan stands.
1076///
1077/// Two things here are deliberate and easy to "fix" wrongly:
1078///
1079/// - The end of a token may be ONE PAST the source. A `\` at the last
1080///   character is an escape whose escaped character is not there, and the
1081///   scanners' `i += 2` steps off the end. The canonical port advances its
1082///   column by that unclamped width and takes the CLAMPED span for the
1083///   token's text, so `@host\` yields a node ending at column 7 inside a
1084///   stylesheet ending at column 8, one past a source six characters long.
1085///   The engine will not move its cursor past the end, so the overshoot is
1086///   recorded under [`OVERSHOOT`] and [`lex_subscriber`] adds it to the
1087///   end-of-source token, the only token that can follow it.
1088/// - An unclosed comment is reported WITHOUT advancing, so the diagnostic
1089///   points at the start of the scan, as the canonical port's does.
1090///
1091/// A matcher must never advance and then decline: the engine captured its
1092/// dispatch state before the custom matchers ran.
1093pub(crate) fn css_token(
1094    lexer: &mut Lexer<'_>,
1095    rule: &mut Rule,
1096    ctx: &mut Context,
1097    lower: bool,
1098) -> Option<Token> {
1099    let point = lexer.point();
1100    let mut exhausted = !lexer.windowed();
1101    let (plan, chars, overshoot) = loop {
1102        // `held` is the text the lexer holds, and the cursor is byte
1103        // `s_i` of it, the source's byte `point.site.si`.
1104        let ahead = lexer.window().len();
1105        let held = lexer.source();
1106        let s_i = held.len() - ahead;
1107        let base = point.site.si - s_i;
1108        let bytes = held.as_bytes();
1109        let mut brace = |at: usize| BraceScans::scan(ctx, bytes, base, at, exhausted);
1110        let plan = plan_token(held, s_i, rule.name.as_str(), lower, exhausted, &mut brace);
1111        let (chars, overshoot) = match &plan {
1112            Plan::Decline => return None,
1113            Plan::Edge => {
1114                let read = held[s_i..].chars().count();
1115                lexer.fill(read.saturating_mul(2).max(read + 1));
1116                if lexer.window().len() <= ahead {
1117                    exhausted = true;
1118                }
1119                continue;
1120            }
1121            Plan::Bad => (
1122                safe_slice(held, s_i, held.len())
1123                    .chars()
1124                    .take(BAD_SPAN)
1125                    .count(),
1126                0,
1127            ),
1128            Plan::Emit { end, .. } => (
1129                safe_slice(held, s_i, (*end).min(held.len()))
1130                    .chars()
1131                    .count(),
1132                end.saturating_sub(held.len()),
1133            ),
1134        };
1135        break (plan, chars, overshoot);
1136    };
1137    match plan {
1138        Plan::Decline | Plan::Edge => None,
1139        Plan::Bad => Some(lexer.bad_span(
1140            "unterminated_comment",
1141            point.site.pos,
1142            point.site.pos + chars,
1143        )),
1144        Plan::Emit {
1145            name,
1146            tin,
1147            val,
1148            raw,
1149            ..
1150        } => {
1151            lexer.advance_chars(chars);
1152            if 0 < overshoot {
1153                ctx.u
1154                    .insert(OVERSHOOT.to_string(), Value::Number(overshoot as f64));
1155            }
1156            Some(lexer.token(name, tin, val, raw, point))
1157        }
1158    }
1159}
1160
1161/// The plugin's lex subscriber: it sees every token the engine fetches.
1162///
1163/// - The end-of-source token takes the column overshoot [`css_token`]
1164///   recorded, so the stylesheet's end and an error at the end of the source
1165///   land where the canonical port puts them.
1166/// - It once dropped the unconsumed lookahead when a bad token arrived
1167///   behind a good one, so that an unclosed comment hidden from the
1168///   property scan was the error reported rather than `unexpected` at the
1169///   property. tabnas/parser#274 made the engine raise a fetched bad token
1170///   at once, as the canonical does, and record and skip it under
1171///   recovery, so that work is the engine's now. `test/spec/comments.tsv`
1172///   and `tests/plugin.rs` pin the answers in every mode.
1173/// - In a parse from a reader, which has no source to list them from, it
1174///   lists the astral characters of each token's text for the positions
1175///   (`plugin::note_astral`).
1176pub(crate) fn lex_subscriber(token: &mut Token, _rule: &mut Rule, ctx: &mut Context) {
1177    if TIN_ZZ == token.tin {
1178        if let Some(Value::Number(n)) = ctx.u.get(OVERSHOOT) {
1179            token.site.ci += *n as usize;
1180        }
1181    }
1182    // A parse from a reader a window at a time has no source in its context.
1183    if ctx.source.is_empty() {
1184        crate::plugin::note_astral(token, ctx);
1185    }
1186}
1187
1188/// The number of astral characters (those two UTF-16 code units wide) in
1189/// scalars `[from, to)` of `src`.
1190pub(crate) fn astral_between(src: &str, from: usize, to: usize) -> usize {
1191    src.chars()
1192        .skip(from)
1193        .take(to.saturating_sub(from))
1194        .filter(|c| 2 == c.len_utf16())
1195        .count()
1196}
1197
1198/// An engine error as this crate's, with the column in UTF-16 code units.
1199///
1200/// The engine counts a column in Unicode scalars since the last reset (a
1201/// `\n` in any token, or the end of a run of line characters); the canonical
1202/// port counts UTF-16 code units since the same points. The two differ by
1203/// the astral characters between the reset and the error, which is what is
1204/// added here. The message is the engine's, and is not the contract.
1205impl From<TabnasError> for Error {
1206    fn from(error: TabnasError) -> Error {
1207        let from = error.pos.saturating_sub(error.col.saturating_sub(1));
1208        let wide = astral_between(&error.full_source, from, error.pos);
1209        Error {
1210            code: error.code,
1211            message: error.detail,
1212            line: error.row,
1213            column: error.col + wide,
1214        }
1215    }
1216}