tabnas_css/lex.rs
1/* Copyright (c) 2025 Richard Rodger, MIT License */
2
3//! The lexer: the single `cssToken` matcher the plugin installs on the
4//! engine, and the scanners it reads spans with.
5//!
6//! CSS is context-sensitive — the same characters can begin a selector, a
7//! property or a value — so the matcher owns the hard tokenisation and the
8//! grammar only assembles typed nodes from what it emits. Which token comes
9//! out depends on the rule that is active when the character is read: the
10//! engine reads a token only when the alternate being tried needs one, under
11//! the rule trying it, and passes that rule to the matcher.
12//!
13//! See `ts/src/css.ts` for the canonical commentary; the three ports are kept
14//! in step, and this module follows `buildCssTokenMatcher` there line for
15//! line.
16
17use std::fmt;
18use std::sync::Arc;
19
20use tabnas::{
21 Context, Lexer, Rule, TabnasError, Tin as EngineTin, Token, Value, TIN_TX, TIN_VL, TIN_ZZ,
22};
23
24use crate::plugin::{object, put, text};
25
26/// A token kind.
27///
28/// `Ob`/`Cb`/`Cl`/`Ca` are the fixed punctuation the canonical port borrows
29/// from the engine (`{` `}` `:` and `;`, the last remapped onto jsonic's
30/// member separator). The rest are this grammar's own: a custom name is
31/// required for a comment NODE because the engine's builtin comment tin is in
32/// the parser's ignore set, so emitting it would silently drop the node.
33#[derive(Clone, Copy, Debug, PartialEq, Eq)]
34pub enum Tin {
35 /// `{` — start of a block.
36 Ob,
37 /// `}` — end of a block.
38 Cb,
39 /// `:` — declaration separator.
40 Cl,
41 /// `;` — declaration terminator.
42 Ca,
43 /// A selector, keyframe value, or property name.
44 Tx,
45 /// `,` — selector-group separator.
46 Gc,
47 /// A declaration value (raw text).
48 Vl,
49 /// A comment, at a statement position.
50 Cc,
51 /// An at-rule with a rules body (`@media`, `@supports`, …).
52 Atr,
53 /// An at-rule with a declarations body (`@font-face`, `@page`, …).
54 Atd,
55 /// `@keyframes`.
56 Atk,
57 /// A statement at-rule (`@import`, `@charset`, …).
58 Ats,
59 /// End of input.
60 Zz,
61}
62
63impl Tin {
64 /// The token's grammar name, as `css-grammar.jsonic` writes it.
65 pub fn name(self) -> &'static str {
66 match self {
67 Tin::Ob => "#OB",
68 Tin::Cb => "#CB",
69 Tin::Cl => "#CL",
70 Tin::Ca => "#CA",
71 Tin::Tx => "#TX",
72 Tin::Gc => "#GC",
73 Tin::Vl => "#VL",
74 Tin::Cc => "#CC",
75 Tin::Atr => "#ATR",
76 Tin::Atd => "#ATD",
77 Tin::Atk => "#ATK",
78 Tin::Ats => "#ATS",
79 Tin::Zz => "#ZZ",
80 }
81 }
82
83 /// A human description, for diagnostics and diagram legends.
84 pub fn describe(self) -> &'static str {
85 match self {
86 Tin::Ob => "{ — start of a block",
87 Tin::Cb => "} — end of a block",
88 Tin::Cl => ": — declaration separator",
89 Tin::Ca => "; — declaration terminator",
90 Tin::Tx => "a selector, keyframe value, or property name",
91 Tin::Gc => ", — selector-group separator",
92 Tin::Vl => "a declaration value (raw text)",
93 Tin::Cc => "a comment",
94 Tin::Atr => "an at-rule with a rules body (@media, @supports, …)",
95 Tin::Atd => "an at-rule with a declarations body (@font-face, @page)",
96 Tin::Atk => "@keyframes",
97 Tin::Ats => "a statement at-rule (@import, @charset, …)",
98 Tin::Zz => "end of input",
99 }
100 }
101}
102
103/// A parse failure.
104///
105/// `code` is the contract: the shared fixtures pin `ERROR:<code>` and compare
106/// it exactly. [`Css::parse`](crate::Css::parse) raises only the two codes the
107/// canonical port inherits from the engine — `unterminated_comment` and
108/// `unexpected` — and this crate declares none of its own, matching
109/// `tabnas.plugin.json` (`errorCodes: []`). The engine's own tree form can
110/// also stop with `cancel`, at [`TREE_RULE_DEPTH`](crate::TREE_RULE_DEPTH).
111#[derive(Clone, Debug, PartialEq, Eq)]
112pub struct Error {
113 /// The error code, compared exactly by the conformance fixtures.
114 pub code: String,
115 /// A human-readable explanation.
116 pub message: String,
117 /// 1-based line where the failure was found.
118 pub line: usize,
119 /// 1-based column (UTF-16 code units) where the failure was found.
120 pub column: usize,
121}
122
123impl fmt::Display for Error {
124 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
125 write!(
126 f,
127 "[css/{}]: {} (line {}, column {})",
128 self.code, self.message, self.line, self.column
129 )
130 }
131}
132
133impl std::error::Error for Error {}
134
135/// The scanners' "there is no valid end here" result: an unclosed
136/// `/* ... */`.
137///
138/// A distinct value rather than an index, because the caller's response is a
139/// REJECTION, not a shorter span. Running the scan to end of source instead is
140/// how an unclosed comment in an at-rule prelude quietly became part of the
141/// prelude: `@import/*red` parsed as an import of `/*red`.
142const UNTERMINATED: usize = usize::MAX;
143
144/// What [`scan_to_brace_or_end`] found first.
145#[derive(Clone, Copy, PartialEq, Eq, Debug)]
146enum Kind {
147 /// A top-level `{` — so the run just scanned is a selector or a prelude.
148 Selector,
149 /// A top-level `;`/`}` or end of input — so it is a property or params.
150 Decl,
151}
152
153/// Rule names at which a `/* */` comment is captured as a NODE: the
154/// statement / declaration / keyframe LIST readers, which lex the first token
155/// of each item.
156///
157/// The item builders (`statement`/`decl`/`keyframe`) are deliberately absent.
158/// They reuse the token the list reader already produced, so a comment seen
159/// mid-construct — between a property and its `:`, say — is read under a
160/// builder, declined here, and skipped as insignificant whitespace.
161///
162/// The block wrappers are present because their empty-block `#OB #CB`
163/// lookahead lexes the first body token, so a comment right after `{` is
164/// captured there.
165const COMMENT_NODE_RULES: [&str; 6] = [
166 "items",
167 "decls",
168 "kfitems",
169 "declbody",
170 "rulesbody",
171 "kfbody",
172];
173
174fn is_comment_node_rule(name: &str) -> bool {
175 COMMENT_NODE_RULES.contains(&name)
176}
177
178/// At-keywords whose body is a declaration list rather than a rules list.
179const DECLS_KW: [&str; 7] = [
180 "font-face",
181 "page",
182 "viewport",
183 "-ms-viewport",
184 "counter-style",
185 "property",
186 "font-palette-values",
187];
188
189/// `^(-[a-z]+-)?keyframes$`, without a regex engine.
190fn is_keyframes_kw(kw: &str) -> bool {
191 if kw == "keyframes" {
192 return true;
193 }
194 match vendor_prefix(kw) {
195 Some(v) => &kw[v.len()..] == "keyframes",
196 None => false,
197 }
198}
199
200// The heads of the at-rule nodes: `makeAtRules`, `makeAtDecls`,
201// `makeKeyframes` and `makeAtStmt` in the canonical port, each without the
202// body array the grammar's constructor adds. Key order is JavaScript's: a
203// key assigned twice keeps its first place.
204
205/// The head of a block at-rule whose body is a list of rules (`@media`,
206/// `@supports`, `@document`, `@host`, and any other with a rules body).
207pub(crate) fn at_rules_head(kw: &str, prelude: &str) -> Value {
208 let mut node = object();
209 match kw {
210 "host" => put(&mut node, "type", text("host")),
211 _ if "document" == kw || kw.ends_with("-document") => {
212 // `vendor` is always present on a document node, the empty
213 // string when the at-keyword carries no prefix.
214 put(&mut node, "type", text("document"));
215 put(&mut node, "document", text(prelude));
216 put(&mut node, "vendor", text(vendor_prefix(kw).unwrap_or("")));
217 }
218 _ => {
219 put(&mut node, "type", text(kw));
220 put(&mut node, kw, text(prelude));
221 }
222 }
223 node
224}
225
226/// The head of a block at-rule whose body is declarations (`@font-face`,
227/// `@page`, and any other with a declarations body). A `@page` prelude is a
228/// selector group: `@page toc, index:blank`.
229pub(crate) fn at_decls_head(kw: &str, prelude: &str) -> Value {
230 let mut node = object();
231 put(&mut node, "type", text(kw));
232 if "page" == kw {
233 let selectors = split_selectors(prelude)
234 .into_iter()
235 .map(Value::String)
236 .collect();
237 put(&mut node, "selectors", Value::array(selectors));
238 }
239 node
240}
241
242/// The head of a `@keyframes` block, possibly vendor-prefixed.
243pub(crate) fn keyframes_head(kw: &str, name: &str) -> Value {
244 let mut node = object();
245 put(&mut node, "type", text("keyframes"));
246 put(&mut node, "name", text(name));
247 if let Some(vendor) = vendor_prefix(kw) {
248 put(&mut node, "vendor", text(vendor));
249 }
250 node
251}
252
253/// A statement at-rule (`@import`, `@charset`, `@namespace`, …), whole: the
254/// at-keyword is the node type and the field carrying its params.
255///
256/// `@custom-media --name <query>` splits its params into a name and a media
257/// query, as `/^(--\S+)\s*([\s\S]*)$/` does in the canonical port: `--` and
258/// at least one non-space character, then the rest, trimmed.
259pub(crate) fn at_stmt_head(kw: &str, params: &str) -> Value {
260 let mut node = object();
261 if "custom-media" == kw && params.starts_with("--") {
262 let name_end = params.find(es_is_whitespace).unwrap_or(params.len());
263 if 2 < name_end {
264 put(&mut node, "type", text("custom-media"));
265 put(&mut node, "name", text(¶ms[..name_end]));
266 put(&mut node, "media", text(es_trim(¶ms[name_end..])));
267 return node;
268 }
269 }
270 put(&mut node, "type", text(kw));
271 put(&mut node, kw, text(params));
272 node
273}
274
275/// `^(-[a-z]+-)`: a vendor prefix on an at-keyword, e.g. `-webkit-keyframes`.
276pub fn vendor_prefix(kw: &str) -> Option<&str> {
277 let b = kw.as_bytes();
278 if b.first() != Some(&b'-') {
279 return None;
280 }
281 let mut i = 1;
282 while i < b.len() && b[i].is_ascii_lowercase() {
283 i += 1;
284 }
285 if 1 < i && b.get(i) == Some(&b'-') {
286 Some(&kw[..i + 1])
287 } else {
288 None
289 }
290}
291
292/// Whether `c` is whitespace as ECMAScript defines it, which is NOT what
293/// [`char::is_whitespace`] answers.
294///
295/// The canonical port trims with JavaScript's `String.prototype.trim`, whose
296/// set is ECMAScript WhiteSpace plus LineTerminator. Rust's set is the Unicode
297/// `White_Space` property. They differ by exactly two code points, and both
298/// differences change the AST:
299///
300/// - **U+FEFF** (zero-width no-break space, the byte-order mark) is
301/// ECMAScript whitespace and is not Unicode `White_Space`. A stylesheet that
302/// opens with a byte-order mark yields the selector `a` in the canonical
303/// port and would yield `\u{feff}a` under [`str::trim`].
304/// - **U+0085** (next line) is Unicode `White_Space` and is not ECMAScript
305/// whitespace. `@media x \u{85}{…}` keeps that character in its prelude in
306/// the canonical port and would lose it under [`str::trim`].
307///
308/// Everything else in either set is in both: the ASCII controls, U+0020, the
309/// `Zs` category, and the two line separators U+2028 and U+2029.
310pub fn es_is_whitespace(c: char) -> bool {
311 ('\u{feff}' == c) || (c.is_whitespace() && '\u{85}' != c)
312}
313
314/// Trim as JavaScript's `String.prototype.trim` does. See
315/// [`es_is_whitespace`] for why this is not [`str::trim`].
316pub fn es_trim(s: &str) -> &str {
317 s.trim_matches(es_is_whitespace)
318}
319
320/// The column width of `s`: its length in UTF-16 code units, which is what a
321/// JavaScript string index counts.
322pub(crate) fn col_width(s: &str) -> usize {
323 s.chars().map(char::len_utf16).sum()
324}
325
326/// Slice `src` by byte offsets, clamping to something sliceable.
327///
328/// This is the Rust half of the Go port's `clampLen`, and it exists for the
329/// same reason: the scanners deliberately overshoot. An `i += 2` over a `\`
330/// escape, an unterminated string or an unterminated comment can step past the
331/// end of the source, and JavaScript CLAMPS an out-of-range slice bound where
332/// Go panics — so the Go port clamps to reproduce the JavaScript result rather
333/// than invent a new one.
334///
335/// Rust panics on one bound more than Go does: an index inside a multi-byte
336/// character. Every index these scanners RETURN is at an ASCII character or at
337/// the end of the source, because each one stops at `{`, `}`, `;`, `,`, `*/`,
338/// a quote or end of input, so the fallback below is unreachable in practice —
339/// it is here so that a future scanner change degrades to a shorter span
340/// instead of a panic in a caller's parse.
341fn safe_slice(src: &str, start: usize, end: usize) -> &str {
342 let mut start = start.min(src.len());
343 let mut end = end.clamp(start, src.len());
344 while 0 < start && !src.is_char_boundary(start) {
345 start -= 1;
346 }
347 while start < end && !src.is_char_boundary(end) {
348 end -= 1;
349 }
350 &src[start..end]
351}
352
353/// The index of the `*` that opens the `*/` closing the comment at `i`, or
354/// `None` if there is no closing `*/`.
355fn find_comment_end(src: &[u8], i: usize) -> Option<usize> {
356 let mut j = i + 2;
357 while j + 1 < src.len() {
358 if src[j] == b'*' && src[j + 1] == b'/' {
359 return Some(j);
360 }
361 j += 1;
362 }
363 None
364}
365
366/// The index after a closed `/* ... */`, or [`UNTERMINATED`].
367///
368/// It does NOT run to end of source. An unclosed `/* ... */` is an error, not
369/// a comment to the end of the file — which is what both the engine's builtin
370/// comment matcher and reworkcss do, and what `test/spec/reworkcss.tsv` pins.
371fn skip_comment(src: &[u8], i: usize) -> usize {
372 match find_comment_end(src, i) {
373 // `e + 1` is the `/`, so `e + 2` is always within the source.
374 Some(e) => e + 2,
375 None => UNTERMINATED,
376 }
377}
378
379/// Skip a quoted string; returns the index after the closing quote.
380fn skip_string(src: &[u8], mut i: usize) -> usize {
381 let quote = src[i];
382 i += 1;
383 while i < src.len() {
384 if src[i] == b'\\' {
385 i += 2;
386 continue;
387 }
388 if src[i] == quote {
389 return i + 1;
390 }
391 i += 1;
392 }
393 // A trailing backslash steps i ONE PAST the end. That overshoot is not a
394 // defect to clamp away here: the canonical port returns it too, and its
395 // column arithmetic counts it. See [`css_token`] and [`safe_slice`].
396 i
397}
398
399/// Scan a key prelude: where it ends, and whether it is a selector (a
400/// top-level `{` comes first) or a declaration (a top-level `;`/`}` or end of
401/// input comes first). Strings, `()`, `[]` and comments are skipped.
402fn scan_to_brace_or_end(src: &[u8], i: usize) -> (Kind, usize) {
403 let mut scan = Scan { i, depth: 0 };
404 loop {
405 if let Some(found) = scan.step(src, 0) {
406 return found;
407 }
408 }
409}
410
411/// [`scan_to_brace_or_end`] one step at a time. The position and the
412/// bracket depth are its whole state, so two scans in the same state go on
413/// to the same answer: what [`BraceScans`] relies on.
414#[derive(Clone, Copy, Debug)]
415struct Scan {
416 i: usize,
417 depth: usize,
418}
419
420impl Scan {
421 /// Take one step: past a string, a comment, an escape or one byte.
422 /// `Some` is the scan's answer. `src` is the text the lexer holds, and
423 /// `base` the source's byte offset of its first byte: the scan's
424 /// positions are the source's, as the kept ones in [`BraceScans`] are.
425 fn step(&mut self, src: &[u8], base: usize) -> Option<(Kind, usize)> {
426 let at = self.i - base;
427 let Some(&c) = src.get(at) else {
428 // i may be one past the end; see skip_string.
429 return Some((Kind::Decl, self.i));
430 };
431 if c == b'"' || c == b'\'' {
432 self.i = base + skip_string(src, at);
433 return None;
434 }
435 if c == b'/' && src.get(at + 1) == Some(&b'*') {
436 let n = skip_comment(src, at);
437 if UNTERMINATED == n {
438 return Some((Kind::Decl, UNTERMINATED));
439 }
440 self.i = base + n;
441 return None;
442 }
443 // A CSS escape (`\(`, `\'`, `\3A `, …) hides the next character from
444 // the structural scan.
445 if c == b'\\' {
446 self.i += 2;
447 return None;
448 }
449 if c == b'(' || c == b'[' {
450 self.depth += 1;
451 } else if c == b')' || c == b']' {
452 self.depth = self.depth.saturating_sub(1);
453 } else if 0 == self.depth {
454 if c == b'{' {
455 return Some((Kind::Selector, self.i));
456 }
457 if c == b';' || c == b'}' {
458 return Some((Kind::Decl, self.i));
459 }
460 }
461 self.i += 1;
462 None
463 }
464}
465
466/// [`scan_to_brace_or_end`] for every token of a parse, in linear time.
467///
468/// Each selector of a group, and each value of a keyframe-selector group,
469/// is classified by scanning to the group's `{`, so a group of `k` items
470/// scanned from each item's start costs `k` times the group's length: a
471/// 200 KB list of selectors took 9 s. The scan from an item's start is the
472/// scan from the group's first item, resumed there, whenever that scan
473/// passes the item's start at bracket depth 0 (the state [`Scan`] carries,
474/// outside any string, comment or escape, which it steps over whole). So
475/// the answer is kept with a cursor that follows the tokens forward, and a
476/// later start the cursor reaches in that state takes the kept answer. Any
477/// other start scans afresh and becomes the one kept. The cursor only moves
478/// forward, so a parse's scanning is linear in its length.
479///
480/// The kept positions are the source's, and the text is read where the
481/// lexer holds it: in a parse from a reader, the window, which may have
482/// dropped the text behind the cursor. A kept cursor whose text the window
483/// no longer holds is passed over, and the start scans afresh, which costs
484/// time and never changes an answer. An answer the scan found only by
485/// reading to the end of the window is not kept while the source may have
486/// more: the matcher fills the window and asks again.
487#[derive(Clone, Copy, Debug)]
488struct BraceScans {
489 from: usize,
490 found: (Kind, usize),
491 cursor: Scan,
492}
493
494/// The `ctx.u` key [`BraceScans`] is kept under, for the parse.
495pub(crate) const BRACE_SCAN: &str = "tabnas-css/brace-scan";
496
497impl BraceScans {
498 /// The answer for a token starting at byte `at` of `src`, the text the
499 /// lexer holds, whose first byte is the source's byte `base`: its kind,
500 /// and where it ended in `src` (or [`UNTERMINATED`]). The state is kept
501 /// under [`BRACE_SCAN`] with one map lookup: the matcher asks on every
502 /// `#TX`, and a flat stylesheet is mostly misses. Unless `exhausted`,
503 /// the end of `src` may not be the source's, and an answer found there
504 /// is not kept.
505 fn scan(
506 ctx: &mut Context,
507 src: &[u8],
508 base: usize,
509 at: usize,
510 exhausted: bool,
511 ) -> (Kind, usize) {
512 let s_i = base + at;
513 let held = |found: (Kind, usize)| match found.1 {
514 UNTERMINATED => found,
515 end => (found.0, end - base),
516 };
517 let Some(Value::Array(slot)) = ctx.u.get_mut(BRACE_SCAN) else {
518 let found = BraceScans::fresh_scan(src, base, at);
519 if exhausted || !BraceScans::at_end(src, base, found) {
520 let kept = BraceScans::fresh(s_i, found);
521 ctx.u
522 .insert(BRACE_SCAN.to_string(), Value::array(kept.fields().to_vec()));
523 }
524 return held(found);
525 };
526 let fields = Arc::make_mut(slot);
527 if let Some(mut kept) = BraceScans::decode(fields) {
528 if kept.from <= s_i
529 && s_i < kept.found.1
530 && base <= kept.cursor.i
531 && kept.cursor.i <= s_i
532 {
533 while kept.cursor.i < s_i && kept.cursor.step(src, base).is_none() {}
534 let reached = kept.cursor.i == s_i && 0 == kept.cursor.depth;
535 kept.encode(fields);
536 if reached {
537 return held(kept.found);
538 }
539 }
540 }
541 // An unclosed comment's answer is kept too: a scan from a later start
542 // the cursor reaches meets the same `/*`, and the engine's recovery
543 // asks again at the very start it failed at, up to its skip budget.
544 let found = BraceScans::fresh_scan(src, base, at);
545 if exhausted || !BraceScans::at_end(src, base, found) {
546 BraceScans::fresh(s_i, found).encode(fields);
547 }
548 held(found)
549 }
550
551 /// [`scan_to_brace_or_end`] from byte `at` of `src`, its answer at the
552 /// source's position.
553 fn fresh_scan(src: &[u8], base: usize, at: usize) -> (Kind, usize) {
554 match scan_to_brace_or_end(src, at) {
555 (kind, UNTERMINATED) => (kind, UNTERMINATED),
556 (kind, end) => (kind, base + end),
557 }
558 }
559
560 /// Whether `found`, at the source's position, was read to the end of
561 /// `src`: an unclosed comment, or the end of the text, which a `\`
562 /// escape can overshoot by one.
563 fn at_end(src: &[u8], base: usize, found: (Kind, usize)) -> bool {
564 UNTERMINATED == found.1 || base + src.len() <= found.1
565 }
566
567 fn fresh(from: usize, found: (Kind, usize)) -> BraceScans {
568 BraceScans {
569 from,
570 found,
571 cursor: Scan { i: from, depth: 0 },
572 }
573 }
574
575 fn decode(fields: &[Value]) -> Option<BraceScans> {
576 let at = |n: usize| match fields.get(n) {
577 Some(Value::Number(v)) => Some(*v as usize),
578 _ => None,
579 };
580 let kind = if 0 == at(1)? {
581 Kind::Selector
582 } else {
583 Kind::Decl
584 };
585 Some(BraceScans {
586 from: at(0)?,
587 found: (kind, at(2)?),
588 cursor: Scan {
589 i: at(3)?,
590 depth: at(4)?,
591 },
592 })
593 }
594
595 fn numbers(&self) -> [f64; 5] {
596 let kind = match self.found.0 {
597 Kind::Selector => 0,
598 Kind::Decl => 1,
599 };
600 [
601 self.from,
602 kind,
603 self.found.1,
604 self.cursor.i,
605 self.cursor.depth,
606 ]
607 .map(|n| n as f64)
608 }
609
610 fn fields(&self) -> [Value; 5] {
611 self.numbers().map(Value::Number)
612 }
613
614 /// Write this state over `fields`, number by number when the shape fits.
615 fn encode(&self, fields: &mut Vec<Value>) {
616 let numbers = self.numbers();
617 if numbers.len() == fields.len() {
618 for (field, n) in fields.iter_mut().zip(numbers) {
619 match field {
620 Value::Number(slot) => *slot = n,
621 other => *other = Value::Number(n),
622 }
623 }
624 } else {
625 *fields = self.fields().to_vec();
626 }
627 }
628}
629
630/// Scan a single selector: to the next top-level `,` (a group separator) or
631/// `{`, or end of input. Strings, `()`, `[]` and comments are skipped.
632fn scan_selector_end(src: &[u8], mut i: usize) -> usize {
633 let mut depth = 0usize;
634 while i < src.len() {
635 let c = src[i];
636 if c == b'"' || c == b'\'' {
637 i = skip_string(src, i);
638 continue;
639 }
640 if c == b'/' && src.get(i + 1) == Some(&b'*') {
641 let n = skip_comment(src, i);
642 if UNTERMINATED == n {
643 return UNTERMINATED;
644 }
645 i = n;
646 continue;
647 }
648 if c == b'\\' {
649 i += 2;
650 continue;
651 }
652 if c == b'(' || c == b'[' {
653 depth += 1;
654 } else if c == b')' || c == b']' {
655 depth = depth.saturating_sub(1);
656 } else if 0 == depth && (c == b',' || c == b'{') {
657 return i;
658 }
659 i += 1;
660 }
661 // i may be one past the end; see skip_string.
662 i
663}
664
665/// Scan a declaration value or at-rule params: to the next top-level `;` or
666/// `}`, or end of input. Strings, `()`, `[]` and comments are skipped.
667fn scan_value_end(src: &[u8], mut i: usize) -> usize {
668 let mut depth = 0usize;
669 while i < src.len() {
670 let c = src[i];
671 if c == b'"' || c == b'\'' {
672 i = skip_string(src, i);
673 continue;
674 }
675 if c == b'/' && src.get(i + 1) == Some(&b'*') {
676 let n = skip_comment(src, i);
677 if UNTERMINATED == n {
678 return UNTERMINATED;
679 }
680 i = n;
681 continue;
682 }
683 if c == b'\\' {
684 i += 2;
685 continue;
686 }
687 if c == b'(' || c == b'[' {
688 depth += 1;
689 } else if c == b')' || c == b']' {
690 depth = depth.saturating_sub(1);
691 } else if 0 == depth && (c == b';' || c == b'}') {
692 return i;
693 }
694 i += 1;
695 }
696 // i may be one past the end; see skip_string.
697 i
698}
699
700/// Remove `/* ... */` comments from a selector or value run, leaving quoted
701/// strings (which may contain `/*`) untouched.
702pub fn strip_comments(s: &str) -> String {
703 if !s.contains("/*") {
704 return s.to_string();
705 }
706 let src = s.as_bytes();
707 let mut out: Vec<u8> = Vec::with_capacity(src.len());
708 let mut i = 0;
709 while i < src.len() {
710 let c = src[i];
711 if c == b'"' || c == b'\'' {
712 let j = skip_string(src, i);
713 out.extend_from_slice(&src[i..j.min(src.len())]);
714 i = j;
715 continue;
716 }
717 if c == b'/' && src.get(i + 1) == Some(&b'*') {
718 let n = skip_comment(src, i);
719 if UNTERMINATED == n {
720 // Unreachable once the callers reject an unclosed comment,
721 // but a sentinel assigned to i would spin. The rest of the
722 // span is comment text; drop it.
723 break;
724 }
725 i = n;
726 continue;
727 }
728 out.push(c);
729 i += 1;
730 }
731 String::from_utf8(out).unwrap_or_else(|e| String::from_utf8_lossy(e.as_bytes()).into_owned())
732}
733
734/// A CSS property name character.
735///
736/// Beyond the identifier characters this admits the legacy IE hack prefixes
737/// `*`, `#`, `/` (as in `//prop`) and a literal backslash, matching the
738/// reworkcss property pattern `\*?[-#\/\*\\\w]+(\[[0-9a-z_-]+\])?` — real
739/// stylesheets rely on them.
740fn is_prop_char(c: u8) -> bool {
741 c.is_ascii_alphanumeric()
742 || c == b'-'
743 || c == b'_'
744 || c == b'#'
745 || c == b'*'
746 || c == b'/'
747 || c == b'\\'
748}
749
750/// A `[0-9a-z_-]` character, the only content allowed in a property name's
751/// bracket suffix (e.g. `opacity[sqrt]`).
752fn is_prop_suffix_char(c: u8) -> bool {
753 c.is_ascii_digit() || c.is_ascii_lowercase() || c == b'-' || c == b'_'
754}
755
756/// Scan a property name: the run of property characters plus an optional
757/// `[...]` suffix. Returns the index after the name (`== i` if there is none).
758fn scan_prop_end(src: &[u8], i: usize) -> usize {
759 let mut e_i = i;
760 while e_i < src.len() && is_prop_char(src[e_i]) {
761 e_i += 1;
762 }
763 if e_i == i {
764 return i;
765 }
766 if src.get(e_i) == Some(&b'[') {
767 let mut b_i = e_i + 1;
768 while b_i < src.len() && is_prop_suffix_char(src[b_i]) {
769 b_i += 1;
770 }
771 if e_i + 1 < b_i && src.get(b_i) == Some(&b']') {
772 return b_i + 1;
773 }
774 }
775 e_i
776}
777
778/// An at-keyword character: letters, digits and `-`.
779fn is_at_char(c: u8) -> bool {
780 c.is_ascii_alphanumeric() || c == b'-'
781}
782
783/// Split a selector-group prelude on its top-level commas (commas inside
784/// strings, `()` or `[]` are part of a single selector), stripping comments
785/// and trimming each selector. An empty prelude yields an empty list.
786pub fn split_selectors(prelude: &str) -> Vec<String> {
787 let src = prelude.as_bytes();
788 let mut out = Vec::new();
789 let mut i = 0;
790 while i <= src.len() {
791 let mut end = scan_selector_end(src, i);
792 if UNTERMINATED == end {
793 // Unreachable in practice: the prelude was scanned (and an
794 // unclosed comment rejected) before it got here. Treated as "the
795 // rest" so a sentinel can never index or spin.
796 end = src.len();
797 }
798 let one = es_trim(&strip_comments(safe_slice(prelude, i, end))).to_string();
799 if !one.is_empty() {
800 out.push(one);
801 }
802 if end >= src.len() {
803 break;
804 }
805 i = end + 1;
806 }
807 out
808}
809
810// ---------------------------------------------------------------------------
811// The `cssToken` matcher, as the engine runs it.
812
813/// The token number a matcher gives a token the engine is to resolve by
814/// NAME, as it lexes: the six this grammar names itself (`#CC`, `#GC` and
815/// the four at-rule tokens).
816///
817/// The engine numbers a name when it is first registered, and
818/// [`Tabnas::merge`](tabnas::Tabnas::merge) renumbers every custom name in
819/// the merged instance without running the plugin again, so a number
820/// captured at install names another token there. The canonical port's
821/// `lex.token('#CC', …)` resolves the name on every call for the same
822/// reason. `#TX` and `#VL` are the engine's own, with fixed numbers.
823const BY_NAME: EngineTin = -1;
824
825/// Where a scan ran past the end of the source, and by how much: the
826/// `ctx.u` key [`css_token`] records it under and [`lex_subscriber`] reads.
827pub(crate) const OVERSHOOT: &str = "tabnas-css/overshoot";
828
829/// The most of the source an `unterminated_comment` diagnostic shows. The
830/// span runs from the scan's start to the end of the source, which for a
831/// large stylesheet would be most of it; the message is informative, not the
832/// contract, so it is cut here.
833const BAD_SPAN: usize = 32;
834
835/// What [`plan_token`] decided for the characters at the cursor.
836enum Plan {
837 /// Not this matcher's: the engine's builtin matchers take it.
838 Decline,
839 /// An unclosed `/* ... */` behind the cursor: `unterminated_comment`.
840 Bad,
841 /// A token, and the byte offset just past it, which may be ONE past the
842 /// end of the source (see [`css_token`]).
843 Emit {
844 name: &'static str,
845 tin: EngineTin,
846 val: Value,
847 raw: String,
848 end: usize,
849 },
850 /// A scan read to the end of the text the lexer holds while the source
851 /// may have more: [`css_token`] fills the window and plans again.
852 Edge,
853}
854
855/// Decide the token at byte `s_i` of `src`, read under the rule `rule`.
856///
857/// `src` is the text the lexer holds: the whole source, or in a parse from
858/// a reader the window. Unless `exhausted`, a scan that reads to the end of
859/// `src` answers [`Plan::Edge`]; with `exhausted`, the end of `src` is the
860/// end of the source.
861///
862/// `buildCssTokenMatcher` in `ts/src/css.ts`, step for step: whitespace is
863/// declined (the engine's space and line matchers take it), a comment is a
864/// node only at a list position, a value runs to the next top-level `;` or
865/// `}` under `declval`, a top-level `,` separates a selector group, `@`
866/// starts an at-rule, `{` `}` `;` are the grammar's, and anything else is a
867/// selector or a property name by `{`-before-`;` lookahead.
868fn plan_token(
869 src: &str,
870 s_i: usize,
871 rule: &str,
872 lower: bool,
873 exhausted: bool,
874 brace: &mut dyn FnMut(usize) -> (Kind, usize),
875) -> Plan {
876 let bytes = src.as_bytes();
877 // Whether a scan that stopped at `i` read to the end of `src`: it ran
878 // off the end, which a `\` escape can overshoot by one, or met an
879 // unclosed comment, whose `*/` may lie past the end.
880 let edge = |i: usize| !exhausted && (UNTERMINATED == i || bytes.len() <= i);
881 let Some(&c) = bytes.get(s_i) else {
882 return if exhausted { Plan::Decline } else { Plan::Edge };
883 };
884 if c == b' ' || c == b'\t' || c == b'\r' || c == b'\n' {
885 return Plan::Decline;
886 }
887 let emit = |name, tin, val: String, raw: &str, end| Plan::Emit {
888 name,
889 tin,
890 val: Value::String(val),
891 raw: raw.to_string(),
892 end,
893 };
894
895 // Comments: a node at a list position, otherwise declined, and then
896 // skipped by the engine's builtin comment matcher. Whether a `/` opens
897 // one depends on the byte after it.
898 if c == b'/' && edge(s_i + 1) {
899 return Plan::Edge;
900 }
901 if c == b'/' && bytes.get(s_i + 1) == Some(&b'*') {
902 if !is_comment_node_rule(rule) {
903 return Plan::Decline;
904 }
905 return match find_comment_end(bytes, s_i) {
906 None if !exhausted => Plan::Edge,
907 None => Plan::Bad,
908 Some(e) => emit(
909 "#CC",
910 BY_NAME,
911 safe_slice(src, s_i + 2, e).to_string(),
912 safe_slice(src, s_i, e + 2),
913 e + 2,
914 ),
915 };
916 }
917
918 // Value position: a declaration value up to the next top-level `;`/`}`,
919 // comments stripped and surrounding space trimmed.
920 if "declval" == rule {
921 if c == b'{' || c == b'}' || c == b';' || c == b':' {
922 return Plan::Decline;
923 }
924 let end = scan_value_end(bytes, s_i);
925 if edge(end) {
926 return Plan::Edge;
927 }
928 if UNTERMINATED == end {
929 return Plan::Bad;
930 }
931 let raw = safe_slice(src, s_i, end);
932 let val = es_trim(&strip_comments(raw)).to_string();
933 return emit("#VL", TIN_VL, val, raw, end);
934 }
935
936 // A top-level selector-group comma.
937 if c == b',' {
938 return emit("#GC", BY_NAME, ",".to_string(), ",", s_i + 1);
939 }
940
941 // An at-rule: block or statement by `{`-before-`;` lookahead, and a
942 // block's body by keyword. The token's value is the node's head: its
943 // type and the fields read from the keyword and the prelude (block) or
944 // params (statement), so that a reader of the rule passes (the
945 // transducer part, `alchemy/transduce.alc`) sees the node as the walk
946 // builds it, a pass carrying a token's value and never its `use` bag.
947 if c == b'@' {
948 let mut k_end = s_i + 1;
949 while k_end < bytes.len() && is_at_char(bytes[k_end]) {
950 k_end += 1;
951 }
952 let kw = safe_slice(src, s_i + 1, k_end).to_string();
953 // The lookahead reads from the `@` on, through the keyword, so where
954 // it stops inside `src` the keyword stopped inside it too, and so
955 // does the params scan below, which stops where it stops.
956 let (kind, index) = brace(s_i);
957 if edge(index) {
958 return Plan::Edge;
959 }
960 if UNTERMINATED == index {
961 return Plan::Bad;
962 }
963 if Kind::Selector == kind {
964 // At-rule preludes KEEP their comments (they are only trimmed),
965 // matching reworkcss.
966 let prelude = es_trim(safe_slice(src, k_end, index)).to_string();
967 let (name, tin) = if is_keyframes_kw(&kw) {
968 ("#ATK", BY_NAME)
969 } else if DECLS_KW.contains(&kw.as_str()) {
970 ("#ATD", BY_NAME)
971 } else {
972 ("#ATR", BY_NAME)
973 };
974 let head = match name {
975 "#ATK" => keyframes_head(&kw, &prelude),
976 "#ATD" => at_decls_head(&kw, &prelude),
977 _ => at_rules_head(&kw, &prelude),
978 };
979 let raw = safe_slice(src, s_i, index);
980 return Plan::Emit {
981 name,
982 tin,
983 val: head,
984 raw: raw.to_string(),
985 end: index,
986 };
987 }
988 // A statement at-rule's params run to the next top-level `;`/`}`. A
989 // `;` is consumed (it ends the statement); a `}` or the end of the
990 // source is left for the enclosing rule.
991 let p_end = scan_value_end(bytes, k_end);
992 if UNTERMINATED == p_end {
993 return Plan::Bad;
994 }
995 let params = es_trim(safe_slice(src, k_end, p_end)).to_string();
996 let end = if bytes.get(p_end) == Some(&b';') {
997 p_end + 1
998 } else {
999 p_end
1000 };
1001 let raw = safe_slice(src, s_i, end);
1002 return Plan::Emit {
1003 name: "#ATS",
1004 tin: BY_NAME,
1005 val: at_stmt_head(&kw, ¶ms),
1006 raw: raw.to_string(),
1007 end,
1008 };
1009 }
1010
1011 // Other fixed punctuation belongs to the grammar. `:` does not: a run
1012 // that starts with one is scanned below, as in the canonical port.
1013 if c == b'{' || c == b'}' || c == b';' {
1014 return Plan::Decline;
1015 }
1016
1017 // A selector or a property name, by `{`-before-`;` lookahead.
1018 let (kind, index) = brace(s_i);
1019 if edge(index) {
1020 return Plan::Edge;
1021 }
1022 if UNTERMINATED == index {
1023 return Plan::Bad;
1024 }
1025 // Both scans below stop at or before what ended the lookahead, which
1026 // skipped what they skip and stopped at a `{`, `;` or `}`, none of which
1027 // they read past, so they read only text the lookahead read.
1028 if Kind::Selector == kind {
1029 // One selector of a (possible) group: up to the next top-level `,`
1030 // or `{`, comments stripped and surrounding space trimmed.
1031 let end = scan_selector_end(bytes, s_i);
1032 if UNTERMINATED == end {
1033 return Plan::Bad;
1034 }
1035 let raw = safe_slice(src, s_i, end);
1036 let val = es_trim(&strip_comments(raw)).to_string();
1037 return emit("#TX", TIN_TX, val, raw, end);
1038 }
1039
1040 // A property name: the run of property characters up to `:`,
1041 // whitespace, `;` or `}`. `/` and `*` are property characters (the
1042 // `*prop` and `//prop` IE hacks), so a trailing `/*...*/` hack comment
1043 // lands inside the scanned name; comments are stripped out of it, as
1044 // reworkcss does.
1045 let e_i = scan_prop_end(bytes, s_i);
1046 if e_i == s_i {
1047 return Plan::Decline;
1048 }
1049 let raw = safe_slice(src, s_i, e_i);
1050 let mut prop = strip_comments(raw);
1051 if prop.is_empty() {
1052 return Plan::Decline;
1053 }
1054 if lower {
1055 prop = prop.to_lowercase();
1056 }
1057 emit("#TX", TIN_TX, prop, raw, e_i)
1058}
1059
1060/// The `cssToken` matcher, registered as `@css-token` and named by the
1061/// grammar's options at order 1e5, ahead of every builtin matcher.
1062///
1063/// The token is planned over the text the lexer holds: the whole source,
1064/// or in a parse from a reader (`lex.window`, admin ADR-30) the window,
1065/// which keeps the text from the oldest token the parser holds to what it
1066/// has read, and which [`Lexer::window`] gives from the cursor on. Each
1067/// scan reads forward from the cursor to what ends it, so one that stops
1068/// inside the window gives what the whole text gives. One that reaches the
1069/// window's end may have stopped short: the plan is then [`Plan::Edge`],
1070/// and the matcher asks the window for twice what it holds past the cursor
1071/// ([`Lexer::fill`]) and plans again over the grown text. Where the window
1072/// does not grow, the source ends there, and the plan is made with the end
1073/// of the window taken for the end of the source. A long token so costs its
1074/// length to read, not its square. In a whole text `fill` only answers, and
1075/// the first plan stands.
1076///
1077/// Two things here are deliberate and easy to "fix" wrongly:
1078///
1079/// - The end of a token may be ONE PAST the source. A `\` at the last
1080/// character is an escape whose escaped character is not there, and the
1081/// scanners' `i += 2` steps off the end. The canonical port advances its
1082/// column by that unclamped width and takes the CLAMPED span for the
1083/// token's text, so `@host\` yields a node ending at column 7 inside a
1084/// stylesheet ending at column 8, one past a source six characters long.
1085/// The engine will not move its cursor past the end, so the overshoot is
1086/// recorded under [`OVERSHOOT`] and [`lex_subscriber`] adds it to the
1087/// end-of-source token, the only token that can follow it.
1088/// - An unclosed comment is reported WITHOUT advancing, so the diagnostic
1089/// points at the start of the scan, as the canonical port's does.
1090///
1091/// A matcher must never advance and then decline: the engine captured its
1092/// dispatch state before the custom matchers ran.
1093pub(crate) fn css_token(
1094 lexer: &mut Lexer<'_>,
1095 rule: &mut Rule,
1096 ctx: &mut Context,
1097 lower: bool,
1098) -> Option<Token> {
1099 let point = lexer.point();
1100 let mut exhausted = !lexer.windowed();
1101 let (plan, chars, overshoot) = loop {
1102 // `held` is the text the lexer holds, and the cursor is byte
1103 // `s_i` of it, the source's byte `point.site.si`.
1104 let ahead = lexer.window().len();
1105 let held = lexer.source();
1106 let s_i = held.len() - ahead;
1107 let base = point.site.si - s_i;
1108 let bytes = held.as_bytes();
1109 let mut brace = |at: usize| BraceScans::scan(ctx, bytes, base, at, exhausted);
1110 let plan = plan_token(held, s_i, rule.name.as_str(), lower, exhausted, &mut brace);
1111 let (chars, overshoot) = match &plan {
1112 Plan::Decline => return None,
1113 Plan::Edge => {
1114 let read = held[s_i..].chars().count();
1115 lexer.fill(read.saturating_mul(2).max(read + 1));
1116 if lexer.window().len() <= ahead {
1117 exhausted = true;
1118 }
1119 continue;
1120 }
1121 Plan::Bad => (
1122 safe_slice(held, s_i, held.len())
1123 .chars()
1124 .take(BAD_SPAN)
1125 .count(),
1126 0,
1127 ),
1128 Plan::Emit { end, .. } => (
1129 safe_slice(held, s_i, (*end).min(held.len()))
1130 .chars()
1131 .count(),
1132 end.saturating_sub(held.len()),
1133 ),
1134 };
1135 break (plan, chars, overshoot);
1136 };
1137 match plan {
1138 Plan::Decline | Plan::Edge => None,
1139 Plan::Bad => Some(lexer.bad_span(
1140 "unterminated_comment",
1141 point.site.pos,
1142 point.site.pos + chars,
1143 )),
1144 Plan::Emit {
1145 name,
1146 tin,
1147 val,
1148 raw,
1149 ..
1150 } => {
1151 lexer.advance_chars(chars);
1152 if 0 < overshoot {
1153 ctx.u
1154 .insert(OVERSHOOT.to_string(), Value::Number(overshoot as f64));
1155 }
1156 Some(lexer.token(name, tin, val, raw, point))
1157 }
1158 }
1159}
1160
1161/// The plugin's lex subscriber: it sees every token the engine fetches.
1162///
1163/// - The end-of-source token takes the column overshoot [`css_token`]
1164/// recorded, so the stylesheet's end and an error at the end of the source
1165/// land where the canonical port puts them.
1166/// - It once dropped the unconsumed lookahead when a bad token arrived
1167/// behind a good one, so that an unclosed comment hidden from the
1168/// property scan was the error reported rather than `unexpected` at the
1169/// property. tabnas/parser#274 made the engine raise a fetched bad token
1170/// at once, as the canonical does, and record and skip it under
1171/// recovery, so that work is the engine's now. `test/spec/comments.tsv`
1172/// and `tests/plugin.rs` pin the answers in every mode.
1173/// - In a parse from a reader, which has no source to list them from, it
1174/// lists the astral characters of each token's text for the positions
1175/// (`plugin::note_astral`).
1176pub(crate) fn lex_subscriber(token: &mut Token, _rule: &mut Rule, ctx: &mut Context) {
1177 if TIN_ZZ == token.tin {
1178 if let Some(Value::Number(n)) = ctx.u.get(OVERSHOOT) {
1179 token.site.ci += *n as usize;
1180 }
1181 }
1182 // A parse from a reader a window at a time has no source in its context.
1183 if ctx.source.is_empty() {
1184 crate::plugin::note_astral(token, ctx);
1185 }
1186}
1187
1188/// The number of astral characters (those two UTF-16 code units wide) in
1189/// scalars `[from, to)` of `src`.
1190pub(crate) fn astral_between(src: &str, from: usize, to: usize) -> usize {
1191 src.chars()
1192 .skip(from)
1193 .take(to.saturating_sub(from))
1194 .filter(|c| 2 == c.len_utf16())
1195 .count()
1196}
1197
1198/// An engine error as this crate's, with the column in UTF-16 code units.
1199///
1200/// The engine counts a column in Unicode scalars since the last reset (a
1201/// `\n` in any token, or the end of a run of line characters); the canonical
1202/// port counts UTF-16 code units since the same points. The two differ by
1203/// the astral characters between the reset and the error, which is what is
1204/// added here. The message is the engine's, and is not the contract.
1205impl From<TabnasError> for Error {
1206 fn from(error: TabnasError) -> Error {
1207 let from = error.pos.saturating_sub(error.col.saturating_sub(1));
1208 let wide = astral_between(&error.full_source, from, error.pos);
1209 Error {
1210 code: error.code,
1211 message: error.detail,
1212 line: error.row,
1213 column: error.col + wide,
1214 }
1215 }
1216}