eui_core/
code.rs

1//! Round 1 (docs/round-1-contract.md §3): CodeBlock's BUILT-IN tokenizer, a
2//! line-by-line port of `src/code.ts` over `catalog/code.json` (embedded).
3//! One forward scan, first matching rule wins; the tokens split into LINES
4//! and adjacent tokens of one kind merge. `fixtures/code-tokens.json` locks
5//! it. The scan walks `char`s where the TS walks UTF-16 units; the merge step
6//! makes the two agree for any text (an astral char is one plain token
7//! either way).
8
9use std::collections::HashSet;
10use std::sync::LazyLock;
11
12use indexmap::IndexMap;
13use serde::{Deserialize, Serialize};
14
15use crate::generated::catalog as g;
16
17#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
18pub struct CodeToken {
19    pub kind: String,
20    pub text: String,
21}
22
23#[derive(Debug, Clone, Default, Deserialize)]
24#[serde(rename_all = "camelCase")]
25pub struct LanguageSpec {
26    #[serde(default)]
27    pub plain: bool,
28    /// EXP-1324: strip ANSI CSI escapes first, SGR colours → token kinds.
29    #[serde(default)]
30    pub ansi: bool,
31    #[serde(default)]
32    pub line_kinds: Option<IndexMap<String, String>>,
33    #[serde(default)]
34    pub line_comment: Vec<String>,
35    #[serde(default)]
36    pub block_comment: Vec<(String, String)>,
37    #[serde(default)]
38    pub strings: Vec<String>,
39    #[serde(default)]
40    pub multiline: Vec<String>,
41    #[serde(default)]
42    pub key_strings: bool,
43    #[serde(default)]
44    pub markup: bool,
45    #[serde(default)]
46    pub numbers: Option<bool>,
47    #[serde(default)]
48    pub ident_extra: Option<String>,
49    #[serde(default)]
50    pub keywords: Vec<String>,
51    #[serde(default)]
52    pub case_insensitive: bool,
53    #[serde(default)]
54    pub key_idents: bool,
55    #[serde(default)]
56    pub type_names: Vec<String>,
57    #[serde(default)]
58    pub types: Option<String>,
59    #[serde(default)]
60    pub operators: Option<String>,
61    #[serde(default)]
62    pub punctuation: Option<String>,
63}
64
65#[derive(Deserialize)]
66struct Defaults {
67    operators: String,
68    punctuation: String,
69}
70
71#[derive(Deserialize)]
72struct CodeFile {
73    defaults: Defaults,
74    languages: IndexMap<String, LanguageSpec>,
75}
76
77static CODE: LazyLock<CodeFile> = LazyLock::new(|| serde_json::from_str(g::CODE_JSON).expect("code.json"));
78
79/// The languages, in `codeLanguage` enum order.
80pub fn code_language_names() -> Vec<&'static str> {
81    CODE.languages.keys().map(String::as_str).collect()
82}
83
84pub fn language(name: &str) -> Option<&'static LanguageSpec> {
85    CODE.languages.get(name)
86}
87
88/// #VAPP-258 (`fixtures/markdown-render.json`): the language a Markdown
89/// fence's info string (` ```ts `) is highlighted in: a catalog language
90/// other than `plain` and `ansi` (whose escapes the tokenizer drops), else
91/// `None` = plain text, no colours (src/code.ts `markdownCodeLanguage`).
92pub fn markdown_code_language(lang: Option<&str>) -> Option<&str> {
93    lang.filter(|l| *l != "plain" && language(l).is_some_and(|spec| !spec.ansi))
94}
95
96fn is_digit(c: Option<char>) -> bool {
97    c.is_some_and(|c| c.is_ascii_digit())
98}
99fn is_alpha(c: Option<char>) -> bool {
100    c.is_some_and(|c| c.is_ascii_alphabetic())
101}
102fn is_space(c: Option<char>) -> bool {
103    matches!(c, Some(' ') | Some('\t'))
104}
105
106struct Scanner<'a> {
107    code: &'a [char],
108    out: Vec<CodeToken>,
109    /// Whether index 0 starts a line (an ansi run may begin mid-line).
110    line_start: bool,
111}
112
113impl Scanner<'_> {
114    fn at(&self, i: usize) -> Option<char> {
115        self.code.get(i).copied()
116    }
117    fn starts_with(&self, p: &str, i: usize) -> bool {
118        (i..).zip(p.chars()).all(|(k, c)| self.at(k) == Some(c))
119    }
120    fn slice(&self, a: usize, b: usize) -> String {
121        self.code[a..b.min(self.code.len())].iter().collect()
122    }
123    fn push(&mut self, kind: &str, text: String) {
124        self.out.push(CodeToken { kind: kind.to_string(), text });
125    }
126    fn next_non_space(&self, j: usize) -> Option<char> {
127        let mut k = j;
128        while is_space(self.at(k)) {
129            k += 1;
130        }
131        self.at(k)
132    }
133    fn index_of(&self, needle: &str, from: usize) -> Option<usize> {
134        let n: Vec<char> = needle.chars().collect();
135        if n.is_empty() {
136            return Some(from);
137        }
138        (from..self.code.len()).find(|&k| self.code[k..].starts_with(&n))
139    }
140    /// True when only spaces/tabs separate `i` from the start of its line.
141    fn space_run_to_line_start(&self, i: usize) -> bool {
142        space_run_to_line_start(self.code, i, self.line_start)
143    }
144}
145
146/// True when only spaces/tabs separate `i` from the start of its line
147/// (`line_start` = whether index 0 starts a line).
148fn space_run_to_line_start(code: &[char], i: usize, line_start: bool) -> bool {
149    let mut k = i as isize - 1;
150    while k >= 0 && is_space(code.get(k as usize).copied()) {
151        k -= 1;
152    }
153    if k < 0 {
154        line_start
155    } else {
156        code[k as usize] == '\n'
157    }
158}
159
160/// The source as flat tokens (text may span lines).
161fn scan(code: &[char], spec: &LanguageSpec, line_start: bool) -> Vec<CodeToken> {
162    let mut s = Scanner { code, out: Vec::new(), line_start };
163    let n = code.len();
164    let operators: Vec<char> = spec.operators.as_deref().unwrap_or(&CODE.defaults.operators).chars().collect();
165    let punctuation: Vec<char> = spec.punctuation.as_deref().unwrap_or(&CODE.defaults.punctuation).chars().collect();
166    let extra: Vec<char> = spec.ident_extra.as_deref().unwrap_or("").chars().collect();
167    let ident_start = |c: Option<char>| is_alpha(c) || c == Some('_') || c == Some('$') || c.is_some_and(|c| extra.contains(&c));
168    let ident_part = |c: Option<char>| ident_start(c) || is_digit(c);
169    let keywords: HashSet<&str> = spec.keywords.iter().map(String::as_str).collect();
170    let type_names: HashSet<&str> = spec.type_names.iter().map(String::as_str).collect();
171    if spec.plain {
172        return if code.is_empty() { Vec::new() } else { vec![CodeToken { kind: "plain".into(), text: code.iter().collect() }] };
173    }
174    let mut i = 0usize;
175    let mut in_tag = false;
176    while i < n {
177        let c = code[i];
178        if c == '\n' {
179            s.push("plain", "\n".into());
180            i += 1;
181            continue;
182        }
183        // (1) line kinds
184        if let Some(kinds) = &spec.line_kinds {
185            let at_line = if i == 0 { line_start } else { code[i - 1] == '\n' || s.space_run_to_line_start(i) };
186            if at_line {
187                // EXP-1324: the LONGEST listed prefix wins (`FAILED` over `FAIL`).
188                let prefix = kinds.keys().filter(|p| s.starts_with(p, i)).fold(None::<&String>, |best, p| match best {
189                    Some(b) if b.chars().count() >= p.chars().count() => Some(b),
190                    _ => Some(p),
191                });
192                if let Some(prefix) = prefix {
193                    if !is_space(Some(c)) {
194                        let mut j = i;
195                        while j < n && code[j] != '\n' {
196                            j += 1;
197                        }
198                        let kind = kinds[prefix].clone();
199                        let text = s.slice(i, j);
200                        s.push(&kind, text);
201                        i = j;
202                        continue;
203                    }
204                }
205            }
206        }
207        // (2) line comments
208        if !in_tag && spec.line_comment.iter().any(|p| s.starts_with(p, i)) {
209            let mut j = i;
210            while j < n && code[j] != '\n' {
211                j += 1;
212            }
213            let text = s.slice(i, j);
214            s.push("comment", text);
215            i = j;
216            continue;
217        }
218        // (3) block comments
219        if !in_tag {
220            if let Some((open, close)) = spec.block_comment.iter().find(|(open, _)| s.starts_with(open, i)) {
221                let j = match s.index_of(close, i + open.chars().count()) {
222                    Some(end) => end + close.chars().count(),
223                    None => n,
224                };
225                let text = s.slice(i, j);
226                s.push("comment", text);
227                i = j;
228                continue;
229            }
230        }
231        // (4) strings
232        let quote = c.to_string();
233        if spec.strings.contains(&quote) && (!spec.markup || in_tag) {
234            let multi = spec.multiline.contains(&quote);
235            let mut j = i + 1;
236            while j < n {
237                if code[j] == '\\' {
238                    j += 2;
239                    continue;
240                }
241                if code[j] == c {
242                    j += 1;
243                    break;
244                }
245                if code[j] == '\n' && !multi {
246                    break;
247                }
248                j += 1;
249            }
250            let j = j.min(n);
251            let kind = if spec.key_strings && s.next_non_space(j) == Some(':') { "property" } else { "string" };
252            let text = s.slice(i, j);
253            s.push(kind, text);
254            i = j;
255            continue;
256        }
257        // (5) markup
258        if spec.markup {
259            if !in_tag {
260                let next = s.at(i + 1);
261                if c == '<' && (is_alpha(next) || next == Some('/') || next == Some('!')) {
262                    let open = if next == Some('/') { "</" } else { "<" };
263                    s.push("punctuation", open.into());
264                    let mut j = i + open.len();
265                    let start = j;
266                    while j < n && (is_alpha(Some(code[j])) || is_digit(Some(code[j])) || matches!(code[j], '-' | ':' | '!')) {
267                        j += 1;
268                    }
269                    if j > start {
270                        let text = s.slice(start, j);
271                        s.push("tag", text);
272                    }
273                    in_tag = true;
274                    i = j;
275                    continue;
276                }
277                let mut j = i;
278                while j < n && code[j] != '<' && code[j] != '\n' {
279                    j += 1;
280                }
281                if j == i {
282                    j = i + 1;
283                }
284                let text = s.slice(i, j);
285                s.push("plain", text);
286                i = j;
287                continue;
288            }
289            if c == '>' || s.starts_with("/>", i) {
290                let close = if c == '>' { ">" } else { "/>" };
291                s.push("punctuation", close.into());
292                in_tag = false;
293                i += close.len();
294                continue;
295            }
296            if c == '=' {
297                s.push("operator", "=".into());
298                i += 1;
299                continue;
300            }
301            if is_alpha(Some(c)) || matches!(c, '_' | ':' | '@') {
302                let mut j = i;
303                while j < n && (is_alpha(Some(code[j])) || is_digit(Some(code[j])) || "_:@.-".contains(code[j])) {
304                    j += 1;
305                }
306                let text = s.slice(i, j);
307                s.push("attribute", text);
308                i = j;
309                continue;
310            }
311            if is_space(Some(c)) {
312                let mut j = i;
313                while is_space(s.at(j)) {
314                    j += 1;
315                }
316                let text = s.slice(i, j);
317                s.push("plain", text);
318                i = j;
319                continue;
320            }
321            s.push("plain", c.to_string());
322            i += 1;
323            continue;
324        }
325        // (6) numbers
326        if spec.numbers != Some(false) && (c.is_ascii_digit() || (c == '.' && is_digit(s.at(i + 1)))) {
327            let mut j = i + 1;
328            while j < n && (code[j].is_ascii_alphanumeric() || code[j] == '_' || code[j] == '.') {
329                j += 1;
330            }
331            let text = s.slice(i, j);
332            s.push("number", text);
333            i = j;
334            continue;
335        }
336        // (7) identifiers
337        if ident_start(Some(c)) {
338            let mut j = i + 1;
339            while j < n && ident_part(Some(code[j])) {
340                j += 1;
341            }
342            let word = s.slice(i, j);
343            let key = if spec.case_insensitive { word.to_lowercase() } else { word.clone() };
344            let next = s.next_non_space(j);
345            let after = {
346                let mut k = j;
347                while is_space(s.at(k)) {
348                    k += 1;
349                }
350                s.at(k + 1)
351            };
352            let prev = s.out.iter().rev().find(|t| !t.text.trim().is_empty());
353            let kind = if keywords.contains(key.as_str()) {
354                "keyword"
355            } else if spec.key_idents && next == Some(':') && after != Some(':') {
356                "property"
357            } else if next == Some('(') {
358                "function"
359            } else if prev.is_some_and(|p| p.kind == "punctuation" && p.text.ends_with('.')) {
360                "property"
361            } else if type_names.contains(word.as_str()) || (spec.types.as_deref() == Some("capitalized") && word.starts_with(|ch: char| ch.is_ascii_uppercase())) {
362                "type"
363            } else {
364                "plain"
365            };
366            s.push(kind, word);
367            i = j;
368            continue;
369        }
370        // (8) spaces
371        if is_space(Some(c)) {
372            let mut j = i;
373            while is_space(s.at(j)) {
374                j += 1;
375            }
376            let text = s.slice(i, j);
377            s.push("plain", text);
378            i = j;
379            continue;
380        }
381        // (9) operators
382        if operators.contains(&c) {
383            let mut j = i;
384            while j < n && operators.contains(&code[j]) {
385                j += 1;
386            }
387            let text = s.slice(i, j);
388            s.push("operator", text);
389            i = j;
390            continue;
391        }
392        // (10) punctuation, (11) anything else
393        s.push(if punctuation.contains(&c) { "punctuation" } else { "plain" }, c.to_string());
394        i += 1;
395    }
396    s.out
397}
398
399/// EXP-1324 `ansi`: one run of the escape-free text with the SGR colour (a
400/// codeToken kind) it was printed in, or none.
401#[derive(Debug, Clone, PartialEq, Eq)]
402pub struct AnsiRun {
403    pub kind: Option<&'static str>,
404    pub text: String,
405}
406
407/// The SGR colour after applying `params` left to right: 31/91 error,
408/// 32/92 success, 33/93 warning, 2/90 comment, 0/39/empty reset; any other
409/// parameter keeps the colour.
410fn sgr_kind(params: &str, kind: Option<&'static str>) -> Option<&'static str> {
411    let mut out = kind;
412    for raw in params.split(';') {
413        let p: Option<u32> = if raw.is_empty() { Some(0) } else { raw.parse().ok() };
414        match p {
415            Some(0) | Some(39) => out = None,
416            Some(31) | Some(91) => out = Some("error"),
417            Some(32) | Some(92) => out = Some("success"),
418            Some(33) | Some(93) => out = Some("warning"),
419            Some(2) | Some(90) => out = Some("comment"),
420            _ => {}
421        }
422    }
423    out
424}
425
426/// The source with every ANSI CSI escape removed, split into runs by SGR
427/// colour (catalog/code.json `$comment`, `ansi`); adjacent runs of one colour
428/// merge, an incomplete escape stays text. Port of `ansiRuns`.
429pub fn ansi_runs(source: &str) -> Vec<AnsiRun> {
430    let chars: Vec<char> = source.chars().collect();
431    let n = chars.len();
432    let mut runs: Vec<AnsiRun> = Vec::new();
433    let mut kind: Option<&'static str> = None;
434    let mut text = String::new();
435    let flush = |runs: &mut Vec<AnsiRun>, kind: Option<&'static str>, text: &mut String| {
436        if text.is_empty() {
437            return;
438        }
439        match runs.last_mut() {
440            Some(last) if last.kind == kind => last.text.push_str(text),
441            _ => runs.push(AnsiRun { kind, text: text.clone() }),
442        }
443        text.clear();
444    };
445    let in_range = |c: char, lo: u32, hi: u32| (lo..=hi).contains(&(c as u32));
446    let mut i = 0;
447    while i < n {
448        if chars[i] == '\u{1b}' && chars.get(i + 1) == Some(&'[') {
449            let mut j = i + 2;
450            while j < n && in_range(chars[j], 0x30, 0x3f) {
451                j += 1;
452            }
453            let params: String = chars[i + 2..j].iter().collect();
454            while j < n && in_range(chars[j], 0x20, 0x2f) {
455                j += 1;
456            }
457            if j < n && in_range(chars[j], 0x40, 0x7e) {
458                if chars[j] == 'm' {
459                    let next = sgr_kind(&params, kind);
460                    if next != kind {
461                        flush(&mut runs, kind, &mut text);
462                        kind = next;
463                    }
464                }
465                i = j + 1;
466                continue;
467            }
468        }
469        text.push(chars[i]);
470        i += 1;
471    }
472    flush(&mut runs, kind, &mut text);
473    runs
474}
475
476/// `ansi`: coloured runs verbatim, uncoloured ones through the scanner.
477fn scan_ansi(code: &str, spec: &LanguageSpec) -> Vec<CodeToken> {
478    let mut out = Vec::new();
479    let mut at_line = true;
480    for run in ansi_runs(code) {
481        let chars: Vec<char> = run.text.chars().collect();
482        match run.kind {
483            Some(kind) => out.push(CodeToken { kind: kind.to_string(), text: run.text.clone() }),
484            None => out.extend(scan(&chars, spec, at_line)),
485        }
486        at_line = space_run_to_line_start(&chars, chars.len(), at_line);
487    }
488    out
489}
490
491/// The text a CodeBlock shows and copies: CRLF → LF and, for `ansi`, every
492/// escape removed. Port of `codeText`.
493pub fn code_text(source: &str, language_name: &str) -> String {
494    let code = source.replace("\r\n", "\n").replace('\r', "\n");
495    if language(language_name).is_some_and(|l| l.ansi) {
496        ansi_runs(&code).into_iter().map(|r| r.text).collect()
497    } else {
498        code
499    }
500}
501
502/// The source tokenized per LINE: each line's tokens concatenate to exactly
503/// its text (no `\n`), adjacent tokens of one kind merged; an empty line is
504/// an empty list. An unknown language tokenizes as `plain`.
505pub fn tokenize_code(source: &str, language_name: &str) -> Vec<Vec<CodeToken>> {
506    let code: String = source.replace("\r\n", "\n").replace('\r', "\n");
507    let chars: Vec<char> = code.chars().collect();
508    let spec = language(language_name).or_else(|| language("plain")).expect("plain language");
509    let mut lines: Vec<Vec<CodeToken>> = vec![Vec::new()];
510    let tokens = if spec.ansi { scan_ansi(&code, spec) } else { scan(&chars, spec, true) };
511    for token in tokens {
512        for (index, piece) in token.text.split('\n').enumerate() {
513            if index > 0 {
514                lines.push(Vec::new());
515            }
516            if piece.is_empty() {
517                continue;
518            }
519            let line = lines.last_mut().expect("a line");
520            match line.last_mut() {
521                Some(last) if last.kind == token.kind => last.text.push_str(piece),
522                _ => line.push(CodeToken { kind: token.kind.clone(), text: piece.to_string() }),
523            }
524        }
525    }
526    lines
527}
528
529#[cfg(test)]
530mod tests {
531    use super::*;
532
533    #[test]
534    fn the_languages_are_the_generated_list() {
535        assert_eq!(code_language_names(), g::CODE_LANGUAGE_NAMES);
536    }
537
538    #[test]
539    fn lines_concatenate_to_their_text() {
540        let src = "const x = 1 // one\nlet s = \"a\\\"b\"\n\nfn(x)";
541        let lines = tokenize_code(src, "js");
542        let joined: Vec<String> = lines.iter().map(|l| l.iter().map(|t| t.text.as_str()).collect()).collect();
543        assert_eq!(joined, src.split('\n').collect::<Vec<_>>());
544        assert_eq!(lines[0][0], CodeToken { kind: "keyword".into(), text: "const".into() });
545        assert!(lines[2].is_empty());
546        assert_eq!(tokenize_code("x", "nope"), vec![vec![CodeToken { kind: "plain".into(), text: "x".into() }]]);
547    }
548}
eui_core/code.rs

eui_core/
code.rs
