eui_core/
code.rs

1//! Round 1 (docs/round-1-contract.md §3): CodeBlock's BUILT-IN tokenizer, a
2//! line-by-line port of `src/code.ts` over `catalog/code.json` (embedded).
3//! One forward scan, first matching rule wins; the tokens split into LINES
4//! and adjacent tokens of one kind merge. `fixtures/code-tokens.json` locks
5//! it. The scan walks `char`s where the TS walks UTF-16 units; the merge step
6//! makes the two agree for any text (an astral char is one plain token
7//! either way).
8
9use std::collections::HashSet;
10use std::sync::LazyLock;
11
12use indexmap::IndexMap;
13use serde::{Deserialize, Serialize};
14
15use crate::generated::catalog as g;
16
17#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
18pub struct CodeToken {
19    pub kind: String,
20    pub text: String,
21}
22
23#[derive(Debug, Clone, Default, Deserialize)]
24#[serde(rename_all = "camelCase")]
25pub struct LanguageSpec {
26    #[serde(default)]
27    pub plain: bool,
28    /// EXP-1324: strip ANSI CSI escapes first, SGR colours → token kinds.
29    #[serde(default)]
30    pub ansi: bool,
31    #[serde(default)]
32    pub line_kinds: Option<IndexMap<String, String>>,
33    #[serde(default)]
34    pub line_comment: Vec<String>,
35    #[serde(default)]
36    pub block_comment: Vec<(String, String)>,
37    #[serde(default)]
38    pub strings: Vec<String>,
39    #[serde(default)]
40    pub multiline: Vec<String>,
41    #[serde(default)]
42    pub key_strings: bool,
43    #[serde(default)]
44    pub markup: bool,
45    #[serde(default)]
46    pub numbers: Option<bool>,
47    #[serde(default)]
48    pub ident_extra: Option<String>,
49    #[serde(default)]
50    pub keywords: Vec<String>,
51    #[serde(default)]
52    pub case_insensitive: bool,
53    #[serde(default)]
54    pub key_idents: bool,
55    #[serde(default)]
56    pub type_names: Vec<String>,
57    #[serde(default)]
58    pub types: Option<String>,
59    #[serde(default)]
60    pub operators: Option<String>,
61    #[serde(default)]
62    pub punctuation: Option<String>,
63}
64
65#[derive(Deserialize)]
66struct Defaults {
67    operators: String,
68    punctuation: String,
69}
70
71#[derive(Deserialize)]
72struct CodeFile {
73    defaults: Defaults,
74    languages: IndexMap<String, LanguageSpec>,
75}
76
77static CODE: LazyLock<CodeFile> = LazyLock::new(|| serde_json::from_str(g::CODE_JSON).expect("code.json"));
78
79/// The languages, in `codeLanguage` enum order.
80pub fn code_language_names() -> Vec<&'static str> {
81    CODE.languages.keys().map(String::as_str).collect()
82}
83
84pub fn language(name: &str) -> Option<&'static LanguageSpec> {
85    CODE.languages.get(name)
86}
87
88fn is_digit(c: Option<char>) -> bool {
89    c.is_some_and(|c| c.is_ascii_digit())
90}
91fn is_alpha(c: Option<char>) -> bool {
92    c.is_some_and(|c| c.is_ascii_alphabetic())
93}
94fn is_space(c: Option<char>) -> bool {
95    matches!(c, Some(' ') | Some('\t'))
96}
97
98struct Scanner<'a> {
99    code: &'a [char],
100    out: Vec<CodeToken>,
101    /// Whether index 0 starts a line (an ansi run may begin mid-line).
102    line_start: bool,
103}
104
105impl Scanner<'_> {
106    fn at(&self, i: usize) -> Option<char> {
107        self.code.get(i).copied()
108    }
109    fn starts_with(&self, p: &str, i: usize) -> bool {
110        (i..).zip(p.chars()).all(|(k, c)| self.at(k) == Some(c))
111    }
112    fn slice(&self, a: usize, b: usize) -> String {
113        self.code[a..b.min(self.code.len())].iter().collect()
114    }
115    fn push(&mut self, kind: &str, text: String) {
116        self.out.push(CodeToken { kind: kind.to_string(), text });
117    }
118    fn next_non_space(&self, j: usize) -> Option<char> {
119        let mut k = j;
120        while is_space(self.at(k)) {
121            k += 1;
122        }
123        self.at(k)
124    }
125    fn index_of(&self, needle: &str, from: usize) -> Option<usize> {
126        let n: Vec<char> = needle.chars().collect();
127        if n.is_empty() {
128            return Some(from);
129        }
130        (from..self.code.len()).find(|&k| self.code[k..].starts_with(&n))
131    }
132    /// True when only spaces/tabs separate `i` from the start of its line.
133    fn space_run_to_line_start(&self, i: usize) -> bool {
134        space_run_to_line_start(self.code, i, self.line_start)
135    }
136}
137
138/// True when only spaces/tabs separate `i` from the start of its line
139/// (`line_start` = whether index 0 starts a line).
140fn space_run_to_line_start(code: &[char], i: usize, line_start: bool) -> bool {
141    let mut k = i as isize - 1;
142    while k >= 0 && is_space(code.get(k as usize).copied()) {
143        k -= 1;
144    }
145    if k < 0 {
146        line_start
147    } else {
148        code[k as usize] == '\n'
149    }
150}
151
152/// The source as flat tokens (text may span lines).
153fn scan(code: &[char], spec: &LanguageSpec, line_start: bool) -> Vec<CodeToken> {
154    let mut s = Scanner { code, out: Vec::new(), line_start };
155    let n = code.len();
156    let operators: Vec<char> = spec.operators.as_deref().unwrap_or(&CODE.defaults.operators).chars().collect();
157    let punctuation: Vec<char> = spec.punctuation.as_deref().unwrap_or(&CODE.defaults.punctuation).chars().collect();
158    let extra: Vec<char> = spec.ident_extra.as_deref().unwrap_or("").chars().collect();
159    let ident_start = |c: Option<char>| is_alpha(c) || c == Some('_') || c == Some('$') || c.is_some_and(|c| extra.contains(&c));
160    let ident_part = |c: Option<char>| ident_start(c) || is_digit(c);
161    let keywords: HashSet<&str> = spec.keywords.iter().map(String::as_str).collect();
162    let type_names: HashSet<&str> = spec.type_names.iter().map(String::as_str).collect();
163    if spec.plain {
164        return if code.is_empty() { Vec::new() } else { vec![CodeToken { kind: "plain".into(), text: code.iter().collect() }] };
165    }
166    let mut i = 0usize;
167    let mut in_tag = false;
168    while i < n {
169        let c = code[i];
170        if c == '\n' {
171            s.push("plain", "\n".into());
172            i += 1;
173            continue;
174        }
175        // (1) line kinds
176        if let Some(kinds) = &spec.line_kinds {
177            let at_line = if i == 0 { line_start } else { code[i - 1] == '\n' || s.space_run_to_line_start(i) };
178            if at_line {
179                // EXP-1324: the LONGEST listed prefix wins (`FAILED` over `FAIL`).
180                let prefix = kinds.keys().filter(|p| s.starts_with(p, i)).fold(None::<&String>, |best, p| match best {
181                    Some(b) if b.chars().count() >= p.chars().count() => Some(b),
182                    _ => Some(p),
183                });
184                if let Some(prefix) = prefix {
185                    if !is_space(Some(c)) {
186                        let mut j = i;
187                        while j < n && code[j] != '\n' {
188                            j += 1;
189                        }
190                        let kind = kinds[prefix].clone();
191                        let text = s.slice(i, j);
192                        s.push(&kind, text);
193                        i = j;
194                        continue;
195                    }
196                }
197            }
198        }
199        // (2) line comments
200        if !in_tag && spec.line_comment.iter().any(|p| s.starts_with(p, i)) {
201            let mut j = i;
202            while j < n && code[j] != '\n' {
203                j += 1;
204            }
205            let text = s.slice(i, j);
206            s.push("comment", text);
207            i = j;
208            continue;
209        }
210        // (3) block comments
211        if !in_tag {
212            if let Some((open, close)) = spec.block_comment.iter().find(|(open, _)| s.starts_with(open, i)) {
213                let j = match s.index_of(close, i + open.chars().count()) {
214                    Some(end) => end + close.chars().count(),
215                    None => n,
216                };
217                let text = s.slice(i, j);
218                s.push("comment", text);
219                i = j;
220                continue;
221            }
222        }
223        // (4) strings
224        let quote = c.to_string();
225        if spec.strings.contains(&quote) && (!spec.markup || in_tag) {
226            let multi = spec.multiline.contains(&quote);
227            let mut j = i + 1;
228            while j < n {
229                if code[j] == '\\' {
230                    j += 2;
231                    continue;
232                }
233                if code[j] == c {
234                    j += 1;
235                    break;
236                }
237                if code[j] == '\n' && !multi {
238                    break;
239                }
240                j += 1;
241            }
242            let j = j.min(n);
243            let kind = if spec.key_strings && s.next_non_space(j) == Some(':') { "property" } else { "string" };
244            let text = s.slice(i, j);
245            s.push(kind, text);
246            i = j;
247            continue;
248        }
249        // (5) markup
250        if spec.markup {
251            if !in_tag {
252                let next = s.at(i + 1);
253                if c == '<' && (is_alpha(next) || next == Some('/') || next == Some('!')) {
254                    let open = if next == Some('/') { "</" } else { "<" };
255                    s.push("punctuation", open.into());
256                    let mut j = i + open.len();
257                    let start = j;
258                    while j < n && (is_alpha(Some(code[j])) || is_digit(Some(code[j])) || matches!(code[j], '-' | ':' | '!')) {
259                        j += 1;
260                    }
261                    if j > start {
262                        let text = s.slice(start, j);
263                        s.push("tag", text);
264                    }
265                    in_tag = true;
266                    i = j;
267                    continue;
268                }
269                let mut j = i;
270                while j < n && code[j] != '<' && code[j] != '\n' {
271                    j += 1;
272                }
273                if j == i {
274                    j = i + 1;
275                }
276                let text = s.slice(i, j);
277                s.push("plain", text);
278                i = j;
279                continue;
280            }
281            if c == '>' || s.starts_with("/>", i) {
282                let close = if c == '>' { ">" } else { "/>" };
283                s.push("punctuation", close.into());
284                in_tag = false;
285                i += close.len();
286                continue;
287            }
288            if c == '=' {
289                s.push("operator", "=".into());
290                i += 1;
291                continue;
292            }
293            if is_alpha(Some(c)) || matches!(c, '_' | ':' | '@') {
294                let mut j = i;
295                while j < n && (is_alpha(Some(code[j])) || is_digit(Some(code[j])) || "_:@.-".contains(code[j])) {
296                    j += 1;
297                }
298                let text = s.slice(i, j);
299                s.push("attribute", text);
300                i = j;
301                continue;
302            }
303            if is_space(Some(c)) {
304                let mut j = i;
305                while is_space(s.at(j)) {
306                    j += 1;
307                }
308                let text = s.slice(i, j);
309                s.push("plain", text);
310                i = j;
311                continue;
312            }
313            s.push("plain", c.to_string());
314            i += 1;
315            continue;
316        }
317        // (6) numbers
318        if spec.numbers != Some(false) && (c.is_ascii_digit() || (c == '.' && is_digit(s.at(i + 1)))) {
319            let mut j = i + 1;
320            while j < n && (code[j].is_ascii_alphanumeric() || code[j] == '_' || code[j] == '.') {
321                j += 1;
322            }
323            let text = s.slice(i, j);
324            s.push("number", text);
325            i = j;
326            continue;
327        }
328        // (7) identifiers
329        if ident_start(Some(c)) {
330            let mut j = i + 1;
331            while j < n && ident_part(Some(code[j])) {
332                j += 1;
333            }
334            let word = s.slice(i, j);
335            let key = if spec.case_insensitive { word.to_lowercase() } else { word.clone() };
336            let next = s.next_non_space(j);
337            let after = {
338                let mut k = j;
339                while is_space(s.at(k)) {
340                    k += 1;
341                }
342                s.at(k + 1)
343            };
344            let prev = s.out.iter().rev().find(|t| !t.text.trim().is_empty());
345            let kind = if keywords.contains(key.as_str()) {
346                "keyword"
347            } else if spec.key_idents && next == Some(':') && after != Some(':') {
348                "property"
349            } else if next == Some('(') {
350                "function"
351            } else if prev.is_some_and(|p| p.kind == "punctuation" && p.text.ends_with('.')) {
352                "property"
353            } else if type_names.contains(word.as_str()) || (spec.types.as_deref() == Some("capitalized") && word.starts_with(|ch: char| ch.is_ascii_uppercase())) {
354                "type"
355            } else {
356                "plain"
357            };
358            s.push(kind, word);
359            i = j;
360            continue;
361        }
362        // (8) spaces
363        if is_space(Some(c)) {
364            let mut j = i;
365            while is_space(s.at(j)) {
366                j += 1;
367            }
368            let text = s.slice(i, j);
369            s.push("plain", text);
370            i = j;
371            continue;
372        }
373        // (9) operators
374        if operators.contains(&c) {
375            let mut j = i;
376            while j < n && operators.contains(&code[j]) {
377                j += 1;
378            }
379            let text = s.slice(i, j);
380            s.push("operator", text);
381            i = j;
382            continue;
383        }
384        // (10) punctuation, (11) anything else
385        s.push(if punctuation.contains(&c) { "punctuation" } else { "plain" }, c.to_string());
386        i += 1;
387    }
388    s.out
389}
390
391/// EXP-1324 `ansi`: one run of the escape-free text with the SGR colour (a
392/// codeToken kind) it was printed in, or none.
393#[derive(Debug, Clone, PartialEq, Eq)]
394pub struct AnsiRun {
395    pub kind: Option<&'static str>,
396    pub text: String,
397}
398
399/// The SGR colour after applying `params` left to right: 31/91 error,
400/// 32/92 success, 33/93 warning, 2/90 comment, 0/39/empty reset; any other
401/// parameter keeps the colour.
402fn sgr_kind(params: &str, kind: Option<&'static str>) -> Option<&'static str> {
403    let mut out = kind;
404    for raw in params.split(';') {
405        let p: Option<u32> = if raw.is_empty() { Some(0) } else { raw.parse().ok() };
406        match p {
407            Some(0) | Some(39) => out = None,
408            Some(31) | Some(91) => out = Some("error"),
409            Some(32) | Some(92) => out = Some("success"),
410            Some(33) | Some(93) => out = Some("warning"),
411            Some(2) | Some(90) => out = Some("comment"),
412            _ => {}
413        }
414    }
415    out
416}
417
418/// The source with every ANSI CSI escape removed, split into runs by SGR
419/// colour (catalog/code.json `$comment`, `ansi`); adjacent runs of one colour
420/// merge, an incomplete escape stays text. Port of `ansiRuns`.
421pub fn ansi_runs(source: &str) -> Vec<AnsiRun> {
422    let chars: Vec<char> = source.chars().collect();
423    let n = chars.len();
424    let mut runs: Vec<AnsiRun> = Vec::new();
425    let mut kind: Option<&'static str> = None;
426    let mut text = String::new();
427    let flush = |runs: &mut Vec<AnsiRun>, kind: Option<&'static str>, text: &mut String| {
428        if text.is_empty() {
429            return;
430        }
431        match runs.last_mut() {
432            Some(last) if last.kind == kind => last.text.push_str(text),
433            _ => runs.push(AnsiRun { kind, text: text.clone() }),
434        }
435        text.clear();
436    };
437    let in_range = |c: char, lo: u32, hi: u32| (lo..=hi).contains(&(c as u32));
438    let mut i = 0;
439    while i < n {
440        if chars[i] == '\u{1b}' && chars.get(i + 1) == Some(&'[') {
441            let mut j = i + 2;
442            while j < n && in_range(chars[j], 0x30, 0x3f) {
443                j += 1;
444            }
445            let params: String = chars[i + 2..j].iter().collect();
446            while j < n && in_range(chars[j], 0x20, 0x2f) {
447                j += 1;
448            }
449            if j < n && in_range(chars[j], 0x40, 0x7e) {
450                if chars[j] == 'm' {
451                    let next = sgr_kind(&params, kind);
452                    if next != kind {
453                        flush(&mut runs, kind, &mut text);
454                        kind = next;
455                    }
456                }
457                i = j + 1;
458                continue;
459            }
460        }
461        text.push(chars[i]);
462        i += 1;
463    }
464    flush(&mut runs, kind, &mut text);
465    runs
466}
467
468/// `ansi`: coloured runs verbatim, uncoloured ones through the scanner.
469fn scan_ansi(code: &str, spec: &LanguageSpec) -> Vec<CodeToken> {
470    let mut out = Vec::new();
471    let mut at_line = true;
472    for run in ansi_runs(code) {
473        let chars: Vec<char> = run.text.chars().collect();
474        match run.kind {
475            Some(kind) => out.push(CodeToken { kind: kind.to_string(), text: run.text.clone() }),
476            None => out.extend(scan(&chars, spec, at_line)),
477        }
478        at_line = space_run_to_line_start(&chars, chars.len(), at_line);
479    }
480    out
481}
482
483/// The text a CodeBlock shows and copies: CRLF → LF and, for `ansi`, every
484/// escape removed. Port of `codeText`.
485pub fn code_text(source: &str, language_name: &str) -> String {
486    let code = source.replace("\r\n", "\n").replace('\r', "\n");
487    if language(language_name).is_some_and(|l| l.ansi) {
488        ansi_runs(&code).into_iter().map(|r| r.text).collect()
489    } else {
490        code
491    }
492}
493
494/// The source tokenized per LINE: each line's tokens concatenate to exactly
495/// its text (no `\n`), adjacent tokens of one kind merged; an empty line is
496/// an empty list. An unknown language tokenizes as `plain`.
497pub fn tokenize_code(source: &str, language_name: &str) -> Vec<Vec<CodeToken>> {
498    let code: String = source.replace("\r\n", "\n").replace('\r', "\n");
499    let chars: Vec<char> = code.chars().collect();
500    let spec = language(language_name).or_else(|| language("plain")).expect("plain language");
501    let mut lines: Vec<Vec<CodeToken>> = vec![Vec::new()];
502    let tokens = if spec.ansi { scan_ansi(&code, spec) } else { scan(&chars, spec, true) };
503    for token in tokens {
504        for (index, piece) in token.text.split('\n').enumerate() {
505            if index > 0 {
506                lines.push(Vec::new());
507            }
508            if piece.is_empty() {
509                continue;
510            }
511            let line = lines.last_mut().expect("a line");
512            match line.last_mut() {
513                Some(last) if last.kind == token.kind => last.text.push_str(piece),
514                _ => line.push(CodeToken { kind: token.kind.clone(), text: piece.to_string() }),
515            }
516        }
517    }
518    lines
519}
520
521#[cfg(test)]
522mod tests {
523    use super::*;
524
525    #[test]
526    fn the_languages_are_the_generated_list() {
527        assert_eq!(code_language_names(), g::CODE_LANGUAGE_NAMES);
528    }
529
530    #[test]
531    fn lines_concatenate_to_their_text() {
532        let src = "const x = 1 // one\nlet s = \"a\\\"b\"\n\nfn(x)";
533        let lines = tokenize_code(src, "js");
534        let joined: Vec<String> = lines.iter().map(|l| l.iter().map(|t| t.text.as_str()).collect()).collect();
535        assert_eq!(joined, src.split('\n').collect::<Vec<_>>());
536        assert_eq!(lines[0][0], CodeToken { kind: "keyword".into(), text: "const".into() });
537        assert!(lines[2].is_empty());
538        assert_eq!(tokenize_code("x", "nope"), vec![vec![CodeToken { kind: "plain".into(), text: "x".into() }]]);
539    }
540}
eui_core/code.rs

eui_core/
code.rs
