facett_git/
highlight.rs

1//! **Deterministic Rust source syntax highlighter** → coloured spans an egui
2//! `LayoutJob` can render. Pure (no egui, no I/O), so it is unit-tested without a
3//! window and produces bit-identical spans for the same input (FC-7). A tiny
4//! hand-written tokenizer — not a full parser — classifying the things that matter
5//! for reading code: keywords, types, strings/chars, line+block comments, numbers,
6//! attributes, and punctuation.
7
8/// The syntactic class of a span — maps to a theme colour at render time.
9#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10pub enum Kind {
11    /// Plain identifier / whitespace / default text.
12    Text,
13    /// A language keyword (`fn`, `let`, `match`, …).
14    Keyword,
15    /// A type-ish name (UpperCamelCase, or a primitive like `u32`).
16    Type,
17    /// A string or char literal (including the quotes).
18    Str,
19    /// A numeric literal.
20    Number,
21    /// A `//` line comment or `/* */` block comment.
22    Comment,
23    /// An `#[attribute]` / `#![inner]`.
24    Attribute,
25    /// Punctuation / operators.
26    Punct,
27}
28
29/// A coloured run: a byte range `[start, end)` into the source + its [`Kind`].
30#[derive(Debug, Clone, Copy, PartialEq, Eq)]
31pub struct Span {
32    pub start: usize,
33    pub end: usize,
34    pub kind: Kind,
35}
36
37const KEYWORDS: &[&str] = &[
38    "as", "async", "await", "break", "const", "continue", "crate", "dyn", "else", "enum", "extern",
39    "false", "fn", "for", "if", "impl", "in", "let", "loop", "match", "mod", "move", "mut", "pub",
40    "ref", "return", "self", "Self", "static", "struct", "super", "trait", "true", "type", "unsafe",
41    "use", "where", "while",
42];
43
44const PRIMITIVES: &[&str] = &[
45    "bool", "char", "str", "u8", "u16", "u32", "u64", "u128", "usize", "i8", "i16", "i32", "i64",
46    "i128", "isize", "f32", "f64",
47];
48
49fn is_ident_start(c: char) -> bool {
50    c == '_' || c.is_ascii_alphabetic()
51}
52fn is_ident_continue(c: char) -> bool {
53    c == '_' || c.is_ascii_alphanumeric()
54}
55
56/// Tokenize Rust `src` into a contiguous, ordered list of coloured [`Span`]s
57/// covering every byte (the concatenation of span ranges == the input). ASCII-byte
58/// driven; multi-byte UTF-8 inside a token is preserved (only token boundaries are
59/// decided on ASCII).
60#[must_use]
61pub fn highlight(src: &str) -> Vec<Span> {
62    let b = src.as_bytes();
63    let n = b.len();
64    let mut out: Vec<Span> = Vec::new();
65    let mut i = 0;
66    let push = |out: &mut Vec<Span>, s: usize, e: usize, k: Kind| {
67        if e > s {
68            out.push(Span { start: s, end: e, kind: k });
69        }
70    };
71    while i < n {
72        let c = b[i] as char;
73        // line comment
74        if c == '/' && i + 1 < n && b[i + 1] == b'/' {
75            let s = i;
76            while i < n && b[i] != b'\n' {
77                i += 1;
78            }
79            push(&mut out, s, i, Kind::Comment);
80            continue;
81        }
82        // block comment (non-nested is fine for colouring)
83        if c == '/' && i + 1 < n && b[i + 1] == b'*' {
84            let s = i;
85            i += 2;
86            while i + 1 < n && !(b[i] == b'*' && b[i + 1] == b'/') {
87                i += 1;
88            }
89            i = (i + 2).min(n);
90            push(&mut out, s, i, Kind::Comment);
91            continue;
92        }
93        // string literal
94        if c == '"' {
95            let s = i;
96            i += 1;
97            while i < n {
98                if b[i] == b'\\' {
99                    i += 2;
100                    continue;
101                }
102                if b[i] == b'"' {
103                    i += 1;
104                    break;
105                }
106                i += 1;
107            }
108            push(&mut out, s, i.min(n), Kind::Str);
109            continue;
110        }
111        // char literal (heuristic: '\?' or 'x' then ')
112        if c == '\'' && i + 1 < n {
113            let s = i;
114            let mut j = i + 1;
115            if b[j] == b'\\' {
116                j += 2;
117            } else {
118                j += 1;
119            }
120            if j < n && b[j] == b'\'' {
121                j += 1;
122                push(&mut out, s, j, Kind::Str);
123                i = j;
124                continue;
125            }
126            // else fall through (it's a lifetime/label) — treat the quote as punct
127        }
128        // attribute #[...] or #![...]
129        if c == '#' && i + 1 < n && (b[i + 1] == b'[' || b[i + 1] == b'!') {
130            let s = i;
131            // consume to the matching ] (shallow)
132            while i < n && b[i] != b'[' {
133                i += 1;
134            }
135            let mut depth = 0i32;
136            while i < n {
137                if b[i] == b'[' {
138                    depth += 1;
139                } else if b[i] == b']' {
140                    depth -= 1;
141                    if depth == 0 {
142                        i += 1;
143                        break;
144                    }
145                }
146                i += 1;
147            }
148            push(&mut out, s, i.min(n), Kind::Attribute);
149            continue;
150        }
151        // number
152        if c.is_ascii_digit() {
153            let s = i;
154            while i < n {
155                let d = b[i] as char;
156                if d.is_ascii_alphanumeric() || d == '_' || d == '.' {
157                    i += 1;
158                } else {
159                    break;
160                }
161            }
162            push(&mut out, s, i, Kind::Number);
163            continue;
164        }
165        // identifier / keyword / type
166        if is_ident_start(c) {
167            let s = i;
168            while i < n && is_ident_continue(b[i] as char) {
169                i += 1;
170            }
171            let word = &src[s..i];
172            let kind = if KEYWORDS.contains(&word) {
173                Kind::Keyword
174            } else if PRIMITIVES.contains(&word) || word.chars().next().is_some_and(|c| c.is_ascii_uppercase()) {
175                Kind::Type
176            } else {
177                Kind::Text
178            };
179            push(&mut out, s, i, kind);
180            continue;
181        }
182        // whitespace → Text run
183        if c.is_ascii_whitespace() {
184            let s = i;
185            while i < n && (b[i] as char).is_ascii_whitespace() {
186                i += 1;
187            }
188            push(&mut out, s, i, Kind::Text);
189            continue;
190        }
191        // punctuation (single byte)
192        push(&mut out, i, i + 1, Kind::Punct);
193        i += 1;
194    }
195    out
196}
197
198#[cfg(test)]
199mod tests {
200    use super::*;
201
202    fn kinds_of<'a>(src: &'a str, spans: &[Span]) -> Vec<(&'a str, Kind)> {
203        spans.iter().map(|s| (&src[s.start..s.end], s.kind)).collect()
204    }
205
206    /// INJECT-ASSERT: spans cover every byte, in order, with no gaps/overlaps —
207    /// the highlighter never drops source (a render invariant).
208    #[test]
209    fn spans_tile_the_whole_source() {
210        let src = "fn main() { let x = 42; }";
211        let spans = highlight(src);
212        assert!(!spans.is_empty());
213        assert_eq!(spans.first().unwrap().start, 0);
214        assert_eq!(spans.last().unwrap().end, src.len());
215        for w in spans.windows(2) {
216            assert_eq!(w[0].end, w[1].start, "contiguous, no gap/overlap");
217        }
218    }
219
220    /// INJECT-ASSERT: keywords, types, numbers, strings, comments, attributes each
221    /// get the right class.
222    #[test]
223    fn classifies_the_main_token_kinds() {
224        let src = "#[derive(Clone)]\nfn f() -> u32 { let s = \"hi\"; /* c */ return 7; } // tail";
225        let got = kinds_of(src, &highlight(src));
226        let find = |needle: &str, k: Kind| got.iter().any(|(t, kk)| *t == needle && *kk == k);
227        assert!(find("fn", Kind::Keyword), "fn is a keyword");
228        assert!(find("let", Kind::Keyword) && find("return", Kind::Keyword));
229        assert!(find("u32", Kind::Type), "u32 is a primitive type");
230        assert!(find("Clone", Kind::Type) || got.iter().any(|(t, k)| t.contains("Clone") && *k == Kind::Attribute));
231        assert!(got.iter().any(|(t, k)| *t == "\"hi\"" && *k == Kind::Str), "string incl quotes");
232        assert!(got.iter().any(|(t, k)| t.contains("/* c */") && *k == Kind::Comment));
233        assert!(got.iter().any(|(t, k)| t.contains("// tail") && *k == Kind::Comment));
234        assert!(got.iter().any(|(t, k)| *t == "7" && *k == Kind::Number));
235        assert!(got.iter().any(|(t, k)| t.starts_with("#[derive") && *k == Kind::Attribute));
236    }
237
238    /// INJECT-ASSERT (FC-7): highlighting is deterministic.
239    #[test]
240    fn deterministic() {
241        let src = "struct S { a: Vec<u8> }";
242        assert_eq!(highlight(src), highlight(src));
243    }
244
245    /// INJECT-ASSERT: an empty source yields no spans (honest empty).
246    #[test]
247    fn empty_is_empty() {
248        assert!(highlight("").is_empty());
249    }
250}
facett_git/highlight.rs

facett_git/
highlight.rs
