qframe/widgets/
highlight.rs

1//! Syntax highlighting for code shown in the terminal: Rust and TOML.
2
3use std::ops::Range;
4
5/// A language [`CodeView`](crate::widgets::CodeView) can colour.
6#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7pub enum Language {
8    /// Rust source.
9    Rust,
10    /// TOML files such as themes and locales.
11    Toml,
12    /// No highlighting.
13    Plain,
14}
15
16impl Language {
17    /// The language for a fenced code block tag such as `rust` or `toml`.
18    #[must_use]
19    pub fn from_tag(tag: &str) -> Self {
20        match tag.trim().to_ascii_lowercase().as_str() {
21            "rust" | "rs" => Self::Rust,
22            "toml" => Self::Toml,
23            _ => Self::Plain,
24        }
25    }
26}
27
28/// What a highlighted piece of code is; also the theme variant of `code-token`.
29#[derive(Debug, Clone, Copy, PartialEq, Eq)]
30pub(crate) enum Token {
31    Keyword,
32    Type,
33    Function,
34    Macro,
35    String,
36    Number,
37    Comment,
38    Attribute,
39    Lifetime,
40    Punctuation,
41    Table,
42    Key,
43    Plain,
44}
45
46impl Token {
47    pub(crate) fn variant(self) -> &'static str {
48        match self {
49            Self::Keyword => "keyword",
50            Self::Type => "type",
51            Self::Function => "function",
52            Self::Macro => "macro",
53            Self::String => "string",
54            Self::Number => "number",
55            Self::Comment => "comment",
56            Self::Attribute => "attribute",
57            Self::Lifetime => "lifetime",
58            Self::Punctuation => "punctuation",
59            Self::Table => "table",
60            Self::Key => "key",
61            Self::Plain => "plain",
62        }
63    }
64}
65
66const RUST_KEYWORDS: [&str; 41] = [
67    "as", "async", "await", "break", "const", "continue", "crate", "dyn", "else", "enum", "extern", "false", "fn",
68    "for", "if", "impl", "in", "let", "loop", "match", "mod", "move", "mut", "pub", "ref", "return", "self", "Self",
69    "static", "struct", "super", "trait", "true", "type", "unsafe", "use", "where", "while", "yield", "gen", "try",
70];
71
72/// Splits `code` into highlighted byte ranges covering the whole text.
73pub(crate) fn highlight(code: &str, language: Language) -> Vec<(Range<usize>, Token)> {
74    let tokens = match language {
75        Language::Rust => rust(code),
76        Language::Toml => toml(code),
77        Language::Plain => vec![(0..code.len(), Token::Plain)],
78    };
79    fill_gaps(code.len(), tokens)
80}
81
82fn fill_gaps(len: usize, tokens: Vec<(Range<usize>, Token)>) -> Vec<(Range<usize>, Token)> {
83    let mut out = Vec::with_capacity(tokens.len() * 2);
84    let mut position = 0;
85    for (range, token) in tokens {
86        if range.start > position {
87            out.push((position..range.start, Token::Plain));
88        }
89        if range.end > range.start {
90            position = range.end;
91            out.push((range, token));
92        }
93    }
94    if position < len {
95        out.push((position..len, Token::Plain));
96    }
97    out
98}
99
100struct Scanner<'a> {
101    text: &'a str,
102    pos: usize,
103}
104
105impl Scanner<'_> {
106    fn peek(&self) -> Option<char> {
107        self.text[self.pos..].chars().next()
108    }
109
110    fn peek_at(&self, n: usize) -> Option<char> {
111        self.text[self.pos..].chars().nth(n)
112    }
113
114    fn starts_with(&self, s: &str) -> bool {
115        self.text[self.pos..].starts_with(s)
116    }
117
118    fn bump(&mut self) -> Option<char> {
119        let c = self.peek()?;
120        self.pos += c.len_utf8();
121        Some(c)
122    }
123
124    fn eat_while(&mut self, keep: impl Fn(char) -> bool) {
125        while self.peek().is_some_and(&keep) {
126            self.bump();
127        }
128    }
129
130    fn skip_line(&mut self) {
131        self.eat_while(|c| c != '\n');
132    }
133
134    /// Consumes a quoted string starting at the opening quote, honouring backslash escapes.
135    fn quoted(&mut self, quote: char) {
136        self.bump();
137        while let Some(c) = self.bump() {
138            if c == '\\' {
139                self.bump();
140            } else if c == quote {
141                break;
142            }
143        }
144    }
145}
146
147fn is_ident_start(c: char) -> bool {
148    c.is_alphabetic() || c == '_'
149}
150
151fn is_ident(c: char) -> bool {
152    c.is_alphanumeric() || c == '_'
153}
154
155fn rust(code: &str) -> Vec<(Range<usize>, Token)> {
156    let mut s = Scanner { text: code, pos: 0 };
157    let mut out = Vec::new();
158    let mut previous_word = String::new();
159    while let Some(c) = s.peek() {
160        let start = s.pos;
161        if s.starts_with("//") {
162            s.skip_line();
163            out.push((start..s.pos, Token::Comment));
164        } else if s.starts_with("/*") {
165            match code[s.pos + 2..].find("*/") {
166                Some(end) => s.pos += 2 + end + 2,
167                None => s.pos = code.len(),
168            }
169            out.push((start..s.pos, Token::Comment));
170        } else if s.starts_with("#[") || s.starts_with("#![") {
171            let mut depth = 0;
172            while let Some(c) = s.bump() {
173                match c {
174                    '[' => depth += 1,
175                    ']' => {
176                        depth -= 1;
177                        if depth == 0 {
178                            break;
179                        }
180                    }
181                    '\n' => break,
182                    _ => {}
183                }
184            }
185            out.push((start..s.pos, Token::Attribute));
186        } else if c == '"' || ((c == 'b') && s.peek_at(1) == Some('"')) {
187            if c == 'b' {
188                s.bump();
189            }
190            s.quoted('"');
191            out.push((start..s.pos, Token::String));
192        } else if c == 'r' && (s.peek_at(1) == Some('"') || (s.peek_at(1) == Some('#') && s.peek_at(2) != Some('['))) {
193            s.bump();
194            let mut hashes = 0;
195            while s.peek() == Some('#') {
196                s.bump();
197                hashes += 1;
198            }
199            if s.peek() == Some('"') {
200                let closing = format!("\"{}", "#".repeat(hashes));
201                s.bump();
202                match code[s.pos..].find(&closing) {
203                    Some(end) => s.pos += end + closing.len(),
204                    None => s.pos = code.len(),
205                }
206                out.push((start..s.pos, Token::String));
207            } else {
208                s.eat_while(is_ident);
209            }
210        } else if c == '\'' {
211            let is_char = s.peek_at(1) == Some('\\') || s.peek_at(2) == Some('\'');
212            if is_char {
213                s.quoted('\'');
214                out.push((start..s.pos, Token::String));
215            } else {
216                s.bump();
217                s.eat_while(is_ident);
218                out.push((start..s.pos, Token::Lifetime));
219            }
220        } else if c.is_ascii_digit() {
221            s.eat_while(|c| c.is_ascii_alphanumeric() || c == '_' || c == '.');
222            if code[start..s.pos].ends_with('.') {
223                s.pos -= 1;
224            }
225            out.push((start..s.pos, Token::Number));
226        } else if is_ident_start(c) {
227            s.eat_while(is_ident);
228            let word = &code[start..s.pos];
229            let token = if RUST_KEYWORDS.contains(&word) {
230                Token::Keyword
231            } else if s.peek() == Some('!') && s.peek_at(1) != Some('=') {
232                s.bump();
233                Token::Macro
234            } else if previous_word == "fn" || s.peek() == Some('(') || s.starts_with("::<") {
235                Token::Function
236            } else if word.chars().next().is_some_and(char::is_uppercase) {
237                Token::Type
238            } else {
239                Token::Plain
240            };
241            previous_word = word.to_owned();
242            out.push((start..s.pos, token));
243            continue;
244        } else if c.is_whitespace() {
245            s.bump();
246            continue;
247        } else {
248            s.bump();
249            out.push((start..s.pos, Token::Punctuation));
250        }
251        previous_word.clear();
252    }
253    out
254}
255
256fn toml(code: &str) -> Vec<(Range<usize>, Token)> {
257    let mut out = Vec::new();
258    let mut line_start = 0;
259    for line in code.split_inclusive('\n') {
260        let mut s = Scanner { text: code, pos: line_start };
261        let end = line_start + line.len();
262        s.eat_while(|c| c == ' ' || c == '\t');
263        if s.peek() == Some('[') {
264            let header_start = s.pos;
265            while s.pos < end && s.peek() != Some(']') {
266                if s.peek() == Some('"') {
267                    s.quoted('"');
268                } else {
269                    s.bump();
270                }
271            }
272            s.eat_while(|c| c == ']');
273            out.push((header_start..s.pos, Token::Table));
274        } else if s.peek().is_some_and(|c| is_ident(c) || c == '"' || c == '-') {
275            let key_start = s.pos;
276            while s.pos < end && !matches!(s.peek(), Some('=') | Some('\n') | Some('#')) {
277                if s.peek() == Some('"') {
278                    s.quoted('"');
279                } else {
280                    s.bump();
281                }
282            }
283            let key_end = key_start + code[key_start..s.pos].trim_end().len();
284            if s.peek() == Some('=') {
285                out.push((key_start..key_end, Token::Key));
286            } else {
287                s.pos = key_start;
288            }
289        }
290        while s.pos < end {
291            let start = s.pos;
292            match s.peek() {
293                Some('#') => {
294                    s.skip_line();
295                    out.push((start..s.pos, Token::Comment));
296                }
297                Some(q @ ('"' | '\'')) => {
298                    s.quoted(q);
299                    out.push((start..s.pos.min(end), Token::String));
300                    s.pos = s.pos.min(end);
301                }
302                Some(c) if c.is_ascii_digit() || (c == '-' && s.peek_at(1).is_some_and(|d| d.is_ascii_digit())) => {
303                    s.bump();
304                    s.eat_while(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '.' | ':' | '-'));
305                    out.push((start..s.pos, Token::Number));
306                }
307                Some(c) if is_ident_start(c) => {
308                    s.eat_while(is_ident);
309                    let token = match &code[start..s.pos] {
310                        "true" | "false" => Token::Keyword,
311                        _ => Token::Plain,
312                    };
313                    out.push((start..s.pos, token));
314                }
315                Some(c) if c.is_whitespace() => {
316                    s.bump();
317                }
318                Some(_) => {
319                    s.bump();
320                    out.push((start..s.pos, Token::Punctuation));
321                }
322                None => break,
323            }
324        }
325        line_start = end;
326    }
327    out
328}
329
330#[cfg(test)]
331mod tests {
332    use super::*;
333
334    fn kinds(code: &str, language: Language) -> Vec<(&str, Token)> {
335        highlight(code, language)
336            .into_iter()
337            .filter(|(r, t)| *t != Token::Plain || !code[r.clone()].trim().is_empty())
338            .map(|(r, t)| (&code[r], t))
339            .collect()
340    }
341
342    #[test]
343    fn rust_tokens() {
344        let code = "#[derive(Debug)]\nfn view(&self, ui: &mut View<'_, Msg>) {\n    ui.add(Text::new(t!(\"hi\")) // note\n        .width(3));\n}";
345        let tokens = kinds(code, Language::Rust);
346        assert!(tokens.contains(&("#[derive(Debug)]", Token::Attribute)));
347        assert!(tokens.contains(&("fn", Token::Keyword)));
348        assert!(tokens.contains(&("view", Token::Function)));
349        assert!(tokens.contains(&("View", Token::Type)));
350        assert!(tokens.contains(&("'_", Token::Lifetime)));
351        assert!(tokens.contains(&("t!", Token::Macro)));
352        assert!(tokens.contains(&("\"hi\"", Token::String)));
353        assert!(tokens.contains(&("// note", Token::Comment)));
354        assert!(tokens.contains(&("3", Token::Number)));
355        assert!(tokens.contains(&("new", Token::Function)));
356    }
357
358    #[test]
359    fn toml_tokens() {
360        let code = "[style.\"button:hover\"]\nbg = \"$raised\" # surface\npadding = [0, 2]\nslide = true\n";
361        let tokens = kinds(code, Language::Toml);
362        assert!(tokens.contains(&("[style.\"button:hover\"]", Token::Table)));
363        assert!(tokens.contains(&("bg", Token::Key)));
364        assert!(tokens.contains(&("\"$raised\"", Token::String)));
365        assert!(tokens.contains(&("# surface", Token::Comment)));
366        assert!(tokens.contains(&("2", Token::Number)));
367        assert!(tokens.contains(&("true", Token::Keyword)));
368    }
369
370    #[test]
371    fn ranges_cover_the_whole_text() {
372        let code = "let x = 'a'; r#\"raw\"#";
373        let ranges = highlight(code, Language::Rust);
374        let joined: String = ranges.iter().map(|(r, _)| &code[r.clone()]).collect();
375        assert_eq!(joined, code);
376        assert!(ranges.iter().any(|(r, t)| &code[r.clone()] == "r#\"raw\"#" && *t == Token::String));
377    }
378}
qframe/widgets/highlight.rs

qframe/widgets/
highlight.rs
