qframe/widgets/highlight/
mod.rs

1//! Syntax highlighting for code shown in the terminal: Rust, TOML and shell scripts.
2
3mod shell;
4
5use std::ops::Range;
6
7/// A language [`CodeView`](crate::widgets::CodeView) can colour.
8#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9pub enum Language {
10    /// Rust source.
11    Rust,
12    /// TOML files such as themes and locales.
13    Toml,
14    /// Shell scripts in bash syntax, such as a `PKGBUILD` or a package's `.install` file.
15    Shell,
16    /// No highlighting.
17    Plain,
18}
19
20impl Language {
21    /// The language for a fenced code block tag such as `rust` or `toml`.
22    #[must_use]
23    pub fn from_tag(tag: &str) -> Self {
24        match tag.trim().to_ascii_lowercase().as_str() {
25            "rust" | "rs" => Self::Rust,
26            "toml" => Self::Toml,
27            "sh" | "bash" | "shell" | "zsh" | "pkgbuild" => Self::Shell,
28            _ => Self::Plain,
29        }
30    }
31
32    /// The language of the file named `name` (a bare name or a path): `.rs` is Rust, `.toml`
33    /// is TOML, `.sh`, `.bash`, `.zsh`, `.install` and a `PKGBUILD` are shell scripts, and
34    /// anything else is plain text.
35    #[must_use]
36    pub fn from_file_name(name: &str) -> Self {
37        let path = std::path::Path::new(name);
38        if path.file_name().is_some_and(|file| file == "PKGBUILD") {
39            return Self::Shell;
40        }
41        match path.extension().and_then(|extension| extension.to_str()) {
42            Some("rs") => Self::Rust,
43            Some("toml") => Self::Toml,
44            Some("sh" | "bash" | "zsh" | "install") => Self::Shell,
45            _ => Self::Plain,
46        }
47    }
48}
49
50/// What a highlighted piece of code is; also the theme variant of `code-token`.
51#[derive(Debug, Clone, Copy, PartialEq, Eq)]
52pub(crate) enum Token {
53    Keyword,
54    Type,
55    Function,
56    Macro,
57    String,
58    Number,
59    Comment,
60    Attribute,
61    Lifetime,
62    Punctuation,
63    Table,
64    Key,
65    Variable,
66    Plain,
67}
68
69impl Token {
70    pub(crate) fn variant(self) -> &'static str {
71        match self {
72            Self::Keyword => "keyword",
73            Self::Type => "type",
74            Self::Function => "function",
75            Self::Macro => "macro",
76            Self::String => "string",
77            Self::Number => "number",
78            Self::Comment => "comment",
79            Self::Attribute => "attribute",
80            Self::Lifetime => "lifetime",
81            Self::Punctuation => "punctuation",
82            Self::Table => "table",
83            Self::Key => "key",
84            Self::Variable => "variable",
85            Self::Plain => "plain",
86        }
87    }
88}
89
90const RUST_KEYWORDS: [&str; 41] = [
91    "as", "async", "await", "break", "const", "continue", "crate", "dyn", "else", "enum", "extern", "false", "fn",
92    "for", "if", "impl", "in", "let", "loop", "match", "mod", "move", "mut", "pub", "ref", "return", "self", "Self",
93    "static", "struct", "super", "trait", "true", "type", "unsafe", "use", "where", "while", "yield", "gen", "try",
94];
95
96/// Splits `code` into highlighted byte ranges covering the whole text.
97pub(crate) fn highlight(code: &str, language: Language) -> Vec<(Range<usize>, Token)> {
98    let tokens = match language {
99        Language::Rust => rust(code),
100        Language::Toml => toml(code),
101        Language::Shell => shell::shell(code),
102        Language::Plain => vec![(0..code.len(), Token::Plain)],
103    };
104    fill_gaps(code.len(), tokens)
105}
106
107fn fill_gaps(len: usize, tokens: Vec<(Range<usize>, Token)>) -> Vec<(Range<usize>, Token)> {
108    let mut out = Vec::with_capacity(tokens.len() * 2);
109    let mut position = 0;
110    for (range, token) in tokens {
111        if range.start > position {
112            out.push((position..range.start, Token::Plain));
113        }
114        if range.end > range.start {
115            position = range.end;
116            out.push((range, token));
117        }
118    }
119    if position < len {
120        out.push((position..len, Token::Plain));
121    }
122    out
123}
124
125struct Scanner<'a> {
126    text: &'a str,
127    pos: usize,
128}
129
130impl Scanner<'_> {
131    fn peek(&self) -> Option<char> {
132        self.text[self.pos..].chars().next()
133    }
134
135    fn peek_at(&self, n: usize) -> Option<char> {
136        self.text[self.pos..].chars().nth(n)
137    }
138
139    fn starts_with(&self, s: &str) -> bool {
140        self.text[self.pos..].starts_with(s)
141    }
142
143    fn bump(&mut self) -> Option<char> {
144        let c = self.peek()?;
145        self.pos += c.len_utf8();
146        Some(c)
147    }
148
149    fn eat_while(&mut self, keep: impl Fn(char) -> bool) {
150        while self.peek().is_some_and(&keep) {
151            self.bump();
152        }
153    }
154
155    fn skip_line(&mut self) {
156        self.eat_while(|c| c != '\n');
157    }
158
159    /// Consumes a quoted string starting at the opening quote, honouring backslash escapes.
160    fn quoted(&mut self, quote: char) {
161        self.bump();
162        while let Some(c) = self.bump() {
163            if c == '\\' {
164                self.bump();
165            } else if c == quote {
166                break;
167            }
168        }
169    }
170}
171
172fn is_ident_start(c: char) -> bool {
173    c.is_alphabetic() || c == '_'
174}
175
176fn is_ident(c: char) -> bool {
177    c.is_alphanumeric() || c == '_'
178}
179
180fn rust(code: &str) -> Vec<(Range<usize>, Token)> {
181    let mut s = Scanner { text: code, pos: 0 };
182    let mut out = Vec::new();
183    let mut previous_word = String::new();
184    while let Some(c) = s.peek() {
185        let start = s.pos;
186        if s.starts_with("//") {
187            s.skip_line();
188            out.push((start..s.pos, Token::Comment));
189        } else if s.starts_with("/*") {
190            match code[s.pos + 2..].find("*/") {
191                Some(end) => s.pos += 2 + end + 2,
192                None => s.pos = code.len(),
193            }
194            out.push((start..s.pos, Token::Comment));
195        } else if s.starts_with("#[") || s.starts_with("#![") {
196            let mut depth = 0;
197            while let Some(c) = s.bump() {
198                match c {
199                    '[' => depth += 1,
200                    ']' => {
201                        depth -= 1;
202                        if depth == 0 {
203                            break;
204                        }
205                    }
206                    '\n' => break,
207                    _ => {}
208                }
209            }
210            out.push((start..s.pos, Token::Attribute));
211        } else if c == '"' || ((c == 'b') && s.peek_at(1) == Some('"')) {
212            if c == 'b' {
213                s.bump();
214            }
215            s.quoted('"');
216            out.push((start..s.pos, Token::String));
217        } else if c == 'r' && (s.peek_at(1) == Some('"') || (s.peek_at(1) == Some('#') && s.peek_at(2) != Some('['))) {
218            s.bump();
219            let mut hashes = 0;
220            while s.peek() == Some('#') {
221                s.bump();
222                hashes += 1;
223            }
224            if s.peek() == Some('"') {
225                let closing = format!("\"{}", "#".repeat(hashes));
226                s.bump();
227                match code[s.pos..].find(&closing) {
228                    Some(end) => s.pos += end + closing.len(),
229                    None => s.pos = code.len(),
230                }
231                out.push((start..s.pos, Token::String));
232            } else {
233                s.eat_while(is_ident);
234            }
235        } else if c == '\'' {
236            let is_char = s.peek_at(1) == Some('\\') || s.peek_at(2) == Some('\'');
237            if is_char {
238                s.quoted('\'');
239                out.push((start..s.pos, Token::String));
240            } else {
241                s.bump();
242                s.eat_while(is_ident);
243                out.push((start..s.pos, Token::Lifetime));
244            }
245        } else if c.is_ascii_digit() {
246            s.eat_while(|c| c.is_ascii_alphanumeric() || c == '_' || c == '.');
247            if code[start..s.pos].ends_with('.') {
248                s.pos -= 1;
249            }
250            out.push((start..s.pos, Token::Number));
251        } else if is_ident_start(c) {
252            s.eat_while(is_ident);
253            let word = &code[start..s.pos];
254            let token = if RUST_KEYWORDS.contains(&word) {
255                Token::Keyword
256            } else if s.peek() == Some('!') && s.peek_at(1) != Some('=') {
257                s.bump();
258                Token::Macro
259            } else if previous_word == "fn" || s.peek() == Some('(') || s.starts_with("::<") {
260                Token::Function
261            } else if word.chars().next().is_some_and(char::is_uppercase) {
262                Token::Type
263            } else {
264                Token::Plain
265            };
266            previous_word = word.to_owned();
267            out.push((start..s.pos, token));
268            continue;
269        } else if c.is_whitespace() {
270            s.bump();
271            continue;
272        } else {
273            s.bump();
274            out.push((start..s.pos, Token::Punctuation));
275        }
276        previous_word.clear();
277    }
278    out
279}
280
281fn toml(code: &str) -> Vec<(Range<usize>, Token)> {
282    let mut out = Vec::new();
283    let mut line_start = 0;
284    for line in code.split_inclusive('\n') {
285        let mut s = Scanner { text: code, pos: line_start };
286        let end = line_start + line.len();
287        s.eat_while(|c| c == ' ' || c == '\t');
288        if s.peek() == Some('[') {
289            let header_start = s.pos;
290            while s.pos < end && s.peek() != Some(']') {
291                if s.peek() == Some('"') {
292                    s.quoted('"');
293                } else {
294                    s.bump();
295                }
296            }
297            s.eat_while(|c| c == ']');
298            out.push((header_start..s.pos, Token::Table));
299        } else if s.peek().is_some_and(|c| is_ident(c) || c == '"' || c == '-') {
300            let key_start = s.pos;
301            while s.pos < end && !matches!(s.peek(), Some('=') | Some('\n') | Some('#')) {
302                if s.peek() == Some('"') {
303                    s.quoted('"');
304                } else {
305                    s.bump();
306                }
307            }
308            let key_end = key_start + code[key_start..s.pos].trim_end().len();
309            if s.peek() == Some('=') {
310                out.push((key_start..key_end, Token::Key));
311            } else {
312                s.pos = key_start;
313            }
314        }
315        while s.pos < end {
316            let start = s.pos;
317            match s.peek() {
318                Some('#') => {
319                    s.skip_line();
320                    out.push((start..s.pos, Token::Comment));
321                }
322                Some(q @ ('"' | '\'')) => {
323                    s.quoted(q);
324                    out.push((start..s.pos.min(end), Token::String));
325                    s.pos = s.pos.min(end);
326                }
327                Some(c) if c.is_ascii_digit() || (c == '-' && s.peek_at(1).is_some_and(|d| d.is_ascii_digit())) => {
328                    s.bump();
329                    s.eat_while(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '.' | ':' | '-'));
330                    out.push((start..s.pos, Token::Number));
331                }
332                Some(c) if is_ident_start(c) => {
333                    s.eat_while(is_ident);
334                    let token = match &code[start..s.pos] {
335                        "true" | "false" => Token::Keyword,
336                        _ => Token::Plain,
337                    };
338                    out.push((start..s.pos, token));
339                }
340                Some(c) if c.is_whitespace() => {
341                    s.bump();
342                }
343                Some(_) => {
344                    s.bump();
345                    out.push((start..s.pos, Token::Punctuation));
346                }
347                None => break,
348            }
349        }
350        line_start = end;
351    }
352    out
353}
354
355#[cfg(test)]
356mod tests {
357    use super::*;
358
359    fn kinds(code: &str, language: Language) -> Vec<(&str, Token)> {
360        highlight(code, language)
361            .into_iter()
362            .filter(|(r, t)| *t != Token::Plain || !code[r.clone()].trim().is_empty())
363            .map(|(r, t)| (&code[r], t))
364            .collect()
365    }
366
367    #[test]
368    fn rust_tokens() {
369        let code = "#[derive(Debug)]\nfn view(&self, ui: &mut View<'_, Msg>) {\n    ui.add(Text::new(t!(\"hi\")) // note\n        .width(3));\n}";
370        let tokens = kinds(code, Language::Rust);
371        assert!(tokens.contains(&("#[derive(Debug)]", Token::Attribute)));
372        assert!(tokens.contains(&("fn", Token::Keyword)));
373        assert!(tokens.contains(&("view", Token::Function)));
374        assert!(tokens.contains(&("View", Token::Type)));
375        assert!(tokens.contains(&("'_", Token::Lifetime)));
376        assert!(tokens.contains(&("t!", Token::Macro)));
377        assert!(tokens.contains(&("\"hi\"", Token::String)));
378        assert!(tokens.contains(&("// note", Token::Comment)));
379        assert!(tokens.contains(&("3", Token::Number)));
380        assert!(tokens.contains(&("new", Token::Function)));
381    }
382
383    #[test]
384    fn toml_tokens() {
385        let code = "[style.\"button:hover\"]\nbg = \"$raised\" # surface\npadding = [0, 2]\nslide = true\n";
386        let tokens = kinds(code, Language::Toml);
387        assert!(tokens.contains(&("[style.\"button:hover\"]", Token::Table)));
388        assert!(tokens.contains(&("bg", Token::Key)));
389        assert!(tokens.contains(&("\"$raised\"", Token::String)));
390        assert!(tokens.contains(&("# surface", Token::Comment)));
391        assert!(tokens.contains(&("2", Token::Number)));
392        assert!(tokens.contains(&("true", Token::Keyword)));
393    }
394
395    #[test]
396    fn ranges_cover_the_whole_text() {
397        let code = "let x = 'a'; r#\"raw\"#";
398        let ranges = highlight(code, Language::Rust);
399        let joined: String = ranges.iter().map(|(r, _)| &code[r.clone()]).collect();
400        assert_eq!(joined, code);
401        assert!(ranges.iter().any(|(r, t)| &code[r.clone()] == "r#\"raw\"#" && *t == Token::String));
402    }
403
404    #[test]
405    fn shell_tokens() {
406        let code = "# Maintainer: someone\npkgname=hello\npkgrel=1\nbuild() {\n  cd \"$srcdir/${pkgname}-$pkgver\"\n  if [ -f x ]; then echo 'it is $here'; fi\n  local n=$(nproc) # cores\n  cat <<-'EOF' > notes\n\tread $me\n\tEOF\n  echo \\$done \"a\\\"b\"\n}";
407        let tokens = kinds(code, Language::Shell);
408        assert!(tokens.contains(&("# Maintainer: someone", Token::Comment)), "{tokens:?}");
409        assert!(tokens.contains(&("pkgname", Token::Variable)));
410        assert!(tokens.contains(&("1", Token::Number)));
411        assert!(tokens.contains(&("build", Token::Function)));
412        assert!(tokens.contains(&("\"", Token::String)));
413        assert!(tokens.contains(&("$srcdir", Token::Variable)));
414        assert!(tokens.contains(&("${pkgname}", Token::Variable)));
415        assert!(tokens.contains(&("$pkgver", Token::Variable)));
416        assert!(tokens.contains(&("/", Token::String)), "the text between variables stays a string");
417        assert!(tokens.contains(&("if", Token::Keyword)));
418        assert!(tokens.contains(&("then", Token::Keyword)));
419        assert!(tokens.contains(&("fi", Token::Keyword)));
420        assert!(tokens.contains(&("'it is $here'", Token::String)), "single quotes expand nothing");
421        assert!(tokens.contains(&("local", Token::Keyword)));
422        assert!(tokens.contains(&("$(", Token::Variable)));
423        assert!(tokens.contains(&("nproc", Token::Plain)));
424        assert!(tokens.contains(&(")", Token::Variable)));
425        assert!(tokens.contains(&("# cores", Token::Comment)));
426        assert!(tokens.contains(&("'EOF'", Token::String)));
427        assert!(tokens.contains(&("\tread $me\n\tEOF", Token::String)), "the heredoc body runs to its terminator");
428        assert!(!tokens.iter().any(|(text, token)| *text == "$done" && *token == Token::Variable), "an escaped dollar");
429        assert!(tokens.contains(&("\"a\\\"b\"", Token::String)), "an escaped quote stays inside");
430        assert!(tokens.contains(&("}", Token::Punctuation)));
431    }
432
433    #[test]
434    fn shell_details() {
435        let tokens = kinds(
436            "function greet {\n  echo ${#names[@]} $1 $@ x#y\n}\ncase $a in\n  *) exit 2 ;;\nesac",
437            Language::Shell,
438        );
439        assert!(tokens.contains(&("greet", Token::Function)), "{tokens:?}");
440        assert!(tokens.contains(&("function", Token::Keyword)));
441        assert!(tokens.contains(&("${#names[@]}", Token::Variable)));
442        assert!(tokens.contains(&("$1", Token::Variable)));
443        assert!(tokens.contains(&("$@", Token::Variable)));
444        assert!(!tokens.iter().any(|(_, token)| *token == Token::Comment), "a hash inside a word is not a comment");
445        assert!(tokens.contains(&("case", Token::Keyword)));
446        assert!(tokens.contains(&("in", Token::Keyword)));
447        assert!(tokens.contains(&("esac", Token::Keyword)));
448        assert!(tokens.contains(&("2", Token::Number)));
449        let unquoted = kinds("cat <<EOF\nhi $USER\nEOF\necho done", Language::Shell);
450        assert!(unquoted.contains(&("hi $USER\nEOF", Token::String)), "{unquoted:?}");
451        assert!(unquoted.contains(&("done", Token::Keyword)), "code resumes after the terminator");
452        let arithmetic = kinds("n=$((a + 1))", Language::Shell);
453        assert!(arithmetic.contains(&("$(", Token::Variable)), "{arithmetic:?}");
454        assert!(arithmetic.contains(&("(", Token::Punctuation)));
455        assert_eq!(arithmetic.iter().filter(|(text, token)| *text == ")" && *token == Token::Variable).count(), 1);
456    }
457
458    #[test]
459    fn shell_ranges_cover_the_whole_text() {
460        let code = "x=\"unterminated $(echo \"in\") ${a:-b}\ncat <<E\nno end";
461        let joined: String = highlight(code, Language::Shell).iter().map(|(r, _)| &code[r.clone()]).collect();
462        assert_eq!(joined, code);
463    }
464
465    #[test]
466    fn languages_from_tags_and_file_names() {
467        for tag in ["sh", "bash", "shell", "zsh", "PKGBUILD"] {
468            assert_eq!(Language::from_tag(tag), Language::Shell, "{tag}");
469        }
470        assert_eq!(Language::from_file_name("PKGBUILD"), Language::Shell);
471        assert_eq!(Language::from_file_name("aur/hello/hello.install"), Language::Shell);
472        assert_eq!(Language::from_file_name("build.sh"), Language::Shell);
473        assert_eq!(Language::from_file_name("main.rs"), Language::Rust);
474        assert_eq!(Language::from_file_name("Cargo.toml"), Language::Toml);
475        assert_eq!(Language::from_file_name("README"), Language::Plain);
476    }
477}
qframe/widgets/highlight/mod.rs

qframe/widgets/highlight/
mod.rs
