termesh_syntax/
lib.rs

1//! Tree-sitter highlighting (ARCHITECTURE.md Appendix A).
2//!
3//! Produces plain `(start, end, SyntaxKind)` spans in **char** offsets, which the editor
4//! turns into decorations. Nothing tree-sitter crosses this boundary, so adding a
5//! language is a table entry here and touches nothing above.
6//!
7//! **Reparses whole.** ARCHITECTURE.md §8 wants incremental parsing fed by the change
8//! stream, and the transaction spine already carries everything needed to do it — the
9//! missing piece is keeping the `Tree` and calling `Tree::edit` with each change. A full
10//! reparse of an ordinary source file is well under a millisecond and happens on edit
11//! rather than on render, so this is a real but bounded shortcut, and the seam for fixing
12//! it is [`Highlighter::highlight`].
13#![forbid(unsafe_code)]
14
15use std::path::Path;
16
17use termesh_editor::SyntaxKind;
18use tree_sitter_highlight::{
19    Highlight, HighlightConfiguration, HighlightEvent, Highlighter as TsHighlighter,
20};
21
22/// A language we can highlight.
23#[derive(Debug, Clone, Copy, PartialEq, Eq)]
24pub enum Language {
25    Rust,
26}
27
28impl Language {
29    /// The language for a file, by extension.
30    ///
31    /// Rust only for now: ARCHITECTURE.md §14 makes Rust the flagship and says more
32    /// languages arrive as recipes, so this is the table those recipes extend.
33    pub fn from_path(path: &Path) -> Option<Self> {
34        match path.extension()?.to_str()? {
35            "rs" => Some(Language::Rust),
36            _ => None,
37        }
38    }
39}
40
41/// The capture names we ask tree-sitter for, in the order their indices are assigned.
42///
43/// Deliberately short. A highlighter with forty token classes needs a theme with forty
44/// colours, and a terminal has far fewer than that to spend legibly.
45const CAPTURES: &[(&str, SyntaxKind)] = &[
46    ("keyword", SyntaxKind::Keyword),
47    ("string", SyntaxKind::StringLit),
48    ("comment", SyntaxKind::Comment),
49    ("number", SyntaxKind::Number),
50    ("type", SyntaxKind::Type),
51    ("function", SyntaxKind::Function),
52    // Aliases the Rust queries actually emit, folded onto the same kinds.
53    ("constructor", SyntaxKind::Type),
54    ("type.builtin", SyntaxKind::Type),
55    ("function.method", SyntaxKind::Function),
56    ("function.macro", SyntaxKind::Function),
57    ("constant", SyntaxKind::Number),
58    ("constant.builtin", SyntaxKind::Number),
59    ("escape", SyntaxKind::StringLit),
60];
61
62/// A highlighted span, in char offsets.
63pub type Span = (usize, usize, SyntaxKind);
64
65/// Parses and highlights one language.
66pub struct Highlighter {
67    inner: TsHighlighter,
68    config: HighlightConfiguration,
69}
70
71impl Highlighter {
72    /// Build a highlighter, or `None` if the grammar and its queries disagree — a
73    /// mismatch between the grammar crate and its query file is a packaging problem, and
74    /// an editor that refuses to open a file over it would be worse than one that shows
75    /// it unhighlighted.
76    pub fn new(language: Language) -> Option<Self> {
77        let names: Vec<&str> = CAPTURES.iter().map(|(name, _)| *name).collect();
78
79        let mut config = match language {
80            Language::Rust => HighlightConfiguration::new(
81                tree_sitter_rust::LANGUAGE.into(),
82                "rust",
83                tree_sitter_rust::HIGHLIGHTS_QUERY,
84                "",
85                "",
86            )
87            .ok()?,
88        };
89        config.configure(&names);
90
91        Some(Self { inner: TsHighlighter::new(), config })
92    }
93
94    /// Highlight `text`.
95    ///
96    /// Returns spans in char offsets, non-overlapping and in document order. On any parse
97    /// failure the answer is "no highlighting" rather than an error: unhighlighted code is
98    /// perfectly editable, and half-coloured code from a partial parse is not better.
99    pub fn highlight(&mut self, text: &str) -> Vec<Span> {
100        let Ok(events) = self.inner.highlight(&self.config, text.as_bytes(), None, |_| None) else {
101            return Vec::new();
102        };
103
104        // tree-sitter works in bytes; everything above works in chars (ADR-0006 §1).
105        // Building the prefix table once beats counting per span.
106        let char_at = ByteToChar::new(text);
107
108        let mut spans = Vec::new();
109        let mut stack: Vec<Highlight> = Vec::new();
110        for event in events.flatten() {
111            match event {
112                HighlightEvent::HighlightStart(h) => stack.push(h),
113                HighlightEvent::HighlightEnd => {
114                    stack.pop();
115                }
116                HighlightEvent::Source { start, end } => {
117                    // The innermost capture wins, which is what nesting means.
118                    if let Some(kind) = stack.last().and_then(|h| kind_of(*h)) {
119                        if start < end {
120                            spans.push((char_at.get(start), char_at.get(end), kind));
121                        }
122                    }
123                }
124            }
125        }
126        spans
127    }
128}
129
130fn kind_of(highlight: Highlight) -> Option<SyntaxKind> {
131    CAPTURES.get(highlight.0).map(|(_, kind)| *kind)
132}
133
134/// Byte offset → char offset, precomputed.
135struct ByteToChar {
136    /// `chars[b]` is the number of chars before byte `b`.
137    chars: Vec<usize>,
138}
139
140impl ByteToChar {
141    fn new(text: &str) -> Self {
142        let mut chars = vec![0; text.len() + 1];
143        let mut count = 0;
144        for (byte, _) in text.char_indices() {
145            chars[byte] = count;
146            count += 1;
147        }
148        // Every trailing byte of a multi-byte char, and the end, map to the running total.
149        let mut last = 0;
150        for slot in chars.iter_mut() {
151            if *slot == 0 && last != 0 {
152                *slot = last;
153            } else {
154                last = *slot;
155            }
156        }
157        chars[text.len()] = count;
158        Self { chars }
159    }
160
161    fn get(&self, byte: usize) -> usize {
162        self.chars.get(byte).copied().unwrap_or_else(|| self.chars.last().copied().unwrap_or(0))
163    }
164}
165
166#[cfg(test)]
167mod tests {
168    use super::*;
169
170    fn highlight(source: &str) -> Vec<Span> {
171        Highlighter::new(Language::Rust).expect("the Rust grammar loads").highlight(source)
172    }
173
174    /// The span for the first occurrence of `text`, if it was highlighted.
175    fn kind_of_word(source: &str, word: &str) -> Option<SyntaxKind> {
176        let at = source.find(word).expect("the word is in the source");
177        let start = source[..at].chars().count();
178        highlight(source).into_iter().find(|(s, e, _)| *s <= start && start < *e).map(|(_, _, k)| k)
179    }
180
181    #[test]
182    fn the_rust_grammar_loads() {
183        assert!(Highlighter::new(Language::Rust).is_some());
184    }
185
186    #[test]
187    fn a_language_is_chosen_by_extension() {
188        assert_eq!(Language::from_path(Path::new("src/main.rs")), Some(Language::Rust));
189        assert_eq!(Language::from_path(Path::new("README.md")), None);
190        assert_eq!(Language::from_path(Path::new("noextension")), None);
191    }
192
193    #[test]
194    fn keywords_comments_and_strings_are_distinguished() {
195        let source = "// a note\nfn main() {\n    let s = \"hello\";\n}\n";
196        assert_eq!(kind_of_word(source, "// a note"), Some(SyntaxKind::Comment));
197        assert_eq!(kind_of_word(source, "fn"), Some(SyntaxKind::Keyword));
198        assert_eq!(kind_of_word(source, "\"hello\""), Some(SyntaxKind::StringLit));
199    }
200
201    #[test]
202    fn numbers_are_highlighted() {
203        assert_eq!(kind_of_word("fn f() { let x = 42; }", "42"), Some(SyntaxKind::Number));
204    }
205
206    #[test]
207    fn spans_are_char_offsets_not_byte_offsets() {
208        // The comment holds a multi-byte character, so every span after it would be
209        // wrong if these were byte offsets.
210        let source = "// héllo\nfn main() {}\n";
211        let at = source.find("fn").unwrap();
212        assert_ne!(at, source[..at].chars().count(), "the test is only meaningful if they differ");
213
214        let start = source[..at].chars().count();
215        assert!(
216            highlight(source).iter().any(|(s, _, k)| *s == start && *k == SyntaxKind::Keyword),
217            "`fn` should be highlighted at its char offset"
218        );
219    }
220
221    #[test]
222    fn spans_never_run_past_the_end_of_the_text() {
223        let source = "fn main() {}\n";
224        let chars = source.chars().count();
225        assert!(highlight(source).iter().all(|(_, end, _)| *end <= chars));
226    }
227
228    #[test]
229    fn spans_come_back_in_document_order() {
230        let spans = highlight("// one\nfn two() {}\n// three\n");
231        assert!(spans.windows(2).all(|w| w[0].0 <= w[1].0), "got {spans:?}");
232    }
233
234    #[test]
235    fn empty_and_broken_input_produce_no_highlighting_rather_than_an_error() {
236        assert!(highlight("").is_empty());
237        // Unparseable code is still editable; half-coloured is not better than plain.
238        let _ = highlight("fn fn fn ((( unclosed");
239    }
240}
termesh_syntax/lib.rs

termesh_syntax/
lib.rs
