clankerdiff_syntax/
highlight.rs

1//! Tree-sitter syntax highlighting with UTF-8 byte spans and a bounded cache.
2
3use crate::language::{LanguageHint, resolve_language};
4use arborium::{Config, Highlighter};
5use arborium_highlight::spans_to_flat_tokens;
6use arborium_theme::tag_to_name;
7use clankerdiff_fingerprint::SourceSequenceId;
8use clankerdiff_theme::{Fingerprint, HighlightSpan, SyntaxTheme};
9use lru::LruCache;
10use std::{
11    fmt,
12    num::NonZeroUsize,
13    ops::Range,
14    sync::{Arc, OnceLock},
15};
16const DEFAULT_CAPACITY: usize = 512;
17const DEFAULT_MAX_DOCUMENTS: usize = 32;
18const DEFAULT_STREAM_BYTES: usize = 8 * 1024 * 1024;
19const SOURCE_KEY_DOMAIN: &[u8] = b"syntax-source-v1";
20const DOCUMENT_KEY_DOMAIN: &[u8] = b"syntax-document-v1";
21
22/// A shared empty span set, for text with nothing to highlight.
23#[must_use]
24pub fn empty_spans() -> Arc<[HighlightSpan]> {
25    static EMPTY: OnceLock<Arc<[HighlightSpan]>> = OnceLock::new();
26    Arc::clone(EMPTY.get_or_init(|| Arc::from(Vec::<HighlightSpan>::new())))
27}
28
29/// Counters useful for measuring highlighting and cache behavior.
30#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
31pub struct HighlightStats {
32    pub calls: u64,
33    pub hits: u64,
34    pub misses: u64,
35    pub evictions: u64,
36    /// Bytes actually supplied to Tree-sitter parsers.
37    pub bytes: usize,
38}
39
40/// Opaque syntax-cache key with a stable diagnostic fingerprint.
41#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
42pub struct CacheKey {
43    fingerprint: Fingerprint,
44}
45
46impl CacheKey {
47    #[must_use]
48    pub const fn fingerprint(self) -> Fingerprint {
49        self.fingerprint
50    }
51
52    fn source(theme: Fingerprint, language: &str, source: &str) -> Self {
53        Self::new([
54            SOURCE_KEY_DOMAIN,
55            theme.as_bytes().as_slice(),
56            language.as_bytes(),
57            source.as_bytes(),
58        ])
59    }
60
61    fn document(theme: Fingerprint, language: &str, sequence: SourceSequenceId) -> Self {
62        let sequence = Fingerprint::from(sequence);
63        Self::new([
64            DOCUMENT_KEY_DOMAIN,
65            theme.as_bytes().as_slice(),
66            language.as_bytes(),
67            sequence.as_bytes().as_slice(),
68        ])
69    }
70
71    fn new<const N: usize>(fields: [&[u8]; N]) -> Self {
72        Self {
73            fingerprint: Fingerprint::of(fields),
74        }
75    }
76}
77
78/// Fixed resource limits for a [`SyntaxHighlighter`].
79#[derive(Debug, Clone, Copy, PartialEq, Eq)]
80pub struct CacheConfig {
81    pub max_entries: usize,
82    pub max_documents: usize,
83    pub max_stream_bytes: usize,
84}
85
86impl Default for CacheConfig {
87    fn default() -> Self {
88        Self {
89            max_entries: DEFAULT_CAPACITY,
90            max_documents: DEFAULT_MAX_DOCUMENTS,
91            max_stream_bytes: DEFAULT_STREAM_BYTES,
92        }
93    }
94}
95
96impl From<usize> for CacheConfig {
97    fn from(max_entries: usize) -> Self {
98        Self {
99            max_entries,
100            ..Self::default()
101        }
102    }
103}
104
105#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
106pub struct CacheUsage {
107    pub span_entries: usize,
108    pub document_entries: usize,
109}
110
111#[derive(Debug, Clone, Default)]
112pub struct SyntaxStream {
113    hint: String,
114    source: String,
115    revision: u64,
116    theme_revision: Option<Fingerprint>,
117    highlights: Arc<DocumentHighlights>,
118}
119
120#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
121pub enum SyntaxStreamError {
122    #[error("syntax stream input requires {attempted} bytes, exceeding the {limit}-byte limit")]
123    InputLimit { limit: usize, attempted: usize },
124}
125
126#[derive(Debug, Clone)]
127pub struct SyntaxStreamUpdate {
128    pub base_revision: u64,
129    pub revision: u64,
130    pub changed_lines: Range<usize>,
131    pub highlights: Arc<DocumentHighlights>,
132}
133
134impl SyntaxStream {
135    #[must_use]
136    pub fn new<'a>(hint: impl Into<LanguageHint<'a>>) -> Self {
137        Self {
138            hint: hint.into().as_str().to_owned(),
139            ..Self::default()
140        }
141    }
142
143    #[must_use]
144    pub fn source(&self) -> &str {
145        &self.source
146    }
147
148    #[must_use]
149    pub const fn revision(&self) -> u64 {
150        self.revision
151    }
152
153    #[must_use]
154    pub fn highlights(&self) -> &Arc<DocumentHighlights> {
155        &self.highlights
156    }
157}
158
159/// Highlight spans projected onto every source line of one parsed document.
160#[derive(Debug, Clone, Default)]
161pub struct DocumentHighlights {
162    lines: Vec<Arc<[HighlightSpan]>>,
163}
164
165impl DocumentHighlights {
166    #[must_use]
167    pub fn line(&self, index: usize) -> Option<&[HighlightSpan]> {
168        self.lines.get(index).map(AsRef::as_ref)
169    }
170
171    #[must_use]
172    pub fn line_shared(&self, index: usize) -> Option<Arc<[HighlightSpan]>> {
173        self.lines.get(index).cloned()
174    }
175
176    #[must_use]
177    pub fn line_count(&self) -> usize {
178        self.lines.len()
179    }
180
181    fn from_spans(spans: &[HighlightSpan], text: &str) -> Self {
182        let starts = if text.is_empty() {
183            Vec::new()
184        } else {
185            let mut starts = vec![0];
186            starts.extend(
187                text.bytes()
188                    .enumerate()
189                    .filter_map(|(index, byte)| (byte == b'\n').then_some(index + 1)),
190            );
191            if text.ends_with('\n') {
192                starts.pop();
193            }
194            starts
195        };
196        let mut lines = Vec::with_capacity(starts.len());
197        let mut first_span = 0;
198        for (line, &start) in starts.iter().enumerate() {
199            let next = starts.get(line + 1).copied().unwrap_or(text.len());
200            let mut end = next;
201            if end > start && text.as_bytes()[end - 1] == b'\n' {
202                end -= 1;
203            }
204            if end > start && text.as_bytes()[end - 1] == b'\r' {
205                end -= 1;
206            }
207            while spans
208                .get(first_span)
209                .is_some_and(|span| span.range.end <= start)
210            {
211                first_span += 1;
212            }
213            let projected = spans[first_span..]
214                .iter()
215                .take_while(|span| span.range.start < end)
216                .filter_map(|span| {
217                    let from = span.range.start.max(start);
218                    let to = span.range.end.min(end);
219                    (from < to).then_some(HighlightSpan {
220                        range: from - start..to - start,
221                        foreground: span.foreground,
222                        font_style: span.font_style,
223                    })
224                })
225                .collect::<Vec<_>>();
226            lines.push(Arc::from(projected));
227        }
228        Self { lines }
229    }
230}
231
232/// A reusable syntax highlighter. Least-recently-used entries are evicted when full.
233pub struct SyntaxHighlighter {
234    highlighter: Highlighter,
235    config: CacheConfig,
236    cache: Option<LruCache<CacheKey, Arc<[HighlightSpan]>>>,
237    documents: Option<LruCache<CacheKey, Arc<DocumentHighlights>>>,
238    stats: HighlightStats,
239}
240
241impl fmt::Debug for SyntaxHighlighter {
242    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
243        f.debug_struct("SyntaxHighlighter")
244            .field("config", &self.config)
245            .field("entries", &self.cache.as_ref().map_or(0, LruCache::len))
246            .field(
247                "documents",
248                &self.documents.as_ref().map_or(0, LruCache::len),
249            )
250            .field("stats", &self.stats)
251            .finish_non_exhaustive()
252    }
253}
254
255impl Default for SyntaxHighlighter {
256    fn default() -> Self {
257        Self::new(DEFAULT_CAPACITY)
258    }
259}
260
261impl SyntaxHighlighter {
262    /// Creates a highlighter with fixed cache resource limits.
263    #[must_use]
264    pub fn new(config: impl Into<CacheConfig>) -> Self {
265        let syntax_config = Config {
266            max_injection_depth: 3,
267            ..Config::default()
268        };
269        let config = config.into();
270        let document_entries = if config.max_entries == 0 {
271            0
272        } else {
273            config.max_documents
274        };
275        Self {
276            highlighter: Highlighter::with_config(syntax_config),
277            config,
278            cache: NonZeroUsize::new(config.max_entries).map(LruCache::new),
279            documents: NonZeroUsize::new(document_entries).map(LruCache::new),
280            stats: HighlightStats::default(),
281        }
282    }
283
284    #[must_use]
285    pub const fn stats(&self) -> HighlightStats {
286        self.stats
287    }
288
289    pub fn reset_stats(&mut self) {
290        self.stats = HighlightStats::default();
291    }
292
293    /// Atomically returns all counters accumulated so far and resets them.
294    pub fn take_stats(&mut self) -> HighlightStats {
295        std::mem::take(&mut self.stats)
296    }
297
298    #[must_use]
299    pub const fn config(&self) -> CacheConfig {
300        self.config
301    }
302
303    #[must_use]
304    pub fn with_theme<'a>(&'a mut self, theme: &'a SyntaxTheme) -> ThemedHighlighter<'a> {
305        ThemedHighlighter {
306            highlighter: self,
307            theme,
308        }
309    }
310
311    #[must_use]
312    pub fn cache_usage(&self) -> CacheUsage {
313        CacheUsage {
314            span_entries: self.cache.as_ref().map_or(0, LruCache::len),
315            document_entries: self.documents.as_ref().map_or(0, LruCache::len),
316        }
317    }
318
319    pub fn clear_cache(&mut self) {
320        if let Some(cache) = &mut self.cache {
321            cache.clear();
322        }
323        if let Some(documents) = &mut self.documents {
324            documents.clear();
325        }
326    }
327
328    fn highlight_source(
329        &mut self,
330        theme: &SyntaxTheme,
331        hint: LanguageHint<'_>,
332        text: &str,
333    ) -> Arc<[HighlightSpan]> {
334        self.stats.calls += 1;
335        let language = resolve_language(hint, text);
336        let id = language.unwrap_or("plain");
337        let key = CacheKey::source(theme.revision(), id, text);
338        if let Some(spans) = self.cache.as_mut().and_then(|cache| cache.get(&key)) {
339            self.stats.hits += 1;
340            return Arc::clone(spans);
341        }
342        self.stats.misses += 1;
343        let Some(language) = language else {
344            let spans = empty_spans();
345            self.store(key, Arc::clone(&spans));
346            return spans;
347        };
348        self.stats.bytes = self.stats.bytes.saturating_add(text.len());
349        let spans = highlight_source(&mut self.highlighter, theme, language, text)
350            .map_or_else(empty_spans, Arc::from);
351        self.store(key, Arc::clone(&spans));
352        spans
353    }
354
355    fn highlight_lines<'line, T>(
356        &mut self,
357        theme: &SyntaxTheme,
358        hint: LanguageHint<'_>,
359        lines: T,
360    ) -> Vec<Vec<HighlightSpan>>
361    where
362        T: IntoIterator<Item = &'line str>,
363    {
364        self.stats.calls += 1;
365        let selected: Vec<(usize, &str)> = lines.into_iter().enumerate().collect();
366        let window = JoinedLines::new(selected);
367        let Some(language) = resolve_language(hint, &window.source) else {
368            return vec![Vec::new(); window.lines.len()];
369        };
370        self.stats.bytes = self.stats.bytes.saturating_add(window.source.len());
371        highlight_source(&mut self.highlighter, theme, language, &window.source)
372            .as_deref()
373            .map_or_else(
374                || vec![Vec::new(); window.lines.len()],
375                |spans| window.split(spans),
376            )
377    }
378
379    fn store(&mut self, key: CacheKey, spans: Arc<[HighlightSpan]>) {
380        if let Some(cache) = &mut self.cache
381            && let Some((evicted, _)) = cache.push(key, spans)
382            && evicted != key
383        {
384            self.stats.evictions += 1;
385        }
386    }
387
388    fn store_document(&mut self, key: CacheKey, highlights: Arc<DocumentHighlights>) {
389        if let Some(documents) = &mut self.documents
390            && let Some((evicted, _)) = documents.push(key, highlights)
391            && evicted != key
392        {
393            self.stats.evictions += 1;
394        }
395    }
396}
397
398/// Theme-bound highlighting operations.
399pub struct ThemedHighlighter<'a> {
400    highlighter: &'a mut SyntaxHighlighter,
401    theme: &'a SyntaxTheme,
402}
403
404impl ThemedHighlighter<'_> {
405    /// Parses a complete source on first use and caches line-projected spans.
406    pub fn highlight_document<'a>(
407        &mut self,
408        sequence: SourceSequenceId,
409        language: impl Into<LanguageHint<'a>>,
410        text: &str,
411    ) -> Arc<DocumentHighlights> {
412        self.highlighter.stats.calls += 1;
413        let resolved = resolve_language(language.into(), text);
414        let key = CacheKey::document(self.theme.revision(), resolved.unwrap_or("plain"), sequence);
415        if let Some(highlights) = self
416            .highlighter
417            .documents
418            .as_mut()
419            .and_then(|documents| documents.get(&key))
420        {
421            self.highlighter.stats.hits += 1;
422            return Arc::clone(highlights);
423        }
424        self.parse_document(key, resolved, text)
425    }
426
427    /// Parses a line sequence as one complete document on first use and caches
428    /// line-projected spans. The lines are only joined and parsed on a miss.
429    pub fn highlight_document_lines<'a, 'line>(
430        &mut self,
431        sequence: SourceSequenceId,
432        language: impl Into<LanguageHint<'a>>,
433        lines: impl IntoIterator<Item = &'line str>,
434    ) -> Arc<DocumentHighlights> {
435        self.highlighter.stats.calls += 1;
436        let mut lines = lines.into_iter().peekable();
437        let resolved = resolve_language(language.into(), lines.peek().copied().unwrap_or_default());
438        let key = CacheKey::document(self.theme.revision(), resolved.unwrap_or("plain"), sequence);
439        if let Some(highlights) = self
440            .highlighter
441            .documents
442            .as_mut()
443            .and_then(|documents| documents.get(&key))
444        {
445            self.highlighter.stats.hits += 1;
446            return Arc::clone(highlights);
447        }
448        let mut text = String::new();
449        for line in lines {
450            text.push_str(line);
451            text.push('\n');
452        }
453        self.parse_document(key, resolved, &text)
454    }
455
456    fn parse_document(
457        &mut self,
458        key: CacheKey,
459        resolved: Option<&str>,
460        text: &str,
461    ) -> Arc<DocumentHighlights> {
462        self.highlighter.stats.misses += 1;
463        let highlights = Arc::new(self.project(resolved, text));
464        self.highlighter
465            .store_document(key, Arc::clone(&highlights));
466        highlights
467    }
468
469    fn project(&mut self, resolved: Option<&str>, text: &str) -> DocumentHighlights {
470        let spans = resolved
471            .and_then(|language| {
472                self.highlighter.stats.bytes += text.len();
473                highlight_source(
474                    &mut self.highlighter.highlighter,
475                    self.theme,
476                    language,
477                    text,
478                )
479            })
480            .unwrap_or_default();
481        DocumentHighlights::from_spans(&spans, text)
482    }
483
484    /// Highlights a complete source. Hints may be IDs, aliases, or repository paths.
485    pub fn highlight_source<'a>(
486        &mut self,
487        language: impl Into<LanguageHint<'a>>,
488        text: &str,
489    ) -> Arc<[HighlightSpan]> {
490        self.highlighter
491            .highlight_source(self.theme, language.into(), text)
492    }
493
494    /// Highlights all supplied lines in one parse, preserving multiline state.
495    pub fn highlight_lines<'line, 'hint, T>(
496        &mut self,
497        language: impl Into<LanguageHint<'hint>>,
498        lines: T,
499    ) -> Vec<Vec<HighlightSpan>>
500    where
501        T: IntoIterator<Item = &'line str>,
502    {
503        self.highlighter
504            .highlight_lines(self.theme, language.into(), lines)
505    }
506
507    pub fn append<'line>(
508        &mut self,
509        stream: &mut SyntaxStream,
510        lines: impl IntoIterator<Item = &'line str>,
511    ) -> Result<SyntaxStreamUpdate, SyntaxStreamError> {
512        let limit = self.highlighter.config.max_stream_bytes;
513        let mut appended = String::new();
514        for line in lines {
515            let newline = usize::from(!line.ends_with('\n'));
516            let attempted = stream.source.len() + appended.len() + line.len() + newline;
517            if attempted > limit {
518                return Err(SyntaxStreamError::InputLimit { limit, attempted });
519            }
520            appended.push_str(line);
521            if newline != 0 {
522                appended.push('\n');
523            }
524        }
525        let base_revision = stream.revision;
526        let theme_revision = self.theme.revision();
527        if appended.is_empty() && stream.theme_revision == Some(theme_revision) {
528            let end = stream.highlights.line_count();
529            return Ok(SyntaxStreamUpdate {
530                base_revision,
531                revision: stream.revision,
532                changed_lines: end..end,
533                highlights: Arc::clone(&stream.highlights),
534            });
535        }
536        if !appended.is_empty() {
537            stream.source.push_str(&appended);
538            stream.revision = stream.revision.wrapping_add(1);
539        }
540        self.highlighter.stats.calls += 1;
541        let resolved = resolve_language(stream.hint.as_str(), &stream.source);
542        let highlights = Arc::new(self.project(resolved, &stream.source));
543        let first_changed = stream
544            .highlights
545            .lines
546            .iter()
547            .zip(&highlights.lines)
548            .position(|(before, after)| before != after)
549            .unwrap_or_else(|| stream.highlights.line_count().min(highlights.line_count()));
550        stream.highlights = Arc::clone(&highlights);
551        stream.theme_revision = Some(theme_revision);
552        Ok(SyntaxStreamUpdate {
553            base_revision,
554            revision: stream.revision,
555            changed_lines: first_changed..highlights.line_count(),
556            highlights,
557        })
558    }
559}
560
561fn highlight_source(
562    highlighter: &mut Highlighter,
563    theme: &SyntaxTheme,
564    language: &str,
565    source: &str,
566) -> Option<Vec<HighlightSpan>> {
567    let raw_spans = highlighter.highlight_spans(language, source).ok()?;
568    let tokens = spans_to_flat_tokens(source, raw_spans);
569    let mut spans = Vec::with_capacity(tokens.len());
570    for token in tokens {
571        let Ok(start) = usize::try_from(token.start) else {
572            continue;
573        };
574        let Ok(end) = usize::try_from(token.end) else {
575            continue;
576        };
577        if start >= end
578            || end > source.len()
579            || !source.is_char_boundary(start)
580            || !source.is_char_boundary(end)
581        {
582            continue;
583        }
584        let Some(capture) = diff_capture_name(token.tag) else {
585            continue;
586        };
587        let Some(style) = theme.style(capture) else {
588            continue;
589        };
590        push_merged(
591            &mut spans,
592            HighlightSpan {
593                range: start..end,
594                foreground: style.foreground,
595                font_style: style.font_style,
596            },
597        );
598    }
599    Some(spans)
600}
601
602fn diff_capture_name(tag: &str) -> Option<&'static str> {
603    Some(match tag_to_name(tag)? {
604        "title" => "markup.heading",
605        "strong" => "markup.bold",
606        "emphasis" => "markup.italic",
607        "link" => "markup.link",
608        "literal" => "markup.raw",
609        "strikethrough" => "markup.strikethrough",
610        name => name,
611    })
612}
613
614fn push_merged(spans: &mut Vec<HighlightSpan>, span: HighlightSpan) {
615    if let Some(last) = spans.last_mut()
616        && last.range.end == span.range.start
617        && last.foreground == span.foreground
618        && last.font_style == span.font_style
619    {
620        last.range.end = span.range.end;
621    } else {
622        spans.push(span);
623    }
624}
625
626struct JoinedLines<'a> {
627    source: String,
628    /// Sequence index, original line, global display start, global display end.
629    lines: Vec<(usize, &'a str, usize, usize)>,
630}
631
632impl<'a> JoinedLines<'a> {
633    fn new(selected: Vec<(usize, &'a str)>) -> Self {
634        let mut source = String::new();
635        let mut lines = Vec::with_capacity(selected.len());
636        for (index, line) in selected {
637            let start = source.len();
638            source.push_str(line);
639            let end = source.len();
640            if !line.ends_with('\n') {
641                source.push('\n');
642            }
643            lines.push((index, line, start, end));
644        }
645        Self { source, lines }
646    }
647
648    /// Splits window-global spans into per-line local spans. Spans must be
649    /// disjoint and ordered, as [`highlight_source`] produces them, so one
650    /// forward sweep serves every line; a span crossing lines is revisited
651    /// only by the lines it overlaps.
652    fn split(&self, spans: &[HighlightSpan]) -> Vec<Vec<HighlightSpan>> {
653        let mut next = 0;
654        self.lines
655            .iter()
656            .map(|(_, line, start, end)| {
657                while spans.get(next).is_some_and(|span| span.range.end <= *start) {
658                    next += 1;
659                }
660                let mut result = Vec::new();
661                for span in &spans[next..] {
662                    if span.range.start >= *end {
663                        break;
664                    }
665                    let overlap_start = span.range.start.max(*start);
666                    let overlap_end = span.range.end.min(*end);
667                    if overlap_start >= overlap_end {
668                        continue;
669                    }
670                    let local_start = overlap_start - start;
671                    let local_end = overlap_end - start;
672                    if local_end <= line.len()
673                        && line.is_char_boundary(local_start)
674                        && line.is_char_boundary(local_end)
675                    {
676                        push_merged(
677                            &mut result,
678                            HighlightSpan {
679                                range: local_start..local_end,
680                                foreground: span.foreground,
681                                font_style: span.font_style,
682                            },
683                        );
684                    }
685                }
686                result
687            })
688            .collect()
689    }
690}
clankerdiff_syntax/highlight.rs

clankerdiff_syntax/
highlight.rs
