clankerdiff_syntax/
highlight.rs

1//! Tree-sitter syntax highlighting with UTF-8 byte spans and a bounded cache.
2
3use crate::language::{LanguageHint, resolve_language};
4use arborium::{Config, Highlighter};
5use arborium_highlight::spans_to_flat_tokens;
6use arborium_theme::tag_to_name;
7use clankerdiff_fingerprint::SourceSequenceId;
8use clankerdiff_theme::{DiffTheme, Fingerprint, HighlightSpan, SyntaxTheme};
9use std::{
10    collections::{HashMap, VecDeque},
11    fmt,
12    ops::Range,
13    sync::{Arc, OnceLock},
14};
15const DEFAULT_CAPACITY: usize = 512;
16const DEFAULT_MAX_DOCUMENTS: usize = 32;
17const DEFAULT_STREAM_BYTES: usize = 8 * 1024 * 1024;
18const SOURCE_KEY_DOMAIN: &[u8] = b"syntax-source-v1";
19const DOCUMENT_KEY_DOMAIN: &[u8] = b"syntax-document-v1";
20
21/// A shared empty span set, for text with nothing to highlight.
22#[must_use]
23pub fn empty_spans() -> Arc<[HighlightSpan]> {
24    static EMPTY: OnceLock<Arc<[HighlightSpan]>> = OnceLock::new();
25    Arc::clone(EMPTY.get_or_init(|| Arc::from(Vec::<HighlightSpan>::new())))
26}
27
28/// Counters useful for measuring highlighting and cache behavior.
29#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
30pub struct HighlightStats {
31    pub calls: u64,
32    pub hits: u64,
33    pub misses: u64,
34    pub evictions: u64,
35    /// Bytes actually supplied to Tree-sitter parsers.
36    pub bytes: usize,
37}
38
39/// Opaque syntax-cache key with a stable diagnostic fingerprint.
40#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
41pub struct CacheKey {
42    fingerprint: Fingerprint,
43}
44
45impl CacheKey {
46    #[must_use]
47    pub const fn fingerprint(self) -> Fingerprint {
48        self.fingerprint
49    }
50
51    fn source(theme: Fingerprint, language: &str, source: &str) -> Self {
52        Self::new([
53            SOURCE_KEY_DOMAIN,
54            theme.as_bytes().as_slice(),
55            language.as_bytes(),
56            source.as_bytes(),
57        ])
58    }
59
60    fn document(theme: Fingerprint, language: &str, sequence: SourceSequenceId) -> Self {
61        let sequence = Fingerprint::from(sequence);
62        Self::new([
63            DOCUMENT_KEY_DOMAIN,
64            theme.as_bytes().as_slice(),
65            language.as_bytes(),
66            sequence.as_bytes().as_slice(),
67        ])
68    }
69
70    fn new<const N: usize>(fields: [&[u8]; N]) -> Self {
71        Self {
72            fingerprint: Fingerprint::of(fields),
73        }
74    }
75}
76
77/// Fixed resource limits for a [`SyntaxHighlighter`].
78#[derive(Debug, Clone, Copy, PartialEq, Eq)]
79pub struct CacheConfig {
80    pub max_entries: usize,
81    pub max_documents: usize,
82    pub max_stream_bytes: usize,
83}
84
85impl Default for CacheConfig {
86    fn default() -> Self {
87        Self {
88            max_entries: DEFAULT_CAPACITY,
89            max_documents: DEFAULT_MAX_DOCUMENTS,
90            max_stream_bytes: DEFAULT_STREAM_BYTES,
91        }
92    }
93}
94
95impl From<usize> for CacheConfig {
96    fn from(max_entries: usize) -> Self {
97        Self {
98            max_entries,
99            ..Self::default()
100        }
101    }
102}
103
104#[derive(Debug, Clone, Default)]
105pub struct SyntaxStream {
106    hint: String,
107    source: String,
108    revision: u64,
109    theme_revision: Option<Fingerprint>,
110    highlights: Arc<DocumentHighlights>,
111}
112
113#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
114pub enum SyntaxStreamError {
115    #[error("syntax stream input requires {attempted} bytes, exceeding the {limit}-byte limit")]
116    InputLimit { limit: usize, attempted: usize },
117}
118
119#[derive(Debug, Clone)]
120pub struct SyntaxStreamUpdate {
121    pub base_revision: u64,
122    pub revision: u64,
123    pub changed_lines: Range<usize>,
124    pub highlights: Arc<DocumentHighlights>,
125}
126
127impl SyntaxStream {
128    #[must_use]
129    pub fn new<'a>(hint: impl Into<LanguageHint<'a>>) -> Self {
130        Self {
131            hint: hint.into().as_str().to_owned(),
132            ..Self::default()
133        }
134    }
135
136    #[must_use]
137    pub fn source(&self) -> &str {
138        &self.source
139    }
140
141    #[must_use]
142    pub const fn revision(&self) -> u64 {
143        self.revision
144    }
145
146    #[must_use]
147    pub fn highlights(&self) -> &Arc<DocumentHighlights> {
148        &self.highlights
149    }
150}
151
152/// Highlight spans projected onto every source line of one parsed document.
153#[derive(Debug, Clone, Default)]
154pub struct DocumentHighlights {
155    lines: Vec<Arc<[HighlightSpan]>>,
156}
157
158impl DocumentHighlights {
159    #[must_use]
160    pub fn line(&self, index: usize) -> Option<&[HighlightSpan]> {
161        self.lines.get(index).map(AsRef::as_ref)
162    }
163
164    #[must_use]
165    pub fn line_shared(&self, index: usize) -> Option<Arc<[HighlightSpan]>> {
166        self.lines.get(index).cloned()
167    }
168
169    #[must_use]
170    pub fn line_count(&self) -> usize {
171        self.lines.len()
172    }
173
174    fn from_spans(spans: &[HighlightSpan], text: &str) -> Self {
175        let starts = if text.is_empty() {
176            Vec::new()
177        } else {
178            let mut starts = vec![0];
179            starts.extend(
180                text.bytes()
181                    .enumerate()
182                    .filter_map(|(index, byte)| (byte == b'\n').then_some(index + 1)),
183            );
184            if text.ends_with('\n') {
185                starts.pop();
186            }
187            starts
188        };
189        let mut lines = Vec::with_capacity(starts.len());
190        let mut first_span = 0;
191        for (line, &start) in starts.iter().enumerate() {
192            let next = starts.get(line + 1).copied().unwrap_or(text.len());
193            let mut end = next;
194            if end > start && text.as_bytes()[end - 1] == b'\n' {
195                end -= 1;
196            }
197            if end > start && text.as_bytes()[end - 1] == b'\r' {
198                end -= 1;
199            }
200            while spans
201                .get(first_span)
202                .is_some_and(|span| span.range.end <= start)
203            {
204                first_span += 1;
205            }
206            let projected = spans[first_span..]
207                .iter()
208                .take_while(|span| span.range.start < end)
209                .filter_map(|span| {
210                    let from = span.range.start.max(start);
211                    let to = span.range.end.min(end);
212                    (from < to).then_some(HighlightSpan {
213                        range: from - start..to - start,
214                        foreground: span.foreground,
215                        font_style: span.font_style,
216                    })
217                })
218                .collect::<Vec<_>>();
219            lines.push(Arc::from(projected));
220        }
221        Self { lines }
222    }
223}
224
225/// A reusable syntax highlighter. Entries are evicted oldest-first when full.
226pub struct SyntaxHighlighter {
227    highlighter: Highlighter,
228    config: CacheConfig,
229    cache: HashMap<CacheKey, Arc<[HighlightSpan]>>,
230    order: VecDeque<CacheKey>,
231    documents: HashMap<CacheKey, Arc<DocumentHighlights>>,
232    document_order: VecDeque<CacheKey>,
233    stats: HighlightStats,
234}
235
236impl fmt::Debug for SyntaxHighlighter {
237    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
238        f.debug_struct("SyntaxHighlighter")
239            .field("config", &self.config)
240            .field("entries", &self.cache.len())
241            .field("documents", &self.documents.len())
242            .field("stats", &self.stats)
243            .finish_non_exhaustive()
244    }
245}
246
247impl Default for SyntaxHighlighter {
248    fn default() -> Self {
249        Self::new(DEFAULT_CAPACITY)
250    }
251}
252
253impl SyntaxHighlighter {
254    /// Creates a highlighter with fixed cache resource limits.
255    #[must_use]
256    pub fn new(config: impl Into<CacheConfig>) -> Self {
257        let syntax_config = Config {
258            max_injection_depth: 3,
259            ..Config::default()
260        };
261        Self {
262            highlighter: Highlighter::with_config(syntax_config),
263            config: config.into(),
264            cache: HashMap::new(),
265            order: VecDeque::new(),
266            documents: HashMap::new(),
267            document_order: VecDeque::new(),
268            stats: HighlightStats::default(),
269        }
270    }
271
272    #[must_use]
273    pub const fn stats(&self) -> HighlightStats {
274        self.stats
275    }
276
277    pub fn reset_stats(&mut self) {
278        self.stats = HighlightStats::default();
279    }
280
281    /// Atomically returns all counters accumulated so far and resets them.
282    pub fn take_stats(&mut self) -> HighlightStats {
283        std::mem::take(&mut self.stats)
284    }
285
286    #[must_use]
287    pub const fn config(&self) -> CacheConfig {
288        self.config
289    }
290
291    #[must_use]
292    pub fn with_theme<'a>(&'a mut self, theme: &'a SyntaxTheme) -> ThemedHighlighter<'a> {
293        ThemedHighlighter {
294            highlighter: self,
295            theme,
296        }
297    }
298
299    pub fn clear_cache(&mut self) {
300        self.cache.clear();
301        self.order.clear();
302        self.documents.clear();
303        self.document_order.clear();
304    }
305
306    fn highlight_source(
307        &mut self,
308        theme: &SyntaxTheme,
309        hint: LanguageHint<'_>,
310        text: &str,
311    ) -> Arc<[HighlightSpan]> {
312        self.stats.calls += 1;
313        let language = resolve_language(hint, text);
314        let id = language.unwrap_or("plain");
315        let key = CacheKey::source(theme.revision(), id, text);
316        if let Some(spans) = self.cache.get(&key) {
317            self.stats.hits += 1;
318            return Arc::clone(spans);
319        }
320        self.stats.misses += 1;
321        let Some(language) = language else {
322            let spans = empty_spans();
323            self.store(key, Arc::clone(&spans));
324            return spans;
325        };
326        self.stats.bytes = self.stats.bytes.saturating_add(text.len());
327        let spans = highlight_source(&mut self.highlighter, theme, language, text)
328            .map_or_else(empty_spans, Arc::from);
329        self.store(key, Arc::clone(&spans));
330        spans
331    }
332
333    fn highlight_lines<'line, T>(
334        &mut self,
335        theme: &SyntaxTheme,
336        hint: LanguageHint<'_>,
337        lines: T,
338    ) -> Vec<Vec<HighlightSpan>>
339    where
340        T: IntoIterator<Item = &'line str>,
341    {
342        self.stats.calls += 1;
343        let selected: Vec<(usize, &str)> = lines.into_iter().enumerate().collect();
344        let window = JoinedLines::new(selected);
345        let Some(language) = resolve_language(hint, &window.source) else {
346            return vec![Vec::new(); window.lines.len()];
347        };
348        self.stats.bytes = self.stats.bytes.saturating_add(window.source.len());
349        highlight_source(&mut self.highlighter, theme, language, &window.source)
350            .as_deref()
351            .map_or_else(
352                || vec![Vec::new(); window.lines.len()],
353                |spans| window.split(spans),
354            )
355    }
356
357    fn store(&mut self, key: CacheKey, spans: Arc<[HighlightSpan]>) {
358        if self.config.max_entries == 0 {
359            return;
360        }
361        if self.cache.insert(key, spans).is_some() {
362            return;
363        }
364        self.order.push_back(key);
365        while self.cache.len() > self.config.max_entries {
366            let Some(evicted) = self.order.pop_front() else {
367                break;
368            };
369            if self.cache.remove(&evicted).is_some() {
370                self.stats.evictions += 1;
371            }
372        }
373    }
374
375    fn store_document(&mut self, key: CacheKey, highlights: Arc<DocumentHighlights>) {
376        if self.config.max_entries == 0 || self.config.max_documents == 0 {
377            return;
378        }
379        if self.documents.insert(key, highlights).is_some() {
380            return;
381        }
382        self.document_order.push_back(key);
383        while self.documents.len() > self.config.max_documents {
384            let Some(evicted) = self.document_order.pop_front() else {
385                break;
386            };
387            if self.documents.remove(&evicted).is_some() {
388                self.stats.evictions += 1;
389            }
390        }
391    }
392}
393
394/// Theme-bound highlighting operations.
395pub struct ThemedHighlighter<'a> {
396    highlighter: &'a mut SyntaxHighlighter,
397    theme: &'a SyntaxTheme,
398}
399
400impl ThemedHighlighter<'_> {
401    /// Parses a complete source on first use and caches line-projected spans.
402    pub fn highlight_document<'a>(
403        &mut self,
404        sequence: SourceSequenceId,
405        language: impl Into<LanguageHint<'a>>,
406        text: &str,
407    ) -> Arc<DocumentHighlights> {
408        self.highlighter.stats.calls += 1;
409        let resolved = resolve_language(language.into(), text);
410        let key = CacheKey::document(self.theme.revision(), resolved.unwrap_or("plain"), sequence);
411        if let Some(highlights) = self.highlighter.documents.get(&key) {
412            self.highlighter.stats.hits += 1;
413            return Arc::clone(highlights);
414        }
415        self.parse_document(key, resolved, text)
416    }
417
418    /// Parses a line sequence as one complete document on first use and caches
419    /// line-projected spans. The lines are only joined and parsed on a miss.
420    pub fn highlight_document_lines<'a, 'line>(
421        &mut self,
422        sequence: SourceSequenceId,
423        language: impl Into<LanguageHint<'a>>,
424        lines: impl IntoIterator<Item = &'line str>,
425    ) -> Arc<DocumentHighlights> {
426        self.highlighter.stats.calls += 1;
427        let mut lines = lines.into_iter().peekable();
428        let resolved = resolve_language(language.into(), lines.peek().copied().unwrap_or_default());
429        let key = CacheKey::document(self.theme.revision(), resolved.unwrap_or("plain"), sequence);
430        if let Some(highlights) = self.highlighter.documents.get(&key) {
431            self.highlighter.stats.hits += 1;
432            return Arc::clone(highlights);
433        }
434        let mut text = String::new();
435        for line in lines {
436            text.push_str(line);
437            text.push('\n');
438        }
439        self.parse_document(key, resolved, &text)
440    }
441
442    fn parse_document(
443        &mut self,
444        key: CacheKey,
445        resolved: Option<&str>,
446        text: &str,
447    ) -> Arc<DocumentHighlights> {
448        self.highlighter.stats.misses += 1;
449        let highlights = Arc::new(self.project(resolved, text));
450        self.highlighter
451            .store_document(key, Arc::clone(&highlights));
452        highlights
453    }
454
455    fn project(&mut self, resolved: Option<&str>, text: &str) -> DocumentHighlights {
456        let spans = resolved
457            .and_then(|language| {
458                self.highlighter.stats.bytes += text.len();
459                highlight_source(
460                    &mut self.highlighter.highlighter,
461                    self.theme,
462                    language,
463                    text,
464                )
465            })
466            .unwrap_or_default();
467        DocumentHighlights::from_spans(&spans, text)
468    }
469
470    /// Highlights a complete source. Hints may be IDs, aliases, or repository paths.
471    pub fn highlight_source<'a>(
472        &mut self,
473        language: impl Into<LanguageHint<'a>>,
474        text: &str,
475    ) -> Arc<[HighlightSpan]> {
476        self.highlighter
477            .highlight_source(self.theme, language.into(), text)
478    }
479
480    /// Highlights all supplied lines in one parse, preserving multiline state.
481    pub fn highlight_lines<'line, 'hint, T>(
482        &mut self,
483        language: impl Into<LanguageHint<'hint>>,
484        lines: T,
485    ) -> Vec<Vec<HighlightSpan>>
486    where
487        T: IntoIterator<Item = &'line str>,
488    {
489        self.highlighter
490            .highlight_lines(self.theme, language.into(), lines)
491    }
492
493    pub fn append<'line>(
494        &mut self,
495        stream: &mut SyntaxStream,
496        lines: impl IntoIterator<Item = &'line str>,
497    ) -> Result<SyntaxStreamUpdate, SyntaxStreamError> {
498        let limit = self.highlighter.config.max_stream_bytes;
499        let mut appended = String::new();
500        for line in lines {
501            let newline = usize::from(!line.ends_with('\n'));
502            let attempted = stream.source.len() + appended.len() + line.len() + newline;
503            if attempted > limit {
504                return Err(SyntaxStreamError::InputLimit { limit, attempted });
505            }
506            appended.push_str(line);
507            if newline != 0 {
508                appended.push('\n');
509            }
510        }
511        let base_revision = stream.revision;
512        let theme_revision = self.theme.revision();
513        if appended.is_empty() && stream.theme_revision == Some(theme_revision) {
514            let end = stream.highlights.line_count();
515            return Ok(SyntaxStreamUpdate {
516                base_revision,
517                revision: stream.revision,
518                changed_lines: end..end,
519                highlights: Arc::clone(&stream.highlights),
520            });
521        }
522        if !appended.is_empty() {
523            stream.source.push_str(&appended);
524            stream.revision = stream.revision.wrapping_add(1);
525        }
526        self.highlighter.stats.calls += 1;
527        let resolved = resolve_language(stream.hint.as_str(), &stream.source);
528        let highlights = Arc::new(self.project(resolved, &stream.source));
529        let first_changed = stream
530            .highlights
531            .lines
532            .iter()
533            .zip(&highlights.lines)
534            .position(|(before, after)| before != after)
535            .unwrap_or_else(|| stream.highlights.line_count().min(highlights.line_count()));
536        stream.highlights = Arc::clone(&highlights);
537        stream.theme_revision = Some(theme_revision);
538        Ok(SyntaxStreamUpdate {
539            base_revision,
540            revision: stream.revision,
541            changed_lines: first_changed..highlights.line_count(),
542            highlights,
543        })
544    }
545}
546
547fn highlight_source(
548    highlighter: &mut Highlighter,
549    theme: &DiffTheme,
550    language: &str,
551    source: &str,
552) -> Option<Vec<HighlightSpan>> {
553    let raw_spans = highlighter.highlight_spans(language, source).ok()?;
554    let tokens = spans_to_flat_tokens(source, raw_spans);
555    let mut spans = Vec::with_capacity(tokens.len());
556    for token in tokens {
557        let Ok(start) = usize::try_from(token.start) else {
558            continue;
559        };
560        let Ok(end) = usize::try_from(token.end) else {
561            continue;
562        };
563        if start >= end
564            || end > source.len()
565            || !source.is_char_boundary(start)
566            || !source.is_char_boundary(end)
567        {
568            continue;
569        }
570        let Some(capture) = diff_capture_name(token.tag) else {
571            continue;
572        };
573        let Some(style) = theme.style(capture) else {
574            continue;
575        };
576        push_merged(
577            &mut spans,
578            HighlightSpan {
579                range: start..end,
580                foreground: style.foreground,
581                font_style: style.font_style,
582            },
583        );
584    }
585    Some(spans)
586}
587
588fn diff_capture_name(tag: &str) -> Option<&'static str> {
589    Some(match tag_to_name(tag)? {
590        "title" => "markup.heading",
591        "strong" => "markup.bold",
592        "emphasis" => "markup.italic",
593        "link" => "markup.link",
594        "literal" => "markup.raw",
595        "strikethrough" => "markup.strikethrough",
596        name => name,
597    })
598}
599
600fn push_merged(spans: &mut Vec<HighlightSpan>, span: HighlightSpan) {
601    if let Some(last) = spans.last_mut()
602        && last.range.end == span.range.start
603        && last.foreground == span.foreground
604        && last.font_style == span.font_style
605    {
606        last.range.end = span.range.end;
607    } else {
608        spans.push(span);
609    }
610}
611
612struct JoinedLines<'a> {
613    source: String,
614    /// Sequence index, original line, global display start, global display end.
615    lines: Vec<(usize, &'a str, usize, usize)>,
616}
617
618impl<'a> JoinedLines<'a> {
619    fn new(selected: Vec<(usize, &'a str)>) -> Self {
620        let mut source = String::new();
621        let mut lines = Vec::with_capacity(selected.len());
622        for (index, line) in selected {
623            let start = source.len();
624            source.push_str(line);
625            let end = source.len();
626            if !line.ends_with('\n') {
627                source.push('\n');
628            }
629            lines.push((index, line, start, end));
630        }
631        Self { source, lines }
632    }
633
634    /// Splits window-global spans into per-line local spans. Spans must be
635    /// disjoint and ordered, as [`highlight_source`] produces them, so one
636    /// forward sweep serves every line; a span crossing lines is revisited
637    /// only by the lines it overlaps.
638    fn split(&self, spans: &[HighlightSpan]) -> Vec<Vec<HighlightSpan>> {
639        let mut next = 0;
640        self.lines
641            .iter()
642            .map(|(_, line, start, end)| {
643                while spans.get(next).is_some_and(|span| span.range.end <= *start) {
644                    next += 1;
645                }
646                let mut result = Vec::new();
647                for span in &spans[next..] {
648                    if span.range.start >= *end {
649                        break;
650                    }
651                    let overlap_start = span.range.start.max(*start);
652                    let overlap_end = span.range.end.min(*end);
653                    if overlap_start >= overlap_end {
654                        continue;
655                    }
656                    let local_start = overlap_start - start;
657                    let local_end = overlap_end - start;
658                    if local_end <= line.len()
659                        && line.is_char_boundary(local_start)
660                        && line.is_char_boundary(local_end)
661                    {
662                        push_merged(
663                            &mut result,
664                            HighlightSpan {
665                                range: local_start..local_end,
666                                foreground: span.foreground,
667                                font_style: span.font_style,
668                            },
669                        );
670                    }
671                }
672                result
673            })
674            .collect()
675    }
676}
677
678#[cfg(test)]
679mod tests {
680    use super::*;
681
682    fn assert_valid_spans(source: &str, spans: &[HighlightSpan]) {
683        let mut previous_end = 0;
684        for span in spans {
685            assert!(span.range.start < span.range.end, "empty span: {span:?}");
686            assert!(
687                span.range.end <= source.len(),
688                "out-of-bounds span: {span:?}"
689            );
690            assert!(source.is_char_boundary(span.range.start));
691            assert!(source.is_char_boundary(span.range.end));
692            assert!(
693                span.range.start >= previous_end,
694                "overlapping or unordered span: {span:?}"
695            );
696            previous_end = span.range.end;
697        }
698    }
699
700    #[test]
701    fn aliases_and_utf8_ranges() {
702        let theme = DiffTheme::default();
703        let mut highlighter = SyntaxHighlighter::new(2);
704        let source = "let café = 1;\n";
705        let spans = highlighter
706            .with_theme(&theme)
707            .highlight_source("rs", source);
708        assert!(!spans.is_empty());
709        assert!(
710            spans
711                .iter()
712                .all(|span| source.is_char_boundary(span.range.start)
713                    && source.is_char_boundary(span.range.end))
714        );
715        let _ = highlighter
716            .with_theme(&theme)
717            .highlight_source("RUST", source);
718        assert_eq!(highlighter.stats().hits, 1);
719    }
720
721    #[test]
722    fn complete_documents_are_parsed_once_and_projected_by_line() {
723        let theme = DiffTheme::default();
724        let mut highlighter = SyntaxHighlighter::default();
725        let source = "/* alpha\r\nbeta */\nlet café = 1;\n";
726        let sequence = SourceSequenceId::from_lines(source.split_terminator('\n'));
727
728        let first = highlighter.with_theme(&theme).highlight_document(
729            sequence,
730            LanguageHint::Path("src/lib.rs"),
731            source,
732        );
733        assert_eq!(first.line_count(), 3);
734        assert!(!first.line(0).unwrap().is_empty());
735        assert!(!first.line(1).unwrap().is_empty());
736        assert!(
737            first
738                .line(2)
739                .unwrap()
740                .iter()
741                .all(|span| span.range.end <= "let café = 1;".len())
742        );
743        let parsed_bytes = highlighter.stats().bytes;
744
745        let second = highlighter.with_theme(&theme).highlight_document(
746            sequence,
747            LanguageHint::Path("src/lib.rs"),
748            source,
749        );
750        assert!(Arc::ptr_eq(&first, &second));
751        assert_eq!(highlighter.stats().hits, 1);
752        assert_eq!(highlighter.stats().bytes, parsed_bytes);
753
754        let empty = highlighter.with_theme(&theme).highlight_document(
755            SourceSequenceId::from_lines([]),
756            "rust",
757            "",
758        );
759        assert_eq!(empty.line_count(), 0);
760    }
761
762    #[test]
763    fn line_sequences_parse_once_and_keep_multiline_context() {
764        let theme = DiffTheme::default();
765        let mut highlighter = SyntaxHighlighter::default();
766        let lines = ["/* alpha", "beta", "gamma */", "let x = 1;"];
767        let sequence = SourceSequenceId::from_lines(lines);
768
769        let first = highlighter.with_theme(&theme).highlight_document_lines(
770            sequence,
771            LanguageHint::Path("src/lib.rs"),
772            lines,
773        );
774        assert_eq!(first.line_count(), 4);
775        for (index, line) in lines.iter().enumerate() {
776            let spans = first.line(index).unwrap();
777            assert!(!spans.is_empty(), "line {index}");
778            assert!(spans.iter().all(|span| span.range.end <= line.len()));
779        }
780
781        let second = highlighter.with_theme(&theme).highlight_document_lines(
782            sequence,
783            LanguageHint::Path("src/lib.rs"),
784            lines,
785        );
786        assert!(Arc::ptr_eq(&first, &second));
787        assert_eq!(highlighter.stats().misses, 1);
788        assert_eq!(highlighter.stats().hits, 1);
789    }
790
791    #[test]
792    fn supported_language_bundle_highlights_representative_sources() {
793        let cases = [
794            ("rust", "fn main() {}"),
795            ("js", "const x = true;"),
796            ("jsx", "const view = <Panel title=\"Hi\" />;"),
797            ("typescript", "const x: number = 1;"),
798            ("tsx", "const view = <Panel title=\"Hi\" />;"),
799            ("py", "def f(): return 1"),
800            ("sh", "echo hi"),
801            ("c", "int main(void) {}"),
802            ("cpp", "class C {};"),
803            ("go", "package main"),
804            ("json", "{\"x\": true}"),
805            ("jsonc", "{\"x\": true /* comment */}"),
806            ("toml", "x = 1"),
807            ("yml", "x: true"),
808            ("html", "<b>x</b>"),
809            ("css", "b { color: red; }"),
810            ("md", "# Heading"),
811        ];
812        let theme = DiffTheme::default();
813        let mut highlighter = SyntaxHighlighter::new(cases.len());
814        for (language, source) in cases {
815            let spans = highlighter
816                .with_theme(&theme)
817                .highlight_source(language, source);
818            assert!(!spans.is_empty(), "{language}");
819            assert_valid_spans(source, &spans);
820        }
821    }
822
823    #[test]
824    fn unknown_language_is_plain_text_and_cached() {
825        let mut highlighter = SyntaxHighlighter::new(2);
826        let theme = DiffTheme::default();
827        assert!(
828            highlighter
829                .with_theme(&theme)
830                .highlight_source("binary.zzz", "abc")
831                .is_empty()
832        );
833        assert!(
834            highlighter
835                .with_theme(&theme)
836                .highlight_source("binary.zzz", "abc")
837                .is_empty()
838        );
839        assert_eq!(highlighter.stats().hits, 1);
840        assert_eq!(highlighter.stats().bytes, 0);
841    }
842
843    #[test]
844    fn fifo_zero_capacity_and_theme_revision_behave_as_before() {
845        let theme = DiffTheme::default();
846        let mut fifo = SyntaxHighlighter::new(1);
847        fifo.with_theme(&theme)
848            .highlight_source("rust", "fn a() {}");
849        fifo.with_theme(&theme)
850            .highlight_source("rust", "fn b() {}");
851        assert_eq!(fifo.stats().evictions, 1);
852        let mut zero = SyntaxHighlighter::new(0);
853        zero.with_theme(&theme)
854            .highlight_source("rust", "fn a() {}");
855        zero.with_theme(&theme)
856            .highlight_source("rust", "fn a() {}");
857        assert_eq!(zero.stats().misses, 2);
858        let mut themes = SyntaxHighlighter::new(8);
859        themes
860            .with_theme(&theme)
861            .highlight_source("rust", "fn main() {}");
862        themes
863            .with_theme(&DiffTheme::ayu().unwrap())
864            .highlight_source("rust", "fn main() {}");
865        assert_eq!(themes.stats().misses, 2);
866    }
867
868    #[test]
869    fn multiline_and_synthetic_newlines_are_clipped() {
870        let theme = DiffTheme::default();
871        let mut highlighter = SyntaxHighlighter::new(0);
872        let lines = ["/*", " café */"];
873        let rendered = highlighter
874            .with_theme(&theme)
875            .highlight_lines("rust", lines);
876        assert_eq!(rendered.len(), 2);
877        assert!(rendered.iter().all(|spans| !spans.is_empty()));
878        for (line, spans) in lines.into_iter().zip(rendered) {
879            assert!(spans.iter().all(|span| span.range.end <= line.len()
880                && line.is_char_boundary(span.range.start)
881                && line.is_char_boundary(span.range.end)));
882        }
883    }
884
885    #[test]
886    fn html_and_markdown_injections_highlight_embedded_languages() {
887        let theme = DiffTheme::default();
888        let mut highlighter = SyntaxHighlighter::new(4);
889        let html = "<p>café</p><script>const π = 3.14;</script><style>b{color:red}</style>";
890        let html_spans = highlighter
891            .with_theme(&theme)
892            .highlight_source("html", html);
893        assert_valid_spans(html, &html_spans);
894        for needle in ["const π", "color:red"] {
895            let start = html.find(needle).unwrap();
896            let end = start + needle.len();
897            assert!(
898                html_spans
899                    .iter()
900                    .any(|span| span.range.start < end && span.range.end > start),
901                "no injected highlight for {needle}"
902            );
903        }
904
905        let markdown = "# Title\n\n```rust\nfn main() {}\n```\n";
906        let markdown_spans = highlighter
907            .with_theme(&theme)
908            .highlight_source("markdown", markdown);
909        assert_valid_spans(markdown, &markdown_spans);
910        let start = markdown.find("fn main").unwrap();
911        let end = start + "fn main".len();
912        assert!(
913            markdown_spans
914                .iter()
915                .any(|span| span.range.start < end && span.range.end > start),
916            "no injected Rust highlight in Markdown"
917        );
918    }
919}
clankerdiff_syntax/highlight.rs

clankerdiff_syntax/
highlight.rs
