Skip to main content

gpui_m3/
syntax.rs

1//! Syntax highlighting for code blocks and viewers, line by line.
2//!
3//! Built on [syntect](https://crates.io/crates/syntect) (MIT) with its bundled Sublime Text
4//! syntaxes and the pure-Rust `fancy-regex` engine (no C toolchain needed). syntect parses one
5//! line at a time and its state can be cloned, which is what an editor needs to re-highlight
6//! from the edited line on. TypeScript/TSX, TOML and PowerShell are not in the bundled set, so
7//! compact grammars for them ship in `assets/syntaxes/`.
8//!
9//! Tokens are reduced to a few [`TokenKind`]s, the same classes the reference design colors
10//! (`--sx-k`, `--sx-s`…), and painted with a [`SyntaxPalette`] derived from the theme.
11//!
12//! ```ignore
13//! let mut hl = SyntaxHighlighter::new("rust").unwrap();
14//! for line in code.lines() {
15//!     let tokens = hl.highlight_line(line); // Vec<(Range<usize>, TokenKind)>
16//! }
17//! ```
18
19use std::cell::RefCell;
20use std::collections::HashMap;
21use std::ops::Range;
22use std::rc::Rc;
23use std::sync::OnceLock;
24
25use gpui::{
26    App, FontStyle, FontWeight, Global, HighlightStyle, Hsla, SharedString, UnderlineStyle, px,
27};
28use syntect::parsing::{
29    ParseState, Scope, ScopeStack, SyntaxDefinition, SyntaxReference, SyntaxSet, SyntaxSetBuilder,
30};
31
32use crate::cache::{Lru, content_key};
33use crate::theme::{Scheme, Theme, hex};
34
35/// What a piece of code is, as far as coloring goes.
36#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
37pub enum TokenKind {
38    /// Keywords, storage (`fn`, `const`), word operators and `self`/`this` (`--sx-k`).
39    Keyword,
40    /// Strings, characters, escapes and regular expressions (`--sx-s`).
41    String,
42    /// Function names at definitions and calls (`--sx-f`).
43    Function,
44    /// Types, classes, namespaces and markup tags (`--sx-t`).
45    Type,
46    /// Comments (`--sx-c`, italic).
47    Comment,
48    /// Numbers, booleans, `null` and other constants (`--sx-n`).
49    Number,
50    /// Macros, attributes, annotations and decorators (`--sx-m`).
51    Meta,
52    /// Markup headings and section names (keyword color, bold).
53    Heading,
54    /// Markup links (accent, underlined).
55    Link,
56}
57
58/// Colors for each [`TokenKind`].
59#[derive(Clone, Copy, Debug, PartialEq)]
60pub struct SyntaxPalette {
61    pub keyword: Hsla,
62    pub string: Hsla,
63    pub function: Hsla,
64    pub type_name: Hsla,
65    pub comment: Hsla,
66    pub number: Hsla,
67    pub meta: Hsla,
68    pub link: Hsla,
69}
70
71/// An app-wide override set with [`SyntaxPalette::set`].
72struct PaletteOverride(SyntaxPalette);
73
74impl Global for PaletteOverride {}
75
76impl SyntaxPalette {
77    /// The reference design's M3 Expressive syntax colors: keywords in the primary color, strings
78    /// in the success color, types in the tertiary color, comments in the outline color, numbers
79    /// in the warning color, plus a fixed orange for functions and blue for meta.
80    pub fn from_scheme(s: &Scheme, dark: bool) -> Self {
81        let (function, meta) = if dark {
82            (hex("#ffb68b"), hex("#9ecaff"))
83        } else {
84            (hex("#9a4600"), hex("#00639b"))
85        };
86        Self {
87            keyword: s.primary,
88            string: s.success,
89            function,
90            type_name: s.tertiary,
91            comment: s.outline,
92            number: s.warning,
93            meta,
94            link: s.primary,
95        }
96    }
97
98    /// The override set with [`SyntaxPalette::set`], or the colors derived from the current theme.
99    pub fn of(cx: &App) -> Self {
100        if let Some(p) = cx.try_global::<PaletteOverride>() {
101            return p.0;
102        }
103        let theme = Theme::get(cx);
104        Self::from_scheme(&theme.scheme, theme.dark)
105    }
106
107    /// Uses your own syntax colors (e.g. another visual style's tokens); `None` goes back to the
108    /// theme's.
109    pub fn set(palette: Option<SyntaxPalette>, cx: &mut App) {
110        match palette {
111            Some(p) => cx.set_global(PaletteOverride(p)),
112            None => {
113                if cx.has_global::<PaletteOverride>() {
114                    cx.remove_global::<PaletteOverride>();
115                }
116            }
117        }
118        cx.refresh_windows();
119    }
120
121    pub fn color(&self, kind: TokenKind) -> Hsla {
122        match kind {
123            TokenKind::Keyword | TokenKind::Heading => self.keyword,
124            TokenKind::String => self.string,
125            TokenKind::Function => self.function,
126            TokenKind::Type => self.type_name,
127            TokenKind::Comment => self.comment,
128            TokenKind::Number => self.number,
129            TokenKind::Meta => self.meta,
130            TokenKind::Link => self.link,
131        }
132    }
133
134    /// Text style for a token: its color, italic comments, bold headings and underlined links.
135    pub fn style(&self, kind: TokenKind) -> HighlightStyle {
136        let color = self.color(kind);
137        HighlightStyle {
138            color: Some(color),
139            font_style: (kind == TokenKind::Comment).then_some(FontStyle::Italic),
140            font_weight: (kind == TokenKind::Heading).then_some(FontWeight::BOLD),
141            underline: (kind == TokenKind::Link).then_some(UnderlineStyle {
142                thickness: px(1.),
143                color: Some(color),
144                wavy: false,
145            }),
146            ..Default::default()
147        }
148    }
149
150    /// Highlights for a [`gpui::StyledText`] from token ranges.
151    pub fn highlights(
152        &self,
153        tokens: &[(Range<usize>, TokenKind)],
154    ) -> Vec<(Range<usize>, HighlightStyle)> {
155        tokens
156            .iter()
157            .map(|(range, kind)| (range.clone(), self.style(*kind)))
158            .collect()
159    }
160}
161
162/* ─────────── Languages ─────────── */
163
164/// Language of a code fence: a readable name and whether it can be highlighted.
165#[derive(Clone, Debug, PartialEq, Eq)]
166pub struct LanguageInfo {
167    /// "TypeScript", "Rust"… The fence's own word when unknown, "Text" when empty.
168    pub label: SharedString,
169    /// A grammar is available for it.
170    pub highlighted: bool,
171}
172
173/// Fence word → (label, grammar). The grammar is a syntect syntax name; `None` is never colored.
174const ALIASES: &[(&str, &str, Option<&str>)] = &[
175    ("ts", "TypeScript", Some("TypeScript")),
176    ("typescript", "TypeScript", Some("TypeScript")),
177    ("mts", "TypeScript", Some("TypeScript")),
178    ("cts", "TypeScript", Some("TypeScript")),
179    ("tsx", "TSX", Some("TypeScript")),
180    ("js", "JavaScript", Some("JavaScript")),
181    ("javascript", "JavaScript", Some("JavaScript")),
182    ("jsx", "JSX", Some("JavaScript")),
183    ("mjs", "JavaScript", Some("JavaScript")),
184    ("cjs", "JavaScript", Some("JavaScript")),
185    ("rust", "Rust", Some("Rust")),
186    ("rs", "Rust", Some("Rust")),
187    ("json", "JSON", Some("JSON")),
188    ("jsonc", "JSON", Some("JSON")),
189    ("json5", "JSON", Some("JSON")),
190    ("md", "Markdown", Some("Markdown")),
191    ("markdown", "Markdown", Some("Markdown")),
192    ("css", "CSS", Some("CSS")),
193    ("scss", "SCSS", Some("CSS")),
194    ("html", "HTML", Some("HTML")),
195    ("htm", "HTML", Some("HTML")),
196    ("xml", "XML", Some("XML")),
197    ("svg", "SVG", Some("XML")),
198    ("py", "Python", Some("Python")),
199    ("python", "Python", Some("Python")),
200    ("toml", "TOML", Some("TOML")),
201    ("yaml", "YAML", Some("YAML")),
202    ("yml", "YAML", Some("YAML")),
203    ("sh", "Shell", Some("Bourne Again Shell (bash)")),
204    ("bash", "Bash", Some("Bourne Again Shell (bash)")),
205    ("zsh", "Zsh", Some("Bourne Again Shell (bash)")),
206    ("shell", "Shell", Some("Bourne Again Shell (bash)")),
207    ("console", "Console", None),
208    ("powershell", "PowerShell", Some("PowerShell")),
209    ("ps1", "PowerShell", Some("PowerShell")),
210    ("pwsh", "PowerShell", Some("PowerShell")),
211    ("ps", "PowerShell", Some("PowerShell")),
212    ("diff", "Diff", Some("Diff")),
213    ("patch", "Diff", Some("Diff")),
214    ("text", "Text", None),
215    ("txt", "Text", None),
216    ("plain", "Text", None),
217    ("plaintext", "Text", None),
218];
219
220/// First word of a fence's info string, lowercased (```` ```ts title="x" ```` → `ts`).
221fn fence_key(info: &str) -> String {
222    info.split_whitespace()
223        .next()
224        .unwrap_or("")
225        .trim_start_matches('{')
226        .trim_start_matches('.')
227        .trim_end_matches('}')
228        .to_lowercase()
229}
230
231/// Readable name of a fence's language and whether it can be highlighted.
232pub fn language(info: &str) -> LanguageInfo {
233    let key = fence_key(info);
234    if key.is_empty() {
235        return LanguageInfo {
236            label: "Text".into(),
237            highlighted: false,
238        };
239    }
240    if let Some((_, label, grammar)) = ALIASES.iter().find(|(k, _, _)| *k == key) {
241        return LanguageInfo {
242            label: (*label).into(),
243            highlighted: grammar.is_some(),
244        };
245    }
246    LanguageInfo {
247        highlighted: resolve(&key).is_some(),
248        label: key.into(),
249    }
250}
251
252fn defaults() -> &'static SyntaxSet {
253    static SET: OnceLock<SyntaxSet> = OnceLock::new();
254    SET.get_or_init(SyntaxSet::load_defaults_newlines)
255}
256
257/// The grammars gpui-m3 adds to syntect's bundled set.
258fn extras() -> &'static SyntaxSet {
259    static SET: OnceLock<SyntaxSet> = OnceLock::new();
260    SET.get_or_init(|| {
261        let mut builder = SyntaxSetBuilder::new();
262        for source in [
263            include_str!("../assets/syntaxes/TypeScript.sublime-syntax"),
264            include_str!("../assets/syntaxes/TOML.sublime-syntax"),
265            include_str!("../assets/syntaxes/PowerShell.sublime-syntax"),
266        ] {
267            match SyntaxDefinition::load_from_str(source, true, None) {
268                Ok(def) => builder.add(def),
269                Err(err) => log::warn!("gpui_m3: invalid bundled syntax: {err}"),
270            }
271        }
272        builder.build()
273    })
274}
275
276/// The grammar for a fence word: an alias, then syntect's own names and extensions.
277fn resolve(key: &str) -> Option<(&'static SyntaxSet, &'static SyntaxReference)> {
278    let find = |name: &str| {
279        [extras(), defaults()]
280            .into_iter()
281            .find_map(|set| set.find_syntax_by_name(name).map(|syntax| (set, syntax)))
282    };
283    if let Some((_, _, grammar)) = ALIASES.iter().find(|(k, _, _)| *k == key) {
284        return find((*grammar)?);
285    }
286    [extras(), defaults()]
287        .into_iter()
288        .find_map(|set| set.find_syntax_by_token(key).map(|syntax| (set, syntax)))
289}
290
291/* ─────────── Highlighter ─────────── */
292
293/// How a scope colors its text.
294#[derive(Clone, Copy)]
295enum Class {
296    Kind(TokenKind),
297    /// A keyword operator: colored as a keyword only when it is a word (`typeof`, `and`).
298    Operator,
299    /// Look at the enclosing scope.
300    Pass,
301    /// Embedded code or interpolation: stop, the enclosing string or tag does not apply.
302    Stop,
303}
304
305fn classify(scope: &str) -> Class {
306    use TokenKind::*;
307    let starts = |prefixes: &[&str]| {
308        prefixes
309            .iter()
310            .any(|p| scope == *p || scope.starts_with(&format!("{p}.")))
311    };
312    if starts(&["comment"]) {
313        Class::Kind(Comment)
314    } else if starts(&["meta.interpolation", "meta.embedded", "source", "text"])
315        || scope.starts_with("punctuation.section.embedded")
316    {
317        Class::Stop
318    } else if starts(&["string.other.link", "markup.underline.link", "meta.link"]) {
319        Class::Kind(Link)
320    } else if starts(&["string", "constant.character", "constant.other.placeholder"]) {
321        Class::Kind(String)
322    } else if starts(&["constant.numeric", "constant.language", "support.constant"]) {
323        Class::Kind(Number)
324    } else if starts(&["keyword.operator"]) {
325        Class::Operator
326    } else if starts(&[
327        "support.macro",
328        "entity.name.macro",
329        "meta.annotation",
330        "meta.decorator",
331        "meta.preprocessor",
332        "entity.other.attribute-name",
333        "storage.type.annotation",
334        "support.variable",
335        "keyword.other.directive",
336    ]) {
337        Class::Kind(Meta)
338    } else if starts(&["keyword", "storage", "variable.language"]) {
339        Class::Kind(Keyword)
340    } else if starts(&["markup.heading", "entity.name.section"]) {
341        Class::Kind(Heading)
342    } else if starts(&[
343        "entity.name.function",
344        "support.function",
345        "variable.function",
346    ]) {
347        Class::Kind(Function)
348    } else if starts(&[
349        "entity.name",
350        "entity.other.inherited-class",
351        "support.type",
352        "support.class",
353    ]) {
354        Class::Kind(Type)
355    } else {
356        Class::Pass
357    }
358}
359
360thread_local! {
361    static CLASSES: RefCell<HashMap<Scope, Class>> = RefCell::new(HashMap::new());
362}
363
364/// Kind of the text under a scope stack: the innermost scope that says something wins.
365fn kind_of(stack: &[Scope], text: &str) -> Option<TokenKind> {
366    CLASSES.with(|classes| {
367        let mut classes = classes.borrow_mut();
368        for scope in stack.iter().rev() {
369            let class = *classes
370                .entry(*scope)
371                .or_insert_with(|| classify(&scope.build_string()));
372            match class {
373                Class::Kind(kind) => return Some(kind),
374                Class::Operator => {
375                    if text.chars().any(char::is_alphabetic) {
376                        return Some(TokenKind::Keyword);
377                    }
378                }
379                Class::Pass => {}
380                Class::Stop => return None,
381            }
382        }
383        None
384    })
385}
386
387/// Highlights code one line at a time, keeping the parser state between lines (a string or
388/// comment that spans lines stays colored). Clone it to snapshot the state after a line.
389#[derive(Clone)]
390pub struct SyntaxHighlighter {
391    set: &'static SyntaxSet,
392    state: ParseState,
393    stack: ScopeStack,
394}
395
396impl SyntaxHighlighter {
397    /// A highlighter for a fence word or file extension (`"ts"`, `"rust"`, `"py"`…), or `None`
398    /// when there is no grammar for it.
399    pub fn new(language: &str) -> Option<Self> {
400        let (set, syntax) = resolve(&fence_key(language))?;
401        Some(Self {
402            set,
403            state: ParseState::new(syntax),
404            stack: ScopeStack::new(),
405        })
406    }
407
408    /// A highlighter chosen from a file name's extension (`src/main.rs` → Rust).
409    pub fn for_path(path: &str) -> Option<Self> {
410        let name = path.rsplit(['/', '\\']).next().unwrap_or(path);
411        let ext = name.rsplit_once('.').map(|(_, ext)| ext)?;
412        Self::new(ext)
413    }
414
415    /// Tokens of one line (without its line break), as byte ranges into `line`. Uncolored text
416    /// has no token. Lines must be fed in order.
417    pub fn highlight_line(&mut self, line: &str) -> Vec<(Range<usize>, TokenKind)> {
418        let line = line.strip_suffix('\n').unwrap_or(line);
419        let line = line.strip_suffix('\r').unwrap_or(line);
420        let len = line.len();
421        let mut input = String::with_capacity(len + 1);
422        input.push_str(line);
423        input.push('\n');
424        let Ok(ops) = self.state.parse_line(&input, self.set) else {
425            return Vec::new();
426        };
427        let mut out: Vec<(Range<usize>, TokenKind)> = Vec::new();
428        let mut push = |range: Range<usize>, stack: &ScopeStack| {
429            let range = range.start.min(len)..range.end.min(len);
430            if range.is_empty() {
431                return;
432            }
433            let Some(kind) = kind_of(stack.as_slice(), &line[range.clone()]) else {
434                return;
435            };
436            match out.last_mut() {
437                Some((last, k)) if *k == kind && last.end == range.start => last.end = range.end,
438                _ => out.push((range, kind)),
439            }
440        };
441        let mut at = 0;
442        for (pos, op) in ops {
443            if pos > at {
444                push(at..pos, &self.stack);
445                at = pos;
446            }
447            let _ = self.stack.apply(&op);
448        }
449        if at < len {
450            push(at..len, &self.stack);
451        }
452        out
453    }
454
455    /// Tokens of a whole text, as byte ranges into `code`.
456    pub fn highlight(&mut self, code: &str) -> Vec<(Range<usize>, TokenKind)> {
457        let mut out = Vec::new();
458        let mut start = 0;
459        for line in code.split_inclusive('\n') {
460            out.extend(
461                self.highlight_line(line)
462                    .into_iter()
463                    .map(|(r, k)| (r.start + start..r.end + start, k)),
464            );
465            start += line.len();
466        }
467        out
468    }
469}
470
471/// Tokens of `code` in `language` (byte ranges into `code`), or `None` without a grammar.
472pub fn highlight(code: &str, language: &str) -> Option<Vec<(Range<usize>, TokenKind)>> {
473    Some(SyntaxHighlighter::new(language)?.highlight(code))
474}
475
476/// A text highlighted up to its last complete line, kept to continue a streaming code block.
477struct Partial {
478    language: String,
479    /// Byte length of the complete lines already highlighted.
480    done: usize,
481    prefix_key: u64,
482    tokens: Vec<(Range<usize>, TokenKind)>,
483    state: SyntaxHighlighter,
484}
485
486type Tokens = Rc<[(Range<usize>, TokenKind)]>;
487
488thread_local! {
489    static HIGHLIGHTS: RefCell<Lru<Option<Tokens>>> = const { RefCell::new(Lru::new(4096)) };
490    static PARTIAL: RefCell<Option<Partial>> = const { RefCell::new(None) };
491}
492
493/// [`highlight`] with a per-thread cache, for renders: finished blocks are not highlighted again,
494/// and a block that keeps growing (streaming) continues from its last complete line.
495pub fn highlight_cached(code: &str, language: &str) -> Option<Tokens> {
496    let key = content_key(&[language, code]);
497    if let Some(hit) = HIGHLIGHTS.with(|c| c.borrow_mut().get(key)) {
498        return hit;
499    }
500    let tokens = PARTIAL.with(|partial| {
501        let mut partial = partial.borrow_mut();
502        let mut resumed = partial.take().filter(|p| {
503            p.language == language
504                && code.len() >= p.done
505                && code.is_char_boundary(p.done)
506                && content_key(&[&code[..p.done]]) == p.prefix_key
507        });
508        let (mut tokens, mut state, start) = match resumed.take() {
509            Some(p) => (p.tokens, p.state, p.done),
510            None => (Vec::new(), SyntaxHighlighter::new(language)?, 0),
511        };
512        // Complete lines advance the saved state; the trailing partial line is highlighted on a copy.
513        let done = code.rfind('\n').map_or(0, |i| i + 1).max(start);
514        let mut offset = start;
515        for line in code[start..done].split_inclusive('\n') {
516            tokens.extend(
517                state
518                    .highlight_line(line)
519                    .into_iter()
520                    .map(|(r, k)| (r.start + offset..r.end + offset, k)),
521            );
522            offset += line.len();
523        }
524        let mut all = tokens.clone();
525        if done < code.len() {
526            all.extend(
527                state
528                    .clone()
529                    .highlight_line(&code[done..])
530                    .into_iter()
531                    .map(|(r, k)| (r.start + done..r.end + done, k)),
532            );
533        }
534        *partial = Some(Partial {
535            language: language.to_string(),
536            done,
537            prefix_key: content_key(&[&code[..done]]),
538            tokens,
539            state,
540        });
541        Some(Rc::from(all))
542    });
543    HIGHLIGHTS.with(|c| c.borrow_mut().insert(key, tokens.clone()));
544    tokens
545}
546
547/// The tokens of one line.
548pub type LineTokens = Vec<(Range<usize>, TokenKind)>;
549
550/// Feeds a [`CodeEditor`](crate::CodeEditor) its colors with a [`SyntaxHighlighter`]. It keeps
551/// the parser state every [`SyntaxLines::STEP`] lines, so any visible line is highlighted by
552/// re-parsing at most that many lines before it, and edits only drop what follows them.
553pub struct SyntaxLines {
554    start: SyntaxHighlighter,
555    /// `checkpoints[k]`: the state before line `k * STEP`.
556    checkpoints: Vec<SyntaxHighlighter>,
557    cache: Vec<Option<LineTokens>>,
558}
559
560impl SyntaxLines {
561    pub const STEP: usize = 16;
562
563    pub fn new(highlighter: SyntaxHighlighter) -> Self {
564        Self {
565            checkpoints: vec![highlighter.clone()],
566            start: highlighter,
567            cache: Vec::new(),
568        }
569    }
570
571    /// For a file name's extension, or `None` when there is no grammar for it.
572    pub fn for_path(path: &str) -> Option<Self> {
573        SyntaxHighlighter::for_path(path).map(Self::new)
574    }
575
576    /// The tokens of `line`, given a way to read any line of the text.
577    pub fn tokens<'a>(
578        &mut self,
579        line: usize,
580        text: impl Fn(usize) -> &'a str,
581    ) -> Vec<(Range<usize>, TokenKind)> {
582        if let Some(Some(hit)) = self.cache.get(line) {
583            return hit.clone();
584        }
585        let k = line / Self::STEP;
586        while self.checkpoints.len() <= k {
587            let last = self.checkpoints.len() - 1;
588            let mut hl = self.checkpoints[last].clone();
589            for ix in last * Self::STEP..(last + 1) * Self::STEP {
590                hl.highlight_line(text(ix));
591            }
592            self.checkpoints.push(hl);
593        }
594        let mut hl = self.checkpoints[k].clone();
595        for ix in k * Self::STEP..line {
596            hl.highlight_line(text(ix));
597        }
598        let tokens = hl.highlight_line(text(line));
599        if self.cache.len() <= line {
600            self.cache.resize(line + 1, None);
601        }
602        self.cache[line] = Some(tokens.clone());
603        tokens
604    }
605
606    /// The text changed from `from_line` on.
607    pub fn invalidate(&mut self, from_line: usize) {
608        self.cache.truncate(from_line);
609        self.checkpoints.truncate(from_line / Self::STEP + 1);
610        if self.checkpoints.is_empty() {
611            self.checkpoints.push(self.start.clone());
612        }
613    }
614}
615
616impl crate::LineHighlighter for SyntaxLines {
617    fn highlight_line(
618        &mut self,
619        buffer: &crate::CodeBuffer,
620        line: usize,
621        cx: &App,
622    ) -> crate::LineHighlights {
623        let count = buffer.line_count();
624        let palette = SyntaxPalette::of(cx);
625        self.tokens(line, |ix| if ix < count { buffer.line(ix) } else { "" })
626            .into_iter()
627            .map(|(range, kind)| (range, palette.style(kind)))
628            .collect()
629    }
630
631    fn invalidate(&mut self, from_line: usize) {
632        SyntaxLines::invalidate(self, from_line);
633    }
634}
635
636#[cfg(test)]
637mod tests {
638    use super::*;
639
640    #[test]
641    fn syntax_lines_match_a_straight_pass_and_survive_edits() {
642        let lines: Vec<String> = (0..40)
643            .map(|i| {
644                if i == 20 {
645                    "/* open".into()
646                } else if i == 22 {
647                    "close */".into()
648                } else {
649                    format!("let x{i} = \"s\"; // c")
650                }
651            })
652            .collect();
653        let mut straight = SyntaxHighlighter::new("rust").unwrap();
654        let expected: Vec<_> = lines.iter().map(|l| straight.highlight_line(l)).collect();
655        let mut sl = SyntaxLines::for_path("main.rs").unwrap();
656        for ix in [35, 3, 21, 39, 0, 17] {
657            assert_eq!(
658                sl.tokens(ix, |i| lines[i].as_str()),
659                expected[ix],
660                "line {ix}"
661            );
662        }
663        sl.invalidate(18);
664        assert_eq!(sl.tokens(21, |i| lines[i].as_str()), expected[21]);
665        assert_eq!(sl.tokens(2, |i| lines[i].as_str()), expected[2]);
666    }
667
668    fn kinds(line: &str, tokens: &[(Range<usize>, TokenKind)]) -> Vec<(String, TokenKind)> {
669        tokens
670            .iter()
671            .map(|(r, k)| (line[r.clone()].to_string(), *k))
672            .collect()
673    }
674
675    fn has(language: &str, code: &str, text: &str, kind: TokenKind) -> bool {
676        let tokens = highlight(code, language).unwrap_or_else(|| panic!("no grammar: {language}"));
677        kinds(code, &tokens)
678            .iter()
679            .any(|(t, k)| t.contains(text) && *k == kind)
680    }
681
682    #[test]
683    fn every_listed_language_has_a_grammar() {
684        for (key, label, grammar) in ALIASES {
685            if grammar.is_some() {
686                assert!(resolve(key).is_some(), "{key} ({label}) has no grammar");
687            }
688        }
689    }
690
691    #[test]
692    fn labels_follow_the_reference_aliases() {
693        assert_eq!(language("tsx").label, "TSX");
694        assert_eq!(language("ts title=x.ts").label, "TypeScript");
695        assert_eq!(language("").label, "Text");
696        assert!(!language("console").highlighted);
697        assert_eq!(language("brainfork").label, "brainfork");
698        assert!(!language("brainfork").highlighted);
699        // syntect's own languages work too, under the fence word.
700        assert!(language("go").highlighted);
701    }
702
703    #[test]
704    fn rust_tokens() {
705        let code = "// note\nfn main() { let x = 42; println!(\"hi\"); }";
706        assert!(has("rust", code, "// note", TokenKind::Comment));
707        assert!(has("rust", code, "fn", TokenKind::Keyword));
708        assert!(has("rust", code, "main", TokenKind::Function));
709        assert!(has("rust", code, "42", TokenKind::Number));
710        assert!(has("rust", code, "\"hi\"", TokenKind::String));
711    }
712
713    #[test]
714    fn typescript_tokens() {
715        let code = "interface User { name: string }\nconst n: number = 3; greet(`hi ${n}`); // ok";
716        assert!(has("ts", code, "interface", TokenKind::Keyword));
717        assert!(has("ts", code, "User", TokenKind::Type));
718        assert!(has("ts", code, "greet", TokenKind::Function));
719        assert!(has("ts", code, "`hi ", TokenKind::String));
720        assert!(has("ts", code, "// ok", TokenKind::Comment));
721        let tokens = highlight(code, "ts").unwrap();
722        // The interpolated variable is not part of the string.
723        assert!(
724            !kinds(code, &tokens)
725                .iter()
726                .any(|(t, k)| t.contains("${n}") && *k == TokenKind::String)
727        );
728    }
729
730    #[test]
731    fn toml_and_powershell_tokens() {
732        let toml = "[package]\nname = \"x\" # c\nversion = 2";
733        assert!(has("toml", toml, "package", TokenKind::Heading));
734        assert!(has("toml", toml, "name", TokenKind::Meta));
735        assert!(has("toml", toml, "\"x\"", TokenKind::String));
736        assert!(has("toml", toml, "2", TokenKind::Number));
737        let ps = "Get-ChildItem -Path $env:HOME | Where-Object { $_.Length -gt 10 } # c";
738        assert!(has("powershell", ps, "Get-ChildItem", TokenKind::Function));
739        assert!(has("powershell", ps, "-gt", TokenKind::Keyword));
740        assert!(has("powershell", ps, "$env:HOME", TokenKind::Meta));
741        assert!(has("powershell", ps, "# c", TokenKind::Comment));
742    }
743
744    #[test]
745    fn other_languages_produce_tokens() {
746        for (language, code) in [
747            ("js", "function f() { return 'x' }"),
748            ("json", "{\"a\": 1, \"b\": true}"),
749            ("md", "# Title\n[link](https://x.y)"),
750            ("css", ".a { color: red; }"),
751            ("html", "<div class=\"a\">x</div>"),
752            ("py", "def f():\n    return \"x\"  # c"),
753            ("yaml", "a: 1\nb: \"x\""),
754            ("bash", "echo \"hi\" # c"),
755        ] {
756            let tokens = highlight(code, language).unwrap();
757            assert!(!tokens.is_empty(), "{language} produced no tokens");
758        }
759        assert!(has("md", "# Title", "Title", TokenKind::Heading));
760        assert!(has("py", "def f():\n  pass", "def", TokenKind::Keyword));
761    }
762
763    #[test]
764    fn state_carries_across_lines() {
765        let mut hl = SyntaxHighlighter::new("rust").unwrap();
766        hl.highlight_line("/* open");
767        let tokens = hl.highlight_line("still comment */ fn");
768        assert_eq!(tokens[0].1, TokenKind::Comment);
769        assert_eq!(tokens.last().unwrap().1, TokenKind::Keyword);
770    }
771
772    #[test]
773    fn ranges_stay_inside_the_line_with_crlf() {
774        let code = "let a = 1;\r\nlet b = \"x\";\r\n";
775        let tokens = highlight(code, "rust").unwrap();
776        for (r, _) in &tokens {
777            assert!(!code[r.clone()].contains('\r') && !code[r.clone()].contains('\n'));
778        }
779    }
780
781    #[test]
782    fn cached_streaming_matches_a_full_pass() {
783        let full = "fn a() {\n    \"one\";\n}\n/* two\nlines */ let x = 3;";
784        for end in [5, 12, 20, 30, full.len()] {
785            let part = &full[..end];
786            let cached = highlight_cached(part, "rust").unwrap();
787            let direct = highlight(part, "rust").unwrap();
788            assert_eq!(&*cached, direct.as_slice(), "prefix of {end} bytes");
789        }
790    }
791
792    #[test]
793    fn for_path_uses_the_extension() {
794        assert!(SyntaxHighlighter::for_path("src/main.rs").is_some());
795        assert!(SyntaxHighlighter::for_path("C:\\x\\Cargo.toml").is_some());
796        assert!(SyntaxHighlighter::for_path("README").is_none());
797    }
798}