guise/editor/
highlight.rs

1//! Pluggable, line-based syntax highlighting for the editor.
2//!
3//! A [`Highlighter`] tokenizes one line at a time into byte-range
4//! [`TokenKind`] spans, threading a [`LineState`] through consecutive lines
5//! so block comments carry across them. [`Language`] ships small
6//! keyword/scanner tokenizers for Rust, SQL, and JSON, and [`token_color`]
7//! maps kinds onto the active theme. Ranges are **byte** offsets into the
8//! line (aligned to char boundaries), ready for gpui `TextRun` lengths.
9//!
10//! ```ignore
11//! use guise::editor::{Highlighter, Language, LineState};
12//!
13//! let mut state = LineState::default();
14//! for line in source.lines() {
15//!     let tokens = Language::Rust.line(line, &mut state);
16//!     // tokens: Vec<(Range<usize>, TokenKind)>
17//! }
18//! ```
19
20use std::ops::Range;
21
22use gpui::Hsla;
23
24use crate::theme::{ColorName, Theme};
25
26/// What a token is, for coloring. See [`token_color`].
27#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
28pub enum TokenKind {
29    Keyword,
30    Ident,
31    Number,
32    StringLit,
33    Comment,
34    Punct,
35    Type,
36    Function,
37}
38
39impl TokenKind {
40    /// Every kind, in [`index`](Self::index) order — lets a renderer resolve
41    /// the theme palette once into an array.
42    pub const ALL: [TokenKind; 8] = [
43        TokenKind::Keyword,
44        TokenKind::Ident,
45        TokenKind::Number,
46        TokenKind::StringLit,
47        TokenKind::Comment,
48        TokenKind::Punct,
49        TokenKind::Type,
50        TokenKind::Function,
51    ];
52
53    /// Stable index into [`ALL`](Self::ALL)-sized lookup tables.
54    pub fn index(self) -> usize {
55        self as usize
56    }
57}
58
59/// Tokenizer state carried from one line to the next — currently the open
60/// block-comment depth. Start each document with `LineState::default()`.
61#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
62pub struct LineState {
63    block_depth: u32,
64}
65
66/// Tokenizes one line at a time. `state` carries block-comment continuation
67/// across lines; feed lines in document order.
68pub trait Highlighter {
69    /// Tokenize `text` (a single line, no `\n`). Returned ranges are byte
70    /// offsets into `text`, ascending and non-overlapping; uncovered gaps
71    /// are unstyled.
72    fn line(&self, text: &str, state: &mut LineState) -> Vec<(Range<usize>, TokenKind)>;
73}
74
75/// Built-in languages with keyword/scanner tokenizers.
76#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
77pub enum Language {
78    /// No highlighting — every line is plain text.
79    #[default]
80    None,
81    Rust,
82    Sql,
83    Json,
84}
85
86impl Highlighter for Language {
87    fn line(&self, text: &str, state: &mut LineState) -> Vec<(Range<usize>, TokenKind)> {
88        match self {
89            Language::None => Vec::new(),
90            Language::Rust => tokenize(&RUST, text, state),
91            Language::Sql => tokenize(&SQL, text, state),
92            Language::Json => tokenize(&JSON, text, state),
93        }
94    }
95}
96
97/// The theme color for a token kind (light/dark aware). Comments use the
98/// dimmed text color; identifiers and punctuation the normal text color.
99pub fn token_color(kind: TokenKind, t: &Theme) -> Hsla {
100    let shade = if t.scheme.is_dark() { 4 } else { 7 };
101    match kind {
102        TokenKind::Keyword => t.color(ColorName::Violet, shade).hsla(),
103        TokenKind::StringLit => t.color(ColorName::Green, shade).hsla(),
104        TokenKind::Number => t.color(ColorName::Orange, shade).hsla(),
105        TokenKind::Type => t.color(ColorName::Teal, shade).hsla(),
106        TokenKind::Function => t.color(ColorName::Blue, shade).hsla(),
107        TokenKind::Comment => t.dimmed().hsla(),
108        TokenKind::Ident | TokenKind::Punct => t.text().hsla(),
109    }
110}
111
112// ---- scanner ---------------------------------------------------------------
113
114/// How a quoted string escapes its own quote char.
115#[derive(Clone, Copy)]
116enum Escape {
117    /// `\"` (Rust, JSON).
118    Backslash,
119    /// `''` (SQL).
120    Doubled,
121}
122
123/// A language description the shared scanner runs over.
124struct Syntax {
125    line_comment: Option<&'static str>,
126    block_comment: Option<(&'static str, &'static str)>,
127    /// Whether block comments nest (Rust) or not (SQL).
128    nested_blocks: bool,
129    strings: &'static [(char, Escape)],
130    keywords: &'static [&'static str],
131    types: &'static [&'static str],
132    /// Match keywords/types case-insensitively (SQL).
133    case_insensitive: bool,
134    /// Idents starting uppercase are types (Rust).
135    uppercase_types: bool,
136}
137
138const RUST: Syntax = Syntax {
139    line_comment: Some("//"),
140    block_comment: Some(("/*", "*/")),
141    nested_blocks: true,
142    strings: &[('"', Escape::Backslash)],
143    keywords: &[
144        "as", "async", "await", "break", "const", "continue", "crate", "dyn", "else", "enum",
145        "extern", "false", "fn", "for", "if", "impl", "in", "let", "loop", "match", "mod", "move",
146        "mut", "pub", "ref", "return", "self", "Self", "static", "struct", "super", "trait",
147        "true", "type", "union", "unsafe", "use", "where", "while",
148    ],
149    types: &[],
150    case_insensitive: false,
151    uppercase_types: true,
152};
153
154const SQL: Syntax = Syntax {
155    line_comment: Some("--"),
156    block_comment: Some(("/*", "*/")),
157    nested_blocks: false,
158    strings: &[('\'', Escape::Doubled)],
159    keywords: &[
160        "add",
161        "all",
162        "alter",
163        "and",
164        "as",
165        "asc",
166        "begin",
167        "between",
168        "by",
169        "case",
170        "cast",
171        "check",
172        "column",
173        "commit",
174        "constraint",
175        "create",
176        "cross",
177        "default",
178        "delete",
179        "desc",
180        "distinct",
181        "drop",
182        "else",
183        "end",
184        "exists",
185        "false",
186        "foreign",
187        "from",
188        "full",
189        "group",
190        "having",
191        "if",
192        "in",
193        "index",
194        "inner",
195        "insert",
196        "into",
197        "is",
198        "join",
199        "key",
200        "left",
201        "like",
202        "limit",
203        "not",
204        "null",
205        "offset",
206        "on",
207        "or",
208        "order",
209        "outer",
210        "primary",
211        "references",
212        "replace",
213        "returning",
214        "right",
215        "rollback",
216        "select",
217        "set",
218        "table",
219        "then",
220        "transaction",
221        "true",
222        "union",
223        "unique",
224        "update",
225        "values",
226        "view",
227        "when",
228        "where",
229        "with",
230    ],
231    types: &[
232        "bigint",
233        "blob",
234        "bool",
235        "boolean",
236        "bytea",
237        "char",
238        "date",
239        "decimal",
240        "double",
241        "float",
242        "int",
243        "integer",
244        "interval",
245        "json",
246        "jsonb",
247        "numeric",
248        "real",
249        "serial",
250        "smallint",
251        "text",
252        "time",
253        "timestamp",
254        "timestamptz",
255        "uuid",
256        "varchar",
257    ],
258    case_insensitive: true,
259    uppercase_types: false,
260};
261
262const JSON: Syntax = Syntax {
263    line_comment: None,
264    block_comment: None,
265    nested_blocks: false,
266    strings: &[('"', Escape::Backslash)],
267    keywords: &["false", "null", "true"],
268    types: &[],
269    case_insensitive: false,
270    uppercase_types: false,
271};
272
273/// Run `syntax` over one line. Works on `char_indices` so every emitted
274/// range is char-boundary aligned (multibyte-safe).
275fn tokenize(syntax: &Syntax, text: &str, state: &mut LineState) -> Vec<(Range<usize>, TokenKind)> {
276    let chars: Vec<(usize, char)> = text.char_indices().collect();
277    let n = chars.len();
278    let byte_at = |i: usize| chars.get(i).map(|&(b, _)| b).unwrap_or(text.len());
279    let mut out: Vec<(Range<usize>, TokenKind)> = Vec::new();
280    let mut i = 0;
281
282    // A block comment left open by a previous line swallows the line start.
283    if state.block_depth > 0 {
284        match syntax.block_comment {
285            Some((open, close)) => {
286                let (end, depth) = scan_block(
287                    &chars,
288                    0,
289                    open,
290                    close,
291                    syntax.nested_blocks,
292                    state.block_depth,
293                );
294                state.block_depth = depth;
295                if byte_at(end) > 0 {
296                    out.push((0..byte_at(end), TokenKind::Comment));
297                }
298                i = end;
299            }
300            // Stale state from another language: ignore it.
301            None => state.block_depth = 0,
302        }
303    }
304
305    while i < n {
306        let (b, c) = chars[i];
307        if c.is_whitespace() {
308            i += 1;
309            continue;
310        }
311        if let Some(lc) = syntax.line_comment {
312            if starts_with_at(&chars, i, lc) {
313                out.push((b..text.len(), TokenKind::Comment));
314                break;
315            }
316        }
317        if let Some((open, close)) = syntax.block_comment {
318            if starts_with_at(&chars, i, open) {
319                let after_open = i + open.chars().count();
320                let (end, depth) =
321                    scan_block(&chars, after_open, open, close, syntax.nested_blocks, 1);
322                state.block_depth = depth;
323                out.push((b..byte_at(end), TokenKind::Comment));
324                i = end;
325                continue;
326            }
327        }
328        if let Some(&(_, esc)) = syntax.strings.iter().find(|&&(q, _)| q == c) {
329            let end = scan_string(&chars, i + 1, c, esc);
330            out.push((b..byte_at(end), TokenKind::StringLit));
331            i = end;
332            continue;
333        }
334        if c.is_ascii_digit() {
335            let end = scan_number(&chars, i);
336            out.push((b..byte_at(end), TokenKind::Number));
337            i = end;
338            continue;
339        }
340        if c.is_alphabetic() || c == '_' {
341            let end = scan_ident(&chars, i);
342            let word = &text[b..byte_at(end)];
343            out.push((b..byte_at(end), classify_word(syntax, word, &chars, end)));
344            i = end;
345            continue;
346        }
347        out.push((b..byte_at(i + 1), TokenKind::Punct));
348        i += 1;
349    }
350
351    coalesce(out)
352}
353
354/// Does the char sequence at `i` spell out `pat`?
355fn starts_with_at(chars: &[(usize, char)], i: usize, pat: &str) -> bool {
356    let mut j = i;
357    for p in pat.chars() {
358        match chars.get(j) {
359            Some(&(_, c)) if c == p => j += 1,
360            _ => return false,
361        }
362    }
363    true
364}
365
366/// Scan a block-comment body from `i` at `depth` (>= 1 means inside).
367/// Returns the char index just past the final close, and the depth still
368/// open at the line end (0 = closed).
369fn scan_block(
370    chars: &[(usize, char)],
371    mut i: usize,
372    open: &str,
373    close: &str,
374    nested: bool,
375    mut depth: u32,
376) -> (usize, u32) {
377    let n = chars.len();
378    while i < n {
379        if nested && starts_with_at(chars, i, open) {
380            depth += 1;
381            i += open.chars().count();
382        } else if starts_with_at(chars, i, close) {
383            depth -= 1;
384            i += close.chars().count();
385            if depth == 0 {
386                return (i, 0);
387            }
388        } else {
389            i += 1;
390        }
391    }
392    (n, depth)
393}
394
395/// Scan a string body from `i` (just past the opening quote). Returns the
396/// char index just past the closing quote, or the line end if unterminated
397/// (strings do not continue across lines).
398fn scan_string(chars: &[(usize, char)], mut i: usize, quote: char, esc: Escape) -> usize {
399    let n = chars.len();
400    while i < n {
401        let c = chars[i].1;
402        match esc {
403            Escape::Backslash if c == '\\' => {
404                i += 2;
405                continue;
406            }
407            Escape::Doubled if c == quote => {
408                if i + 1 < n && chars[i + 1].1 == quote {
409                    i += 2;
410                    continue;
411                }
412                return i + 1;
413            }
414            _ if c == quote => return i + 1,
415            _ => i += 1,
416        }
417    }
418    n
419}
420
421/// Scan a number from `i` (a digit): integers, `0x`/`0b`/`0o` prefixes,
422/// decimals, exponents, and trailing type suffixes (`1u8`, `2.5f64`).
423fn scan_number(chars: &[(usize, char)], mut i: usize) -> usize {
424    let n = chars.len();
425    if chars[i].1 == '0' && i + 1 < n && matches!(chars[i + 1].1, 'x' | 'X' | 'b' | 'B' | 'o' | 'O')
426    {
427        i += 2;
428        while i < n && (chars[i].1.is_ascii_alphanumeric() || chars[i].1 == '_') {
429            i += 1;
430        }
431        return i;
432    }
433    while i < n && (chars[i].1.is_ascii_digit() || chars[i].1 == '_') {
434        i += 1;
435    }
436    if i + 1 < n && chars[i].1 == '.' && chars[i + 1].1.is_ascii_digit() {
437        i += 1;
438        while i < n && (chars[i].1.is_ascii_digit() || chars[i].1 == '_') {
439            i += 1;
440        }
441    }
442    if i < n && matches!(chars[i].1, 'e' | 'E') {
443        let mut j = i + 1;
444        if j < n && matches!(chars[j].1, '+' | '-') {
445            j += 1;
446        }
447        if j < n && chars[j].1.is_ascii_digit() {
448            i = j;
449            while i < n && chars[i].1.is_ascii_digit() {
450                i += 1;
451            }
452        }
453    }
454    while i < n && (chars[i].1.is_ascii_alphanumeric() || chars[i].1 == '_') {
455        i += 1;
456    }
457    i
458}
459
460/// Scan an identifier from `i` (a letter or `_`).
461fn scan_ident(chars: &[(usize, char)], mut i: usize) -> usize {
462    let n = chars.len();
463    while i < n && (chars[i].1.is_alphanumeric() || chars[i].1 == '_') {
464        i += 1;
465    }
466    i
467}
468
469/// Keyword / type / function-call / plain ident, in that priority. `end` is
470/// the char index just past the word, for call-site lookahead.
471fn classify_word(syntax: &Syntax, word: &str, chars: &[(usize, char)], end: usize) -> TokenKind {
472    let in_set = |set: &[&str]| {
473        if syntax.case_insensitive {
474            set.iter().any(|k| k.eq_ignore_ascii_case(word))
475        } else {
476            set.contains(&word)
477        }
478    };
479    if in_set(syntax.keywords) {
480        return TokenKind::Keyword;
481    }
482    if in_set(syntax.types) {
483        return TokenKind::Type;
484    }
485    if syntax.uppercase_types && word.chars().next().is_some_and(char::is_uppercase) {
486        return TokenKind::Type;
487    }
488    // `name(` is a call; `name!(` a macro invocation.
489    match chars.get(end).map(|&(_, c)| c) {
490        Some('(') => TokenKind::Function,
491        Some('!') if matches!(chars.get(end + 1), Some(&(_, '('))) => TokenKind::Function,
492        _ => TokenKind::Ident,
493    }
494}
495
496/// Merge adjacent tokens of the same kind with contiguous ranges, so a run
497/// of punctuation becomes one span.
498fn coalesce(tokens: Vec<(Range<usize>, TokenKind)>) -> Vec<(Range<usize>, TokenKind)> {
499    let mut out: Vec<(Range<usize>, TokenKind)> = Vec::new();
500    for (range, kind) in tokens {
501        if let Some((last, last_kind)) = out.last_mut() {
502            if *last_kind == kind && last.end == range.start {
503                last.end = range.end;
504                continue;
505            }
506        }
507        out.push((range, kind));
508    }
509    out
510}
511
512#[cfg(test)]
513mod tests {
514    use super::*;
515
516    fn kinds(lang: Language, line: &str) -> Vec<(String, TokenKind)> {
517        let mut state = LineState::default();
518        lang.line(line, &mut state)
519            .into_iter()
520            .map(|(r, k)| (line[r].to_string(), k))
521            .collect()
522    }
523
524    fn kind_of(lang: Language, line: &str, word: &str) -> TokenKind {
525        kinds(lang, line)
526            .into_iter()
527            .find(|(w, _)| w == word)
528            .map(|(_, k)| k)
529            .unwrap_or_else(|| panic!("token {word:?} not found in {line:?}"))
530    }
531
532    #[test]
533    fn none_language_emits_nothing() {
534        assert!(kinds(Language::None, "let x = 1;").is_empty());
535    }
536
537    #[test]
538    fn ranges_are_ascending_and_in_bounds() {
539        let line = "let s = \"héllo\"; // café";
540        let mut state = LineState::default();
541        let tokens = Language::Rust.line(line, &mut state);
542        let mut at = 0;
543        for (range, _) in &tokens {
544            assert!(range.start >= at, "overlapping range");
545            assert!(range.end <= line.len());
546            assert!(line.is_char_boundary(range.start));
547            assert!(line.is_char_boundary(range.end));
548            at = range.end;
549        }
550    }
551
552    #[test]
553    fn rust_basics() {
554        assert_eq!(
555            kind_of(Language::Rust, "let x = 1;", "let"),
556            TokenKind::Keyword
557        );
558        assert_eq!(kind_of(Language::Rust, "let x = 1;", "x"), TokenKind::Ident);
559        assert_eq!(
560            kind_of(Language::Rust, "let x = 10.5e3;", "10.5e3"),
561            TokenKind::Number
562        );
563        assert_eq!(
564            kind_of(Language::Rust, "let n = 0xff_u8;", "0xff_u8"),
565            TokenKind::Number
566        );
567        assert_eq!(
568            kind_of(
569                Language::Rust,
570                r#"let s = "hi \" there";"#,
571                r#""hi \" there""#
572            ),
573            TokenKind::StringLit
574        );
575        assert_eq!(
576            kind_of(Language::Rust, "let v: Vec<u8>;", "Vec"),
577            TokenKind::Type
578        );
579        assert_eq!(
580            kind_of(Language::Rust, "foo(1)", "foo"),
581            TokenKind::Function
582        );
583        assert_eq!(
584            kind_of(Language::Rust, "println!(\"x\")", "println"),
585            TokenKind::Function
586        );
587        assert_eq!(
588            kind_of(Language::Rust, "a + b // sum", "// sum"),
589            TokenKind::Comment
590        );
591    }
592
593    #[test]
594    fn rust_block_comment_carries_state() {
595        let mut state = LineState::default();
596        let t1 = Language::Rust.line("start /* open", &mut state);
597        assert_eq!(
598            t1.last()
599                .map(|(r, k)| ("start /* open"[r.clone()].to_string(), *k)),
600            Some(("/* open".to_string(), TokenKind::Comment))
601        );
602        let t2 = Language::Rust.line("all comment", &mut state);
603        assert_eq!(t2, vec![(0..11, TokenKind::Comment)]);
604        let t3 = Language::Rust.line("done */ let x", &mut state);
605        assert_eq!(t3[0], (0..7, TokenKind::Comment));
606        assert_eq!("done */ let x"[t3[1].clone().0].to_string(), "let");
607        assert_eq!(state, LineState::default());
608    }
609
610    #[test]
611    fn rust_block_comments_nest() {
612        let mut state = LineState::default();
613        Language::Rust.line("/* a /* b */ still", &mut state);
614        assert_ne!(state, LineState::default());
615        let t = Language::Rust.line("c */ code", &mut state);
616        assert_eq!(t[0], (0..4, TokenKind::Comment));
617        assert_eq!(state, LineState::default());
618    }
619
620    #[test]
621    fn sql_keywords_are_case_insensitive() {
622        assert_eq!(
623            kind_of(Language::Sql, "SELECT * FROM users;", "SELECT"),
624            TokenKind::Keyword
625        );
626        assert_eq!(
627            kind_of(Language::Sql, "select * from users;", "select"),
628            TokenKind::Keyword
629        );
630        assert_eq!(
631            kind_of(Language::Sql, "id INT PRIMARY KEY", "INT"),
632            TokenKind::Type
633        );
634        assert_eq!(
635            kind_of(Language::Sql, "count(*)", "count"),
636            TokenKind::Function
637        );
638    }
639
640    #[test]
641    fn sql_comments_and_strings() {
642        assert_eq!(
643            kind_of(Language::Sql, "x -- note", "-- note"),
644            TokenKind::Comment
645        );
646        assert_eq!(
647            kind_of(Language::Sql, "name = 'it''s'", "'it''s'"),
648            TokenKind::StringLit
649        );
650        let mut state = LineState::default();
651        Language::Sql.line("/* multi", &mut state);
652        let t = Language::Sql.line("line */ SELECT", &mut state);
653        assert_eq!(t[0], (0..7, TokenKind::Comment));
654        assert_eq!(state, LineState::default());
655    }
656
657    #[test]
658    fn sql_blocks_do_not_nest() {
659        let mut state = LineState::default();
660        Language::Sql.line("/* a /* b */ tail", &mut state);
661        // The first `*/` closes the whole comment.
662        assert_eq!(state, LineState::default());
663    }
664
665    #[test]
666    fn json_tokens() {
667        let line = r#"{"key": [1.5, true, null, "value"]}"#;
668        assert_eq!(
669            kind_of(Language::Json, line, r#""key""#),
670            TokenKind::StringLit
671        );
672        assert_eq!(kind_of(Language::Json, line, "1.5"), TokenKind::Number);
673        assert_eq!(kind_of(Language::Json, line, "true"), TokenKind::Keyword);
674        assert_eq!(kind_of(Language::Json, line, "null"), TokenKind::Keyword);
675    }
676
677    #[test]
678    fn multibyte_idents_and_strings() {
679        let line = "let café = \"日本語\"; // été";
680        assert_eq!(kind_of(Language::Rust, line, "café"), TokenKind::Ident);
681        assert_eq!(
682            kind_of(Language::Rust, line, "\"日本語\""),
683            TokenKind::StringLit
684        );
685        assert_eq!(kind_of(Language::Rust, line, "// été"), TokenKind::Comment);
686    }
687
688    #[test]
689    fn unterminated_string_stops_at_line_end() {
690        let line = "let s = \"open";
691        assert_eq!(
692            kind_of(Language::Rust, line, "\"open"),
693            TokenKind::StringLit
694        );
695        // ...and does not leak into the next line.
696        let mut state = LineState::default();
697        Language::Rust.line(line, &mut state);
698        assert_eq!(state, LineState::default());
699    }
700
701    #[test]
702    fn punctuation_coalesces() {
703        let tokens = kinds(Language::Rust, "a->b");
704        assert_eq!(
705            tokens,
706            vec![
707                ("a".to_string(), TokenKind::Ident),
708                ("->".to_string(), TokenKind::Punct),
709                ("b".to_string(), TokenKind::Ident),
710            ]
711        );
712    }
713
714    #[test]
715    fn empty_line_inside_block_comment() {
716        let mut state = LineState::default();
717        Language::Rust.line("/* open", &mut state);
718        let t = Language::Rust.line("", &mut state);
719        assert!(t.is_empty());
720        assert_ne!(state, LineState::default());
721    }
722}
guise/editor/highlight.rs

guise/editor/
highlight.rs
