guise/editor/
highlight.rs

1//! Pluggable, line-based syntax highlighting for the editor.
2//!
3//! A [`Highlighter`] tokenizes one line at a time into byte-range
4//! [`TokenKind`] spans, threading a [`LineState`] through consecutive lines
5//! so block comments carry across them. [`Language`] ships small
6//! keyword/scanner tokenizers for Rust, SQL, and JSON, and [`token_color`]
7//! maps kinds onto the active theme. Ranges are **byte** offsets into the
8//! line (aligned to char boundaries), ready for gpui `TextRun` lengths.
9//!
10//! ```ignore
11//! use guise::editor::{Highlighter, Language, LineState};
12//!
13//! let mut state = LineState::default();
14//! for line in source.lines() {
15//!     let tokens = Language::Rust.line(line, &mut state);
16//!     // tokens: Vec<(Range<usize>, TokenKind)>
17//! }
18//! ```
19
20use std::ops::Range;
21
22use gpui::Hsla;
23
24use crate::theme::{ColorName, Theme};
25
26/// What a token is, for coloring. See [`token_color`].
27#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
28pub enum TokenKind {
29    Keyword,
30    Ident,
31    Number,
32    StringLit,
33    Comment,
34    Punct,
35    Type,
36    Function,
37}
38
39impl TokenKind {
40    /// Every kind, in [`index`](Self::index) order — lets a renderer resolve
41    /// the theme palette once into an array.
42    pub const ALL: [TokenKind; 8] = [
43        TokenKind::Keyword,
44        TokenKind::Ident,
45        TokenKind::Number,
46        TokenKind::StringLit,
47        TokenKind::Comment,
48        TokenKind::Punct,
49        TokenKind::Type,
50        TokenKind::Function,
51    ];
52
53    /// Stable index into [`ALL`](Self::ALL)-sized lookup tables.
54    pub fn index(self) -> usize {
55        self as usize
56    }
57}
58
59/// Tokenizer state carried from one line to the next — currently the open
60/// block-comment depth. Start each document with `LineState::default()`.
61#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
62pub struct LineState {
63    block_depth: u32,
64}
65
66/// Tokenizes one line at a time. `state` carries block-comment continuation
67/// across lines; feed lines in document order.
68pub trait Highlighter {
69    /// Tokenize `text` (a single line, no `\n`). Returned ranges are byte
70    /// offsets into `text`, ascending and non-overlapping; uncovered gaps
71    /// are unstyled.
72    fn line(&self, text: &str, state: &mut LineState) -> Vec<(Range<usize>, TokenKind)>;
73}
74
75/// Whole-document highlighter — the seam for parse-tree backends (the
76/// `treesitter` feature's `TreeSitterHighlighter`). Where [`Highlighter`]
77/// re-scans line by line, an implementation parses the full document once
78/// per edit and serves per-line tokens from that parse. The editor calls
79/// [`update`](Self::update) only when the text changed, never per frame.
80pub trait DocumentHighlighter {
81    /// The document changed; reparse. `text` is the full buffer.
82    fn update(&mut self, text: &str);
83
84    /// Tokens for line `i` of the text last passed to
85    /// [`update`](Self::update): ascending, non-overlapping byte ranges into
86    /// that line, shaped like [`Highlighter::line`] output. Empty for
87    /// out-of-range lines.
88    fn tokens(&self, line: usize) -> &[(Range<usize>, TokenKind)];
89}
90
91/// Built-in languages with keyword/scanner tokenizers.
92#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
93pub enum Language {
94    /// No highlighting — every line is plain text.
95    #[default]
96    None,
97    Rust,
98    Sql,
99    Json,
100    Toml,
101    Python,
102    JavaScript,
103    TypeScript,
104    Go,
105    C,
106    /// Line-structure highlighting: headings, quotes, fences, `code` spans.
107    Markdown,
108}
109
110impl Highlighter for Language {
111    fn line(&self, text: &str, state: &mut LineState) -> Vec<(Range<usize>, TokenKind)> {
112        match self {
113            Language::None => Vec::new(),
114            Language::Rust => tokenize(&RUST, text, state),
115            Language::Sql => tokenize(&SQL, text, state),
116            Language::Json => tokenize(&JSON, text, state),
117            Language::Toml => tokenize(&TOML, text, state),
118            Language::Python => tokenize(&PYTHON, text, state),
119            Language::JavaScript => tokenize(&JAVASCRIPT, text, state),
120            Language::TypeScript => tokenize(&TYPESCRIPT, text, state),
121            Language::Go => tokenize(&GO, text, state),
122            Language::C => tokenize(&C, text, state),
123            Language::Markdown => markdown_line(text, state),
124        }
125    }
126}
127
128/// The theme color for a token kind (light/dark aware). Comments use the
129/// dimmed text color; identifiers and punctuation the normal text color.
130pub fn token_color(kind: TokenKind, t: &Theme) -> Hsla {
131    let shade = if t.scheme.is_dark() { 4 } else { 7 };
132    match kind {
133        TokenKind::Keyword => t.color(ColorName::Violet, shade).hsla(),
134        TokenKind::StringLit => t.color(ColorName::Green, shade).hsla(),
135        TokenKind::Number => t.color(ColorName::Orange, shade).hsla(),
136        TokenKind::Type => t.color(ColorName::Teal, shade).hsla(),
137        TokenKind::Function => t.color(ColorName::Blue, shade).hsla(),
138        TokenKind::Comment => t.dimmed().hsla(),
139        TokenKind::Ident | TokenKind::Punct => t.text().hsla(),
140    }
141}
142
143// ---- scanner ---------------------------------------------------------------
144
145/// How a quoted string escapes its own quote char.
146#[derive(Clone, Copy)]
147enum Escape {
148    /// `\"` (Rust, JSON).
149    Backslash,
150    /// `''` (SQL).
151    Doubled,
152}
153
154/// A language description the shared scanner runs over.
155struct Syntax {
156    line_comment: Option<&'static str>,
157    block_comment: Option<(&'static str, &'static str)>,
158    /// Whether block comments nest (Rust) or not (SQL).
159    nested_blocks: bool,
160    strings: &'static [(char, Escape)],
161    keywords: &'static [&'static str],
162    types: &'static [&'static str],
163    /// Match keywords/types case-insensitively (SQL).
164    case_insensitive: bool,
165    /// Idents starting uppercase are types (Rust).
166    uppercase_types: bool,
167}
168
169const RUST: Syntax = Syntax {
170    line_comment: Some("//"),
171    block_comment: Some(("/*", "*/")),
172    nested_blocks: true,
173    strings: &[('"', Escape::Backslash)],
174    keywords: &[
175        "as", "async", "await", "break", "const", "continue", "crate", "dyn", "else", "enum",
176        "extern", "false", "fn", "for", "if", "impl", "in", "let", "loop", "match", "mod", "move",
177        "mut", "pub", "ref", "return", "self", "Self", "static", "struct", "super", "trait",
178        "true", "type", "union", "unsafe", "use", "where", "while",
179    ],
180    types: &[],
181    case_insensitive: false,
182    uppercase_types: true,
183};
184
185const SQL: Syntax = Syntax {
186    line_comment: Some("--"),
187    block_comment: Some(("/*", "*/")),
188    nested_blocks: false,
189    strings: &[('\'', Escape::Doubled)],
190    keywords: &[
191        "add",
192        "all",
193        "alter",
194        "and",
195        "as",
196        "asc",
197        "begin",
198        "between",
199        "by",
200        "case",
201        "cast",
202        "check",
203        "column",
204        "commit",
205        "constraint",
206        "create",
207        "cross",
208        "default",
209        "delete",
210        "desc",
211        "distinct",
212        "drop",
213        "else",
214        "end",
215        "exists",
216        "false",
217        "foreign",
218        "from",
219        "full",
220        "group",
221        "having",
222        "if",
223        "in",
224        "index",
225        "inner",
226        "insert",
227        "into",
228        "is",
229        "join",
230        "key",
231        "left",
232        "like",
233        "limit",
234        "not",
235        "null",
236        "offset",
237        "on",
238        "or",
239        "order",
240        "outer",
241        "primary",
242        "references",
243        "replace",
244        "returning",
245        "right",
246        "rollback",
247        "select",
248        "set",
249        "table",
250        "then",
251        "transaction",
252        "true",
253        "union",
254        "unique",
255        "update",
256        "values",
257        "view",
258        "when",
259        "where",
260        "with",
261    ],
262    types: &[
263        "bigint",
264        "blob",
265        "bool",
266        "boolean",
267        "bytea",
268        "char",
269        "date",
270        "decimal",
271        "double",
272        "float",
273        "int",
274        "integer",
275        "interval",
276        "json",
277        "jsonb",
278        "numeric",
279        "real",
280        "serial",
281        "smallint",
282        "text",
283        "time",
284        "timestamp",
285        "timestamptz",
286        "uuid",
287        "varchar",
288    ],
289    case_insensitive: true,
290    uppercase_types: false,
291};
292
293const JSON: Syntax = Syntax {
294    line_comment: None,
295    block_comment: None,
296    nested_blocks: false,
297    strings: &[('"', Escape::Backslash)],
298    keywords: &["false", "null", "true"],
299    types: &[],
300    case_insensitive: false,
301    uppercase_types: false,
302};
303
304const TOML: Syntax = Syntax {
305    line_comment: Some("#"),
306    block_comment: None,
307    nested_blocks: false,
308    strings: &[('"', Escape::Backslash), ('\'', Escape::Backslash)],
309    keywords: &["false", "true"],
310    types: &[],
311    case_insensitive: false,
312    uppercase_types: false,
313};
314
315const PYTHON: Syntax = Syntax {
316    line_comment: Some("#"),
317    block_comment: None,
318    nested_blocks: false,
319    strings: &[('"', Escape::Backslash), ('\'', Escape::Backslash)],
320    keywords: &[
321        "False", "None", "True", "and", "as", "assert", "async", "await", "break", "class",
322        "continue", "def", "del", "elif", "else", "except", "finally", "for", "from", "global",
323        "if", "import", "in", "is", "lambda", "match", "nonlocal", "not", "or", "pass", "raise",
324        "return", "try", "while", "with", "yield",
325    ],
326    types: &[
327        "bool", "bytes", "dict", "float", "int", "list", "set", "str", "tuple",
328    ],
329    case_insensitive: false,
330    uppercase_types: true,
331};
332
333const JAVASCRIPT: Syntax = Syntax {
334    line_comment: Some("//"),
335    block_comment: Some(("/*", "*/")),
336    nested_blocks: false,
337    strings: &[
338        ('"', Escape::Backslash),
339        ('\'', Escape::Backslash),
340        ('`', Escape::Backslash),
341    ],
342    keywords: &[
343        "async",
344        "await",
345        "break",
346        "case",
347        "catch",
348        "class",
349        "const",
350        "continue",
351        "debugger",
352        "default",
353        "delete",
354        "do",
355        "else",
356        "export",
357        "extends",
358        "false",
359        "finally",
360        "for",
361        "function",
362        "if",
363        "import",
364        "in",
365        "instanceof",
366        "let",
367        "new",
368        "null",
369        "of",
370        "return",
371        "static",
372        "super",
373        "switch",
374        "this",
375        "throw",
376        "true",
377        "try",
378        "typeof",
379        "undefined",
380        "var",
381        "void",
382        "while",
383        "with",
384        "yield",
385    ],
386    types: &[],
387    case_insensitive: false,
388    uppercase_types: true,
389};
390
391const TYPESCRIPT: Syntax = Syntax {
392    line_comment: Some("//"),
393    block_comment: Some(("/*", "*/")),
394    nested_blocks: false,
395    strings: &[
396        ('"', Escape::Backslash),
397        ('\'', Escape::Backslash),
398        ('`', Escape::Backslash),
399    ],
400    keywords: &[
401        "abstract",
402        "as",
403        "async",
404        "await",
405        "break",
406        "case",
407        "catch",
408        "class",
409        "const",
410        "continue",
411        "debugger",
412        "declare",
413        "default",
414        "delete",
415        "do",
416        "else",
417        "enum",
418        "export",
419        "extends",
420        "false",
421        "finally",
422        "for",
423        "function",
424        "if",
425        "implements",
426        "import",
427        "in",
428        "infer",
429        "instanceof",
430        "interface",
431        "is",
432        "keyof",
433        "let",
434        "namespace",
435        "new",
436        "null",
437        "of",
438        "readonly",
439        "return",
440        "satisfies",
441        "static",
442        "super",
443        "switch",
444        "this",
445        "throw",
446        "true",
447        "try",
448        "type",
449        "typeof",
450        "undefined",
451        "var",
452        "void",
453        "while",
454        "with",
455        "yield",
456    ],
457    types: &[
458        "any", "bigint", "boolean", "never", "number", "object", "string", "symbol", "unknown",
459        "void",
460    ],
461    case_insensitive: false,
462    uppercase_types: true,
463};
464
465const GO: Syntax = Syntax {
466    line_comment: Some("//"),
467    block_comment: Some(("/*", "*/")),
468    nested_blocks: false,
469    strings: &[('"', Escape::Backslash), ('`', Escape::Backslash)],
470    keywords: &[
471        "break",
472        "case",
473        "chan",
474        "const",
475        "continue",
476        "default",
477        "defer",
478        "else",
479        "fallthrough",
480        "false",
481        "for",
482        "func",
483        "go",
484        "goto",
485        "if",
486        "import",
487        "interface",
488        "iota",
489        "map",
490        "nil",
491        "package",
492        "range",
493        "return",
494        "select",
495        "struct",
496        "switch",
497        "true",
498        "type",
499        "var",
500    ],
501    types: &[
502        "any",
503        "bool",
504        "byte",
505        "complex128",
506        "complex64",
507        "error",
508        "float32",
509        "float64",
510        "int",
511        "int16",
512        "int32",
513        "int64",
514        "int8",
515        "rune",
516        "string",
517        "uint",
518        "uint16",
519        "uint32",
520        "uint64",
521        "uint8",
522        "uintptr",
523    ],
524    case_insensitive: false,
525    uppercase_types: true,
526};
527
528const C: Syntax = Syntax {
529    line_comment: Some("//"),
530    block_comment: Some(("/*", "*/")),
531    nested_blocks: false,
532    strings: &[('"', Escape::Backslash), ('\'', Escape::Backslash)],
533    keywords: &[
534        "break", "case", "const", "continue", "default", "do", "else", "enum", "extern", "for",
535        "goto", "if", "inline", "register", "restrict", "return", "sizeof", "static", "struct",
536        "switch", "typedef", "union", "volatile", "while",
537    ],
538    types: &[
539        "bool", "char", "double", "float", "int", "long", "short", "signed", "size_t", "unsigned",
540        "void",
541    ],
542    case_insensitive: false,
543    uppercase_types: false,
544};
545
546/// Markdown is line-structural, not keyword-based, so it gets its own
547/// tokenizer. `LineState::block_depth` doubles as the "inside a code fence"
548/// flag (1 = fenced).
549fn markdown_line(text: &str, state: &mut LineState) -> Vec<(Range<usize>, TokenKind)> {
550    let trimmed = text.trim_start();
551    let indent = text.len() - trimmed.len();
552
553    if trimmed.starts_with("```") || trimmed.starts_with("~~~") {
554        state.block_depth = if state.block_depth > 0 { 0 } else { 1 };
555        return vec![(indent..text.len(), TokenKind::Punct)];
556    }
557    if state.block_depth > 0 {
558        if text.is_empty() {
559            return Vec::new();
560        }
561        return vec![(0..text.len(), TokenKind::StringLit)];
562    }
563    if trimmed.starts_with('#') {
564        return vec![(indent..text.len(), TokenKind::Keyword)];
565    }
566    if trimmed.starts_with('>') {
567        return vec![(indent..text.len(), TokenKind::Comment)];
568    }
569
570    let mut out = Vec::new();
571    // List bullet: "- ", "* ", "+ ", or "1. " — mark just the marker.
572    if trimmed.starts_with("- ") || trimmed.starts_with("* ") || trimmed.starts_with("+ ") {
573        out.push((indent..indent + 1, TokenKind::Punct));
574    } else {
575        let digits = trimmed.chars().take_while(char::is_ascii_digit).count();
576        if digits > 0 && trimmed[digits..].starts_with(". ") {
577            out.push((indent..indent + digits + 1, TokenKind::Punct));
578        }
579    }
580    // Inline `code` spans (ticks included). Unmatched ticks stay plain.
581    let mut open: Option<usize> = None;
582    for (b, c) in text.char_indices() {
583        if c == '`' {
584            match open.take() {
585                Some(start) => out.push((start..b + 1, TokenKind::StringLit)),
586                None => open = Some(b),
587            }
588        }
589    }
590    out.sort_by_key(|(range, _)| range.start);
591    out
592}
593
594/// Run `syntax` over one line. Works on `char_indices` so every emitted
595/// range is char-boundary aligned (multibyte-safe).
596fn tokenize(syntax: &Syntax, text: &str, state: &mut LineState) -> Vec<(Range<usize>, TokenKind)> {
597    let chars: Vec<(usize, char)> = text.char_indices().collect();
598    let n = chars.len();
599    let byte_at = |i: usize| chars.get(i).map(|&(b, _)| b).unwrap_or(text.len());
600    let mut out: Vec<(Range<usize>, TokenKind)> = Vec::new();
601    let mut i = 0;
602
603    // A block comment left open by a previous line swallows the line start.
604    if state.block_depth > 0 {
605        match syntax.block_comment {
606            Some((open, close)) => {
607                let (end, depth) = scan_block(
608                    &chars,
609                    0,
610                    open,
611                    close,
612                    syntax.nested_blocks,
613                    state.block_depth,
614                );
615                state.block_depth = depth;
616                if byte_at(end) > 0 {
617                    out.push((0..byte_at(end), TokenKind::Comment));
618                }
619                i = end;
620            }
621            // Stale state from another language: ignore it.
622            None => state.block_depth = 0,
623        }
624    }
625
626    while i < n {
627        let (b, c) = chars[i];
628        if c.is_whitespace() {
629            i += 1;
630            continue;
631        }
632        if let Some(lc) = syntax.line_comment {
633            if starts_with_at(&chars, i, lc) {
634                out.push((b..text.len(), TokenKind::Comment));
635                break;
636            }
637        }
638        if let Some((open, close)) = syntax.block_comment {
639            if starts_with_at(&chars, i, open) {
640                let after_open = i + open.chars().count();
641                let (end, depth) =
642                    scan_block(&chars, after_open, open, close, syntax.nested_blocks, 1);
643                state.block_depth = depth;
644                out.push((b..byte_at(end), TokenKind::Comment));
645                i = end;
646                continue;
647            }
648        }
649        if let Some(&(_, esc)) = syntax.strings.iter().find(|&&(q, _)| q == c) {
650            let end = scan_string(&chars, i + 1, c, esc);
651            out.push((b..byte_at(end), TokenKind::StringLit));
652            i = end;
653            continue;
654        }
655        if c.is_ascii_digit() {
656            let end = scan_number(&chars, i);
657            out.push((b..byte_at(end), TokenKind::Number));
658            i = end;
659            continue;
660        }
661        if c.is_alphabetic() || c == '_' {
662            let end = scan_ident(&chars, i);
663            let word = &text[b..byte_at(end)];
664            out.push((b..byte_at(end), classify_word(syntax, word, &chars, end)));
665            i = end;
666            continue;
667        }
668        out.push((b..byte_at(i + 1), TokenKind::Punct));
669        i += 1;
670    }
671
672    coalesce(out)
673}
674
675/// Does the char sequence at `i` spell out `pat`?
676fn starts_with_at(chars: &[(usize, char)], i: usize, pat: &str) -> bool {
677    let mut j = i;
678    for p in pat.chars() {
679        match chars.get(j) {
680            Some(&(_, c)) if c == p => j += 1,
681            _ => return false,
682        }
683    }
684    true
685}
686
687/// Scan a block-comment body from `i` at `depth` (>= 1 means inside).
688/// Returns the char index just past the final close, and the depth still
689/// open at the line end (0 = closed).
690fn scan_block(
691    chars: &[(usize, char)],
692    mut i: usize,
693    open: &str,
694    close: &str,
695    nested: bool,
696    mut depth: u32,
697) -> (usize, u32) {
698    let n = chars.len();
699    while i < n {
700        if nested && starts_with_at(chars, i, open) {
701            depth += 1;
702            i += open.chars().count();
703        } else if starts_with_at(chars, i, close) {
704            depth -= 1;
705            i += close.chars().count();
706            if depth == 0 {
707                return (i, 0);
708            }
709        } else {
710            i += 1;
711        }
712    }
713    (n, depth)
714}
715
716/// Scan a string body from `i` (just past the opening quote). Returns the
717/// char index just past the closing quote, or the line end if unterminated
718/// (strings do not continue across lines).
719fn scan_string(chars: &[(usize, char)], mut i: usize, quote: char, esc: Escape) -> usize {
720    let n = chars.len();
721    while i < n {
722        let c = chars[i].1;
723        match esc {
724            Escape::Backslash if c == '\\' => {
725                i += 2;
726                continue;
727            }
728            Escape::Doubled if c == quote => {
729                if i + 1 < n && chars[i + 1].1 == quote {
730                    i += 2;
731                    continue;
732                }
733                return i + 1;
734            }
735            _ if c == quote => return i + 1,
736            _ => i += 1,
737        }
738    }
739    n
740}
741
742/// Scan a number from `i` (a digit): integers, `0x`/`0b`/`0o` prefixes,
743/// decimals, exponents, and trailing type suffixes (`1u8`, `2.5f64`).
744fn scan_number(chars: &[(usize, char)], mut i: usize) -> usize {
745    let n = chars.len();
746    if chars[i].1 == '0' && i + 1 < n && matches!(chars[i + 1].1, 'x' | 'X' | 'b' | 'B' | 'o' | 'O')
747    {
748        i += 2;
749        while i < n && (chars[i].1.is_ascii_alphanumeric() || chars[i].1 == '_') {
750            i += 1;
751        }
752        return i;
753    }
754    while i < n && (chars[i].1.is_ascii_digit() || chars[i].1 == '_') {
755        i += 1;
756    }
757    if i + 1 < n && chars[i].1 == '.' && chars[i + 1].1.is_ascii_digit() {
758        i += 1;
759        while i < n && (chars[i].1.is_ascii_digit() || chars[i].1 == '_') {
760            i += 1;
761        }
762    }
763    if i < n && matches!(chars[i].1, 'e' | 'E') {
764        let mut j = i + 1;
765        if j < n && matches!(chars[j].1, '+' | '-') {
766            j += 1;
767        }
768        if j < n && chars[j].1.is_ascii_digit() {
769            i = j;
770            while i < n && chars[i].1.is_ascii_digit() {
771                i += 1;
772            }
773        }
774    }
775    while i < n && (chars[i].1.is_ascii_alphanumeric() || chars[i].1 == '_') {
776        i += 1;
777    }
778    i
779}
780
781/// Scan an identifier from `i` (a letter or `_`).
782fn scan_ident(chars: &[(usize, char)], mut i: usize) -> usize {
783    let n = chars.len();
784    while i < n && (chars[i].1.is_alphanumeric() || chars[i].1 == '_') {
785        i += 1;
786    }
787    i
788}
789
790/// Keyword / type / function-call / plain ident, in that priority. `end` is
791/// the char index just past the word, for call-site lookahead.
792fn classify_word(syntax: &Syntax, word: &str, chars: &[(usize, char)], end: usize) -> TokenKind {
793    let in_set = |set: &[&str]| {
794        if syntax.case_insensitive {
795            set.iter().any(|k| k.eq_ignore_ascii_case(word))
796        } else {
797            set.contains(&word)
798        }
799    };
800    if in_set(syntax.keywords) {
801        return TokenKind::Keyword;
802    }
803    if in_set(syntax.types) {
804        return TokenKind::Type;
805    }
806    if syntax.uppercase_types && word.chars().next().is_some_and(char::is_uppercase) {
807        return TokenKind::Type;
808    }
809    // `name(` is a call; `name!(` a macro invocation.
810    match chars.get(end).map(|&(_, c)| c) {
811        Some('(') => TokenKind::Function,
812        Some('!') if matches!(chars.get(end + 1), Some(&(_, '('))) => TokenKind::Function,
813        _ => TokenKind::Ident,
814    }
815}
816
817/// Merge adjacent tokens of the same kind with contiguous ranges, so a run
818/// of punctuation becomes one span.
819pub(crate) fn coalesce(tokens: Vec<(Range<usize>, TokenKind)>) -> Vec<(Range<usize>, TokenKind)> {
820    let mut out: Vec<(Range<usize>, TokenKind)> = Vec::new();
821    for (range, kind) in tokens {
822        if let Some((last, last_kind)) = out.last_mut() {
823            if *last_kind == kind && last.end == range.start {
824                last.end = range.end;
825                continue;
826            }
827        }
828        out.push((range, kind));
829    }
830    out
831}
832
833#[cfg(test)]
834mod tests {
835    use super::*;
836
837    fn kinds(lang: Language, line: &str) -> Vec<(String, TokenKind)> {
838        let mut state = LineState::default();
839        lang.line(line, &mut state)
840            .into_iter()
841            .map(|(r, k)| (line[r].to_string(), k))
842            .collect()
843    }
844
845    fn kind_of(lang: Language, line: &str, word: &str) -> TokenKind {
846        kinds(lang, line)
847            .into_iter()
848            .find(|(w, _)| w == word)
849            .map(|(_, k)| k)
850            .unwrap_or_else(|| panic!("token {word:?} not found in {line:?}"))
851    }
852
853    #[test]
854    fn none_language_emits_nothing() {
855        assert!(kinds(Language::None, "let x = 1;").is_empty());
856    }
857
858    #[test]
859    fn new_languages_classify_keywords_strings_comments() {
860        assert_eq!(
861            kind_of(Language::Toml, "name = \"guise\" # crate", "\"guise\""),
862            TokenKind::StringLit
863        );
864        assert_eq!(
865            kind_of(Language::Toml, "flag = true", "true"),
866            TokenKind::Keyword
867        );
868        assert_eq!(
869            kind_of(Language::Python, "def run(): pass  # go", "def"),
870            TokenKind::Keyword
871        );
872        assert_eq!(
873            kind_of(Language::Python, "def run(): pass  # go", "# go"),
874            TokenKind::Comment
875        );
876        assert_eq!(
877            kind_of(Language::JavaScript, "const x = `hi`;", "const"),
878            TokenKind::Keyword
879        );
880        assert_eq!(
881            kind_of(Language::JavaScript, "const x = `hi`;", "`hi`"),
882            TokenKind::StringLit
883        );
884        assert_eq!(
885            kind_of(Language::TypeScript, "let n: number = 5;", "number"),
886            TokenKind::Type
887        );
888        assert_eq!(
889            kind_of(Language::TypeScript, "interface A {}", "interface"),
890            TokenKind::Keyword
891        );
892        assert_eq!(
893            kind_of(Language::Go, "func main() {}", "func"),
894            TokenKind::Keyword
895        );
896        assert_eq!(
897            kind_of(Language::Go, "var n int64", "int64"),
898            TokenKind::Type
899        );
900        assert_eq!(
901            kind_of(Language::C, "static int n = 0; // c", "static"),
902            TokenKind::Keyword
903        );
904        assert_eq!(
905            kind_of(Language::C, "static int n = 0; // c", "int"),
906            TokenKind::Type
907        );
908    }
909
910    #[test]
911    fn markdown_structures_lines() {
912        assert_eq!(
913            kinds(Language::Markdown, "# Title"),
914            vec![("# Title".into(), TokenKind::Keyword)]
915        );
916        assert_eq!(
917            kinds(Language::Markdown, "> quoted"),
918            vec![("> quoted".into(), TokenKind::Comment)]
919        );
920        let bullets = kinds(Language::Markdown, "- item with `code` span");
921        assert_eq!(bullets[0], ("-".into(), TokenKind::Punct));
922        assert_eq!(bullets[1], ("`code`".into(), TokenKind::StringLit));
923        let ordered = kinds(Language::Markdown, "12. step");
924        assert_eq!(ordered[0], ("12.".into(), TokenKind::Punct));
925        // Unmatched ticks stay plain.
926        assert!(kinds(Language::Markdown, "just a ` tick").is_empty());
927    }
928
929    #[test]
930    fn markdown_fences_carry_state() {
931        let mut state = LineState::default();
932        let fence = Language::Markdown.line("```rust", &mut state);
933        assert_eq!(fence[0].1, TokenKind::Punct);
934        let inside = Language::Markdown.line("# not a heading", &mut state);
935        assert_eq!(
936            inside,
937            vec![(0.."# not a heading".len(), TokenKind::StringLit)]
938        );
939        Language::Markdown.line("```", &mut state);
940        let after = Language::Markdown.line("# heading again", &mut state);
941        assert_eq!(after[0].1, TokenKind::Keyword);
942    }
943
944    #[test]
945    fn ranges_are_ascending_and_in_bounds() {
946        let line = "let s = \"héllo\"; // café";
947        let mut state = LineState::default();
948        let tokens = Language::Rust.line(line, &mut state);
949        let mut at = 0;
950        for (range, _) in &tokens {
951            assert!(range.start >= at, "overlapping range");
952            assert!(range.end <= line.len());
953            assert!(line.is_char_boundary(range.start));
954            assert!(line.is_char_boundary(range.end));
955            at = range.end;
956        }
957    }
958
959    #[test]
960    fn rust_basics() {
961        assert_eq!(
962            kind_of(Language::Rust, "let x = 1;", "let"),
963            TokenKind::Keyword
964        );
965        assert_eq!(kind_of(Language::Rust, "let x = 1;", "x"), TokenKind::Ident);
966        assert_eq!(
967            kind_of(Language::Rust, "let x = 10.5e3;", "10.5e3"),
968            TokenKind::Number
969        );
970        assert_eq!(
971            kind_of(Language::Rust, "let n = 0xff_u8;", "0xff_u8"),
972            TokenKind::Number
973        );
974        assert_eq!(
975            kind_of(
976                Language::Rust,
977                r#"let s = "hi \" there";"#,
978                r#""hi \" there""#
979            ),
980            TokenKind::StringLit
981        );
982        assert_eq!(
983            kind_of(Language::Rust, "let v: Vec<u8>;", "Vec"),
984            TokenKind::Type
985        );
986        assert_eq!(
987            kind_of(Language::Rust, "foo(1)", "foo"),
988            TokenKind::Function
989        );
990        assert_eq!(
991            kind_of(Language::Rust, "println!(\"x\")", "println"),
992            TokenKind::Function
993        );
994        assert_eq!(
995            kind_of(Language::Rust, "a + b // sum", "// sum"),
996            TokenKind::Comment
997        );
998    }
999
1000    #[test]
1001    fn rust_block_comment_carries_state() {
1002        let mut state = LineState::default();
1003        let t1 = Language::Rust.line("start /* open", &mut state);
1004        assert_eq!(
1005            t1.last()
1006                .map(|(r, k)| ("start /* open"[r.clone()].to_string(), *k)),
1007            Some(("/* open".to_string(), TokenKind::Comment))
1008        );
1009        let t2 = Language::Rust.line("all comment", &mut state);
1010        assert_eq!(t2, vec![(0..11, TokenKind::Comment)]);
1011        let t3 = Language::Rust.line("done */ let x", &mut state);
1012        assert_eq!(t3[0], (0..7, TokenKind::Comment));
1013        assert_eq!("done */ let x"[t3[1].clone().0].to_string(), "let");
1014        assert_eq!(state, LineState::default());
1015    }
1016
1017    #[test]
1018    fn rust_block_comments_nest() {
1019        let mut state = LineState::default();
1020        Language::Rust.line("/* a /* b */ still", &mut state);
1021        assert_ne!(state, LineState::default());
1022        let t = Language::Rust.line("c */ code", &mut state);
1023        assert_eq!(t[0], (0..4, TokenKind::Comment));
1024        assert_eq!(state, LineState::default());
1025    }
1026
1027    #[test]
1028    fn sql_keywords_are_case_insensitive() {
1029        assert_eq!(
1030            kind_of(Language::Sql, "SELECT * FROM users;", "SELECT"),
1031            TokenKind::Keyword
1032        );
1033        assert_eq!(
1034            kind_of(Language::Sql, "select * from users;", "select"),
1035            TokenKind::Keyword
1036        );
1037        assert_eq!(
1038            kind_of(Language::Sql, "id INT PRIMARY KEY", "INT"),
1039            TokenKind::Type
1040        );
1041        assert_eq!(
1042            kind_of(Language::Sql, "count(*)", "count"),
1043            TokenKind::Function
1044        );
1045    }
1046
1047    #[test]
1048    fn sql_comments_and_strings() {
1049        assert_eq!(
1050            kind_of(Language::Sql, "x -- note", "-- note"),
1051            TokenKind::Comment
1052        );
1053        assert_eq!(
1054            kind_of(Language::Sql, "name = 'it''s'", "'it''s'"),
1055            TokenKind::StringLit
1056        );
1057        let mut state = LineState::default();
1058        Language::Sql.line("/* multi", &mut state);
1059        let t = Language::Sql.line("line */ SELECT", &mut state);
1060        assert_eq!(t[0], (0..7, TokenKind::Comment));
1061        assert_eq!(state, LineState::default());
1062    }
1063
1064    #[test]
1065    fn sql_blocks_do_not_nest() {
1066        let mut state = LineState::default();
1067        Language::Sql.line("/* a /* b */ tail", &mut state);
1068        // The first `*/` closes the whole comment.
1069        assert_eq!(state, LineState::default());
1070    }
1071
1072    #[test]
1073    fn json_tokens() {
1074        let line = r#"{"key": [1.5, true, null, "value"]}"#;
1075        assert_eq!(
1076            kind_of(Language::Json, line, r#""key""#),
1077            TokenKind::StringLit
1078        );
1079        assert_eq!(kind_of(Language::Json, line, "1.5"), TokenKind::Number);
1080        assert_eq!(kind_of(Language::Json, line, "true"), TokenKind::Keyword);
1081        assert_eq!(kind_of(Language::Json, line, "null"), TokenKind::Keyword);
1082    }
1083
1084    #[test]
1085    fn multibyte_idents_and_strings() {
1086        let line = "let café = \"日本語\"; // été";
1087        assert_eq!(kind_of(Language::Rust, line, "café"), TokenKind::Ident);
1088        assert_eq!(
1089            kind_of(Language::Rust, line, "\"日本語\""),
1090            TokenKind::StringLit
1091        );
1092        assert_eq!(kind_of(Language::Rust, line, "// été"), TokenKind::Comment);
1093    }
1094
1095    #[test]
1096    fn unterminated_string_stops_at_line_end() {
1097        let line = "let s = \"open";
1098        assert_eq!(
1099            kind_of(Language::Rust, line, "\"open"),
1100            TokenKind::StringLit
1101        );
1102        // ...and does not leak into the next line.
1103        let mut state = LineState::default();
1104        Language::Rust.line(line, &mut state);
1105        assert_eq!(state, LineState::default());
1106    }
1107
1108    #[test]
1109    fn punctuation_coalesces() {
1110        let tokens = kinds(Language::Rust, "a->b");
1111        assert_eq!(
1112            tokens,
1113            vec![
1114                ("a".to_string(), TokenKind::Ident),
1115                ("->".to_string(), TokenKind::Punct),
1116                ("b".to_string(), TokenKind::Ident),
1117            ]
1118        );
1119    }
1120
1121    #[test]
1122    fn empty_line_inside_block_comment() {
1123        let mut state = LineState::default();
1124        Language::Rust.line("/* open", &mut state);
1125        let t = Language::Rust.line("", &mut state);
1126        assert!(t.is_empty());
1127        assert_ne!(state, LineState::default());
1128    }
1129}
guise/editor/highlight.rs

guise/editor/
highlight.rs
