guise/editor/
highlight.rs

1//! Pluggable, line-based syntax highlighting for the editor.
2//!
3//! A [`Highlighter`] tokenizes one line at a time into byte-range
4//! [`TokenKind`] spans, threading a [`LineState`] through consecutive lines
5//! so block comments carry across them. [`Language`] ships small
6//! keyword/scanner tokenizers for Rust, SQL, and JSON, and [`token_color`]
7//! maps kinds onto the active theme. Ranges are **byte** offsets into the
8//! line (aligned to char boundaries), ready for gpui `TextRun` lengths.
9//!
10//! ```ignore
11//! use guise::editor::{Highlighter, Language, LineState};
12//!
13//! let mut state = LineState::default();
14//! for line in source.lines() {
15//!     let tokens = Language::Rust.line(line, &mut state);
16//!     // tokens: Vec<(Range<usize>, TokenKind)>
17//! }
18//! ```
19
20use std::ops::Range;
21
22use gpui::Hsla;
23
24use crate::theme::{ColorName, Theme};
25
26/// What a token is, for coloring. See [`token_color`].
27#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
28pub enum TokenKind {
29  Keyword,
30  Ident,
31  Number,
32  StringLit,
33  Comment,
34  Punct,
35  Type,
36  Function,
37}
38
39impl TokenKind {
40  /// Every kind, in [`index`](Self::index) order — lets a renderer resolve
41  /// the theme palette once into an array.
42  pub const ALL: [TokenKind; 8] = [
43    TokenKind::Keyword,
44    TokenKind::Ident,
45    TokenKind::Number,
46    TokenKind::StringLit,
47    TokenKind::Comment,
48    TokenKind::Punct,
49    TokenKind::Type,
50    TokenKind::Function,
51  ];
52
53  /// Stable index into [`ALL`](Self::ALL)-sized lookup tables.
54  pub fn index(self) -> usize {
55    self as usize
56  }
57}
58
59/// Tokenizer state carried from one line to the next — currently the open
60/// block-comment depth. Start each document with `LineState::default()`.
61#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
62pub struct LineState {
63  block_depth: u32,
64}
65
66/// Tokenizes one line at a time. `state` carries block-comment continuation
67/// across lines; feed lines in document order.
68pub trait Highlighter {
69  /// Tokenize `text` (a single line, no `\n`). Returned ranges are byte
70  /// offsets into `text`, ascending and non-overlapping; uncovered gaps
71  /// are unstyled.
72  fn line(&self, text: &str, state: &mut LineState) -> Vec<(Range<usize>, TokenKind)>;
73}
74
75/// Whole-document highlighter — the seam for parse-tree backends (the
76/// `treesitter` feature's `TreeSitterHighlighter`). Where [`Highlighter`]
77/// re-scans line by line, an implementation parses the full document once
78/// per edit and serves per-line tokens from that parse. The editor calls
79/// [`update`](Self::update) only when the text changed, never per frame.
80pub trait DocumentHighlighter {
81  /// The document changed; reparse. `text` is the full buffer.
82  fn update(&mut self, text: &str);
83
84  /// Tokens for line `i` of the text last passed to
85  /// [`update`](Self::update): ascending, non-overlapping byte ranges into
86  /// that line, shaped like [`Highlighter::line`] output. Empty for
87  /// out-of-range lines.
88  fn tokens(&self, line: usize) -> &[(Range<usize>, TokenKind)];
89}
90
91/// Built-in languages with keyword/scanner tokenizers.
92#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
93pub enum Language {
94  /// No highlighting — every line is plain text.
95  #[default]
96  None,
97  Rust,
98  Sql,
99  Json,
100  Toml,
101  Python,
102  JavaScript,
103  TypeScript,
104  Go,
105  C,
106  /// Line-structure highlighting: headings, quotes, fences, `code` spans.
107  Markdown,
108}
109
110impl Highlighter for Language {
111  fn line(&self, text: &str, state: &mut LineState) -> Vec<(Range<usize>, TokenKind)> {
112    match self {
113      Language::None => Vec::new(),
114      Language::Rust => tokenize(&RUST, text, state),
115      Language::Sql => tokenize(&SQL, text, state),
116      Language::Json => tokenize(&JSON, text, state),
117      Language::Toml => tokenize(&TOML, text, state),
118      Language::Python => tokenize(&PYTHON, text, state),
119      Language::JavaScript => tokenize(&JAVASCRIPT, text, state),
120      Language::TypeScript => tokenize(&TYPESCRIPT, text, state),
121      Language::Go => tokenize(&GO, text, state),
122      Language::C => tokenize(&C, text, state),
123      Language::Markdown => markdown_line(text, state),
124    }
125  }
126}
127
128/// The theme color for a token kind (light/dark aware). Comments use the
129/// dimmed text color; identifiers and punctuation the normal text color.
130pub fn token_color(kind: TokenKind, t: &Theme) -> Hsla {
131  let shade = if t.scheme.is_dark() { 4 } else { 7 };
132  match kind {
133    TokenKind::Keyword => t.color(ColorName::Violet, shade).hsla(),
134    TokenKind::StringLit => t.color(ColorName::Green, shade).hsla(),
135    TokenKind::Number => t.color(ColorName::Orange, shade).hsla(),
136    TokenKind::Type => t.color(ColorName::Teal, shade).hsla(),
137    TokenKind::Function => t.color(ColorName::Blue, shade).hsla(),
138    TokenKind::Comment => t.dimmed().hsla(),
139    TokenKind::Ident | TokenKind::Punct => t.text().hsla(),
140  }
141}
142
143// ---- scanner ---------------------------------------------------------------
144
145/// How a quoted string escapes its own quote char.
146#[derive(Clone, Copy)]
147enum Escape {
148  /// `\"` (Rust, JSON).
149  Backslash,
150  /// `''` (SQL).
151  Doubled,
152}
153
154/// A language description the shared scanner runs over.
155struct Syntax {
156  line_comment: Option<&'static str>,
157  block_comment: Option<(&'static str, &'static str)>,
158  /// Whether block comments nest (Rust) or not (SQL).
159  nested_blocks: bool,
160  strings: &'static [(char, Escape)],
161  keywords: &'static [&'static str],
162  types: &'static [&'static str],
163  /// Match keywords/types case-insensitively (SQL).
164  case_insensitive: bool,
165  /// Idents starting uppercase are types (Rust).
166  uppercase_types: bool,
167}
168
169const RUST: Syntax = Syntax {
170  line_comment: Some("//"),
171  block_comment: Some(("/*", "*/")),
172  nested_blocks: true,
173  strings: &[('"', Escape::Backslash)],
174  keywords: &[
175    "as", "async", "await", "break", "const", "continue", "crate", "dyn", "else", "enum", "extern",
176    "false", "fn", "for", "if", "impl", "in", "let", "loop", "match", "mod", "move", "mut", "pub",
177    "ref", "return", "self", "Self", "static", "struct", "super", "trait", "true", "type", "union",
178    "unsafe", "use", "where", "while",
179  ],
180  types: &[],
181  case_insensitive: false,
182  uppercase_types: true,
183};
184
185const SQL: Syntax = Syntax {
186  line_comment: Some("--"),
187  block_comment: Some(("/*", "*/")),
188  nested_blocks: false,
189  strings: &[('\'', Escape::Doubled)],
190  keywords: &[
191    "add",
192    "all",
193    "alter",
194    "and",
195    "as",
196    "asc",
197    "begin",
198    "between",
199    "by",
200    "case",
201    "cast",
202    "check",
203    "column",
204    "commit",
205    "constraint",
206    "create",
207    "cross",
208    "default",
209    "delete",
210    "desc",
211    "distinct",
212    "drop",
213    "else",
214    "end",
215    "exists",
216    "false",
217    "foreign",
218    "from",
219    "full",
220    "group",
221    "having",
222    "if",
223    "in",
224    "index",
225    "inner",
226    "insert",
227    "into",
228    "is",
229    "join",
230    "key",
231    "left",
232    "like",
233    "limit",
234    "not",
235    "null",
236    "offset",
237    "on",
238    "or",
239    "order",
240    "outer",
241    "primary",
242    "references",
243    "replace",
244    "returning",
245    "right",
246    "rollback",
247    "select",
248    "set",
249    "table",
250    "then",
251    "transaction",
252    "true",
253    "union",
254    "unique",
255    "update",
256    "values",
257    "view",
258    "when",
259    "where",
260    "with",
261  ],
262  types: &[
263    "bigint",
264    "blob",
265    "bool",
266    "boolean",
267    "bytea",
268    "char",
269    "date",
270    "decimal",
271    "double",
272    "float",
273    "int",
274    "integer",
275    "interval",
276    "json",
277    "jsonb",
278    "numeric",
279    "real",
280    "serial",
281    "smallint",
282    "text",
283    "time",
284    "timestamp",
285    "timestamptz",
286    "uuid",
287    "varchar",
288  ],
289  case_insensitive: true,
290  uppercase_types: false,
291};
292
293const JSON: Syntax = Syntax {
294  line_comment: None,
295  block_comment: None,
296  nested_blocks: false,
297  strings: &[('"', Escape::Backslash)],
298  keywords: &["false", "null", "true"],
299  types: &[],
300  case_insensitive: false,
301  uppercase_types: false,
302};
303
304const TOML: Syntax = Syntax {
305  line_comment: Some("#"),
306  block_comment: None,
307  nested_blocks: false,
308  strings: &[('"', Escape::Backslash), ('\'', Escape::Backslash)],
309  keywords: &["false", "true"],
310  types: &[],
311  case_insensitive: false,
312  uppercase_types: false,
313};
314
315const PYTHON: Syntax = Syntax {
316  line_comment: Some("#"),
317  block_comment: None,
318  nested_blocks: false,
319  strings: &[('"', Escape::Backslash), ('\'', Escape::Backslash)],
320  keywords: &[
321    "False", "None", "True", "and", "as", "assert", "async", "await", "break", "class", "continue",
322    "def", "del", "elif", "else", "except", "finally", "for", "from", "global", "if", "import",
323    "in", "is", "lambda", "match", "nonlocal", "not", "or", "pass", "raise", "return", "try",
324    "while", "with", "yield",
325  ],
326  types: &[
327    "bool", "bytes", "dict", "float", "int", "list", "set", "str", "tuple",
328  ],
329  case_insensitive: false,
330  uppercase_types: true,
331};
332
333const JAVASCRIPT: Syntax = Syntax {
334  line_comment: Some("//"),
335  block_comment: Some(("/*", "*/")),
336  nested_blocks: false,
337  strings: &[
338    ('"', Escape::Backslash),
339    ('\'', Escape::Backslash),
340    ('`', Escape::Backslash),
341  ],
342  keywords: &[
343    "async",
344    "await",
345    "break",
346    "case",
347    "catch",
348    "class",
349    "const",
350    "continue",
351    "debugger",
352    "default",
353    "delete",
354    "do",
355    "else",
356    "export",
357    "extends",
358    "false",
359    "finally",
360    "for",
361    "function",
362    "if",
363    "import",
364    "in",
365    "instanceof",
366    "let",
367    "new",
368    "null",
369    "of",
370    "return",
371    "static",
372    "super",
373    "switch",
374    "this",
375    "throw",
376    "true",
377    "try",
378    "typeof",
379    "undefined",
380    "var",
381    "void",
382    "while",
383    "with",
384    "yield",
385  ],
386  types: &[],
387  case_insensitive: false,
388  uppercase_types: true,
389};
390
391const TYPESCRIPT: Syntax = Syntax {
392  line_comment: Some("//"),
393  block_comment: Some(("/*", "*/")),
394  nested_blocks: false,
395  strings: &[
396    ('"', Escape::Backslash),
397    ('\'', Escape::Backslash),
398    ('`', Escape::Backslash),
399  ],
400  keywords: &[
401    "abstract",
402    "as",
403    "async",
404    "await",
405    "break",
406    "case",
407    "catch",
408    "class",
409    "const",
410    "continue",
411    "debugger",
412    "declare",
413    "default",
414    "delete",
415    "do",
416    "else",
417    "enum",
418    "export",
419    "extends",
420    "false",
421    "finally",
422    "for",
423    "function",
424    "if",
425    "implements",
426    "import",
427    "in",
428    "infer",
429    "instanceof",
430    "interface",
431    "is",
432    "keyof",
433    "let",
434    "namespace",
435    "new",
436    "null",
437    "of",
438    "readonly",
439    "return",
440    "satisfies",
441    "static",
442    "super",
443    "switch",
444    "this",
445    "throw",
446    "true",
447    "try",
448    "type",
449    "typeof",
450    "undefined",
451    "var",
452    "void",
453    "while",
454    "with",
455    "yield",
456  ],
457  types: &[
458    "any", "bigint", "boolean", "never", "number", "object", "string", "symbol", "unknown", "void",
459  ],
460  case_insensitive: false,
461  uppercase_types: true,
462};
463
464const GO: Syntax = Syntax {
465  line_comment: Some("//"),
466  block_comment: Some(("/*", "*/")),
467  nested_blocks: false,
468  strings: &[('"', Escape::Backslash), ('`', Escape::Backslash)],
469  keywords: &[
470    "break",
471    "case",
472    "chan",
473    "const",
474    "continue",
475    "default",
476    "defer",
477    "else",
478    "fallthrough",
479    "false",
480    "for",
481    "func",
482    "go",
483    "goto",
484    "if",
485    "import",
486    "interface",
487    "iota",
488    "map",
489    "nil",
490    "package",
491    "range",
492    "return",
493    "select",
494    "struct",
495    "switch",
496    "true",
497    "type",
498    "var",
499  ],
500  types: &[
501    "any",
502    "bool",
503    "byte",
504    "complex128",
505    "complex64",
506    "error",
507    "float32",
508    "float64",
509    "int",
510    "int16",
511    "int32",
512    "int64",
513    "int8",
514    "rune",
515    "string",
516    "uint",
517    "uint16",
518    "uint32",
519    "uint64",
520    "uint8",
521    "uintptr",
522  ],
523  case_insensitive: false,
524  uppercase_types: true,
525};
526
527const C: Syntax = Syntax {
528  line_comment: Some("//"),
529  block_comment: Some(("/*", "*/")),
530  nested_blocks: false,
531  strings: &[('"', Escape::Backslash), ('\'', Escape::Backslash)],
532  keywords: &[
533    "break", "case", "const", "continue", "default", "do", "else", "enum", "extern", "for", "goto",
534    "if", "inline", "register", "restrict", "return", "sizeof", "static", "struct", "switch",
535    "typedef", "union", "volatile", "while",
536  ],
537  types: &[
538    "bool", "char", "double", "float", "int", "long", "short", "signed", "size_t", "unsigned",
539    "void",
540  ],
541  case_insensitive: false,
542  uppercase_types: false,
543};
544
545/// Markdown is line-structural, not keyword-based, so it gets its own
546/// tokenizer. `LineState::block_depth` doubles as the "inside a code fence"
547/// flag (1 = fenced).
548fn markdown_line(text: &str, state: &mut LineState) -> Vec<(Range<usize>, TokenKind)> {
549  let trimmed = text.trim_start();
550  let indent = text.len() - trimmed.len();
551
552  if trimmed.starts_with("```") || trimmed.starts_with("~~~") {
553    state.block_depth = if state.block_depth > 0 { 0 } else { 1 };
554    return vec![(indent..text.len(), TokenKind::Punct)];
555  }
556  if state.block_depth > 0 {
557    if text.is_empty() {
558      return Vec::new();
559    }
560    return vec![(0..text.len(), TokenKind::StringLit)];
561  }
562  if trimmed.starts_with('#') {
563    return vec![(indent..text.len(), TokenKind::Keyword)];
564  }
565  if trimmed.starts_with('>') {
566    return vec![(indent..text.len(), TokenKind::Comment)];
567  }
568
569  let mut out = Vec::new();
570  // List bullet: "- ", "* ", "+ ", or "1. " — mark just the marker.
571  if trimmed.starts_with("- ") || trimmed.starts_with("* ") || trimmed.starts_with("+ ") {
572    out.push((indent..indent + 1, TokenKind::Punct));
573  } else {
574    let digits = trimmed.chars().take_while(char::is_ascii_digit).count();
575    if digits > 0 && trimmed[digits..].starts_with(". ") {
576      out.push((indent..indent + digits + 1, TokenKind::Punct));
577    }
578  }
579  // Inline `code` spans (ticks included). Unmatched ticks stay plain.
580  let mut open: Option<usize> = None;
581  for (b, c) in text.char_indices() {
582    if c == '`' {
583      match open.take() {
584        Some(start) => out.push((start..b + 1, TokenKind::StringLit)),
585        None => open = Some(b),
586      }
587    }
588  }
589  out.sort_by_key(|(range, _)| range.start);
590  out
591}
592
593/// Run `syntax` over one line. Works on `char_indices` so every emitted
594/// range is char-boundary aligned (multibyte-safe).
595fn tokenize(syntax: &Syntax, text: &str, state: &mut LineState) -> Vec<(Range<usize>, TokenKind)> {
596  let chars: Vec<(usize, char)> = text.char_indices().collect();
597  let n = chars.len();
598  let byte_at = |i: usize| chars.get(i).map(|&(b, _)| b).unwrap_or(text.len());
599  let mut out: Vec<(Range<usize>, TokenKind)> = Vec::new();
600  let mut i = 0;
601
602  // A block comment left open by a previous line swallows the line start.
603  if state.block_depth > 0 {
604    match syntax.block_comment {
605      Some((open, close)) => {
606        let (end, depth) = scan_block(
607          &chars,
608          0,
609          open,
610          close,
611          syntax.nested_blocks,
612          state.block_depth,
613        );
614        state.block_depth = depth;
615        if byte_at(end) > 0 {
616          out.push((0..byte_at(end), TokenKind::Comment));
617        }
618        i = end;
619      }
620      // Stale state from another language: ignore it.
621      None => state.block_depth = 0,
622    }
623  }
624
625  while i < n {
626    let (b, c) = chars[i];
627    if c.is_whitespace() {
628      i += 1;
629      continue;
630    }
631    if let Some(lc) = syntax.line_comment {
632      if starts_with_at(&chars, i, lc) {
633        out.push((b..text.len(), TokenKind::Comment));
634        break;
635      }
636    }
637    if let Some((open, close)) = syntax.block_comment {
638      if starts_with_at(&chars, i, open) {
639        let after_open = i + open.chars().count();
640        let (end, depth) = scan_block(&chars, after_open, open, close, syntax.nested_blocks, 1);
641        state.block_depth = depth;
642        out.push((b..byte_at(end), TokenKind::Comment));
643        i = end;
644        continue;
645      }
646    }
647    if let Some(&(_, esc)) = syntax.strings.iter().find(|&&(q, _)| q == c) {
648      let end = scan_string(&chars, i + 1, c, esc);
649      out.push((b..byte_at(end), TokenKind::StringLit));
650      i = end;
651      continue;
652    }
653    if c.is_ascii_digit() {
654      let end = scan_number(&chars, i);
655      out.push((b..byte_at(end), TokenKind::Number));
656      i = end;
657      continue;
658    }
659    if c.is_alphabetic() || c == '_' {
660      let end = scan_ident(&chars, i);
661      let word = &text[b..byte_at(end)];
662      out.push((b..byte_at(end), classify_word(syntax, word, &chars, end)));
663      i = end;
664      continue;
665    }
666    out.push((b..byte_at(i + 1), TokenKind::Punct));
667    i += 1;
668  }
669
670  coalesce(out)
671}
672
673/// Does the char sequence at `i` spell out `pat`?
674fn starts_with_at(chars: &[(usize, char)], i: usize, pat: &str) -> bool {
675  let mut j = i;
676  for p in pat.chars() {
677    match chars.get(j) {
678      Some(&(_, c)) if c == p => j += 1,
679      _ => return false,
680    }
681  }
682  true
683}
684
685/// Scan a block-comment body from `i` at `depth` (>= 1 means inside).
686/// Returns the char index just past the final close, and the depth still
687/// open at the line end (0 = closed).
688fn scan_block(
689  chars: &[(usize, char)],
690  mut i: usize,
691  open: &str,
692  close: &str,
693  nested: bool,
694  mut depth: u32,
695) -> (usize, u32) {
696  let n = chars.len();
697  while i < n {
698    if nested && starts_with_at(chars, i, open) {
699      depth += 1;
700      i += open.chars().count();
701    } else if starts_with_at(chars, i, close) {
702      depth -= 1;
703      i += close.chars().count();
704      if depth == 0 {
705        return (i, 0);
706      }
707    } else {
708      i += 1;
709    }
710  }
711  (n, depth)
712}
713
714/// Scan a string body from `i` (just past the opening quote). Returns the
715/// char index just past the closing quote, or the line end if unterminated
716/// (strings do not continue across lines).
717fn scan_string(chars: &[(usize, char)], mut i: usize, quote: char, esc: Escape) -> usize {
718  let n = chars.len();
719  while i < n {
720    let c = chars[i].1;
721    match esc {
722      Escape::Backslash if c == '\\' => {
723        i += 2;
724        continue;
725      }
726      Escape::Doubled if c == quote => {
727        if i + 1 < n && chars[i + 1].1 == quote {
728          i += 2;
729          continue;
730        }
731        return i + 1;
732      }
733      _ if c == quote => return i + 1,
734      _ => i += 1,
735    }
736  }
737  n
738}
739
740/// Scan a number from `i` (a digit): integers, `0x`/`0b`/`0o` prefixes,
741/// decimals, exponents, and trailing type suffixes (`1u8`, `2.5f64`).
742fn scan_number(chars: &[(usize, char)], mut i: usize) -> usize {
743  let n = chars.len();
744  if chars[i].1 == '0' && i + 1 < n && matches!(chars[i + 1].1, 'x' | 'X' | 'b' | 'B' | 'o' | 'O') {
745    i += 2;
746    while i < n && (chars[i].1.is_ascii_alphanumeric() || chars[i].1 == '_') {
747      i += 1;
748    }
749    return i;
750  }
751  while i < n && (chars[i].1.is_ascii_digit() || chars[i].1 == '_') {
752    i += 1;
753  }
754  if i + 1 < n && chars[i].1 == '.' && chars[i + 1].1.is_ascii_digit() {
755    i += 1;
756    while i < n && (chars[i].1.is_ascii_digit() || chars[i].1 == '_') {
757      i += 1;
758    }
759  }
760  if i < n && matches!(chars[i].1, 'e' | 'E') {
761    let mut j = i + 1;
762    if j < n && matches!(chars[j].1, '+' | '-') {
763      j += 1;
764    }
765    if j < n && chars[j].1.is_ascii_digit() {
766      i = j;
767      while i < n && chars[i].1.is_ascii_digit() {
768        i += 1;
769      }
770    }
771  }
772  while i < n && (chars[i].1.is_ascii_alphanumeric() || chars[i].1 == '_') {
773    i += 1;
774  }
775  i
776}
777
778/// Scan an identifier from `i` (a letter or `_`).
779fn scan_ident(chars: &[(usize, char)], mut i: usize) -> usize {
780  let n = chars.len();
781  while i < n && (chars[i].1.is_alphanumeric() || chars[i].1 == '_') {
782    i += 1;
783  }
784  i
785}
786
787/// Keyword / type / function-call / plain ident, in that priority. `end` is
788/// the char index just past the word, for call-site lookahead.
789fn classify_word(syntax: &Syntax, word: &str, chars: &[(usize, char)], end: usize) -> TokenKind {
790  let in_set = |set: &[&str]| {
791    if syntax.case_insensitive {
792      set.iter().any(|k| k.eq_ignore_ascii_case(word))
793    } else {
794      set.contains(&word)
795    }
796  };
797  if in_set(syntax.keywords) {
798    return TokenKind::Keyword;
799  }
800  if in_set(syntax.types) {
801    return TokenKind::Type;
802  }
803  if syntax.uppercase_types && word.chars().next().is_some_and(char::is_uppercase) {
804    return TokenKind::Type;
805  }
806  // `name(` is a call; `name!(` a macro invocation.
807  match chars.get(end).map(|&(_, c)| c) {
808    Some('(') => TokenKind::Function,
809    Some('!') if matches!(chars.get(end + 1), Some(&(_, '('))) => TokenKind::Function,
810    _ => TokenKind::Ident,
811  }
812}
813
814/// Merge adjacent tokens of the same kind with contiguous ranges, so a run
815/// of punctuation becomes one span.
816pub(crate) fn coalesce(tokens: Vec<(Range<usize>, TokenKind)>) -> Vec<(Range<usize>, TokenKind)> {
817  let mut out: Vec<(Range<usize>, TokenKind)> = Vec::new();
818  for (range, kind) in tokens {
819    if let Some((last, last_kind)) = out.last_mut() {
820      if *last_kind == kind && last.end == range.start {
821        last.end = range.end;
822        continue;
823      }
824    }
825    out.push((range, kind));
826  }
827  out
828}
829
830#[cfg(test)]
831mod tests {
832  use super::*;
833
834  fn kinds(lang: Language, line: &str) -> Vec<(String, TokenKind)> {
835    let mut state = LineState::default();
836    lang
837      .line(line, &mut state)
838      .into_iter()
839      .map(|(r, k)| (line[r].to_string(), k))
840      .collect()
841  }
842
843  fn kind_of(lang: Language, line: &str, word: &str) -> TokenKind {
844    kinds(lang, line)
845      .into_iter()
846      .find(|(w, _)| w == word)
847      .map(|(_, k)| k)
848      .unwrap_or_else(|| panic!("token {word:?} not found in {line:?}"))
849  }
850
851  #[test]
852  fn none_language_emits_nothing() {
853    assert!(kinds(Language::None, "let x = 1;").is_empty());
854  }
855
856  #[test]
857  fn new_languages_classify_keywords_strings_comments() {
858    assert_eq!(
859      kind_of(Language::Toml, "name = \"guise\" # crate", "\"guise\""),
860      TokenKind::StringLit
861    );
862    assert_eq!(
863      kind_of(Language::Toml, "flag = true", "true"),
864      TokenKind::Keyword
865    );
866    assert_eq!(
867      kind_of(Language::Python, "def run(): pass  # go", "def"),
868      TokenKind::Keyword
869    );
870    assert_eq!(
871      kind_of(Language::Python, "def run(): pass  # go", "# go"),
872      TokenKind::Comment
873    );
874    assert_eq!(
875      kind_of(Language::JavaScript, "const x = `hi`;", "const"),
876      TokenKind::Keyword
877    );
878    assert_eq!(
879      kind_of(Language::JavaScript, "const x = `hi`;", "`hi`"),
880      TokenKind::StringLit
881    );
882    assert_eq!(
883      kind_of(Language::TypeScript, "let n: number = 5;", "number"),
884      TokenKind::Type
885    );
886    assert_eq!(
887      kind_of(Language::TypeScript, "interface A {}", "interface"),
888      TokenKind::Keyword
889    );
890    assert_eq!(
891      kind_of(Language::Go, "func main() {}", "func"),
892      TokenKind::Keyword
893    );
894    assert_eq!(
895      kind_of(Language::Go, "var n int64", "int64"),
896      TokenKind::Type
897    );
898    assert_eq!(
899      kind_of(Language::C, "static int n = 0; // c", "static"),
900      TokenKind::Keyword
901    );
902    assert_eq!(
903      kind_of(Language::C, "static int n = 0; // c", "int"),
904      TokenKind::Type
905    );
906  }
907
908  #[test]
909  fn markdown_structures_lines() {
910    assert_eq!(
911      kinds(Language::Markdown, "# Title"),
912      vec![("# Title".into(), TokenKind::Keyword)]
913    );
914    assert_eq!(
915      kinds(Language::Markdown, "> quoted"),
916      vec![("> quoted".into(), TokenKind::Comment)]
917    );
918    let bullets = kinds(Language::Markdown, "- item with `code` span");
919    assert_eq!(bullets[0], ("-".into(), TokenKind::Punct));
920    assert_eq!(bullets[1], ("`code`".into(), TokenKind::StringLit));
921    let ordered = kinds(Language::Markdown, "12. step");
922    assert_eq!(ordered[0], ("12.".into(), TokenKind::Punct));
923    // Unmatched ticks stay plain.
924    assert!(kinds(Language::Markdown, "just a ` tick").is_empty());
925  }
926
927  #[test]
928  fn markdown_fences_carry_state() {
929    let mut state = LineState::default();
930    let fence = Language::Markdown.line("```rust", &mut state);
931    assert_eq!(fence[0].1, TokenKind::Punct);
932    let inside = Language::Markdown.line("# not a heading", &mut state);
933    assert_eq!(
934      inside,
935      vec![(0.."# not a heading".len(), TokenKind::StringLit)]
936    );
937    Language::Markdown.line("```", &mut state);
938    let after = Language::Markdown.line("# heading again", &mut state);
939    assert_eq!(after[0].1, TokenKind::Keyword);
940  }
941
942  #[test]
943  fn ranges_are_ascending_and_in_bounds() {
944    let line = "let s = \"héllo\"; // café";
945    let mut state = LineState::default();
946    let tokens = Language::Rust.line(line, &mut state);
947    let mut at = 0;
948    for (range, _) in &tokens {
949      assert!(range.start >= at, "overlapping range");
950      assert!(range.end <= line.len());
951      assert!(line.is_char_boundary(range.start));
952      assert!(line.is_char_boundary(range.end));
953      at = range.end;
954    }
955  }
956
957  #[test]
958  fn rust_basics() {
959    assert_eq!(
960      kind_of(Language::Rust, "let x = 1;", "let"),
961      TokenKind::Keyword
962    );
963    assert_eq!(kind_of(Language::Rust, "let x = 1;", "x"), TokenKind::Ident);
964    assert_eq!(
965      kind_of(Language::Rust, "let x = 10.5e3;", "10.5e3"),
966      TokenKind::Number
967    );
968    assert_eq!(
969      kind_of(Language::Rust, "let n = 0xff_u8;", "0xff_u8"),
970      TokenKind::Number
971    );
972    assert_eq!(
973      kind_of(
974        Language::Rust,
975        r#"let s = "hi \" there";"#,
976        r#""hi \" there""#
977      ),
978      TokenKind::StringLit
979    );
980    assert_eq!(
981      kind_of(Language::Rust, "let v: Vec<u8>;", "Vec"),
982      TokenKind::Type
983    );
984    assert_eq!(
985      kind_of(Language::Rust, "foo(1)", "foo"),
986      TokenKind::Function
987    );
988    assert_eq!(
989      kind_of(Language::Rust, "println!(\"x\")", "println"),
990      TokenKind::Function
991    );
992    assert_eq!(
993      kind_of(Language::Rust, "a + b // sum", "// sum"),
994      TokenKind::Comment
995    );
996  }
997
998  #[test]
999  fn rust_block_comment_carries_state() {
1000    let mut state = LineState::default();
1001    let t1 = Language::Rust.line("start /* open", &mut state);
1002    assert_eq!(
1003      t1.last()
1004        .map(|(r, k)| ("start /* open"[r.clone()].to_string(), *k)),
1005      Some(("/* open".to_string(), TokenKind::Comment))
1006    );
1007    let t2 = Language::Rust.line("all comment", &mut state);
1008    assert_eq!(t2, vec![(0..11, TokenKind::Comment)]);
1009    let t3 = Language::Rust.line("done */ let x", &mut state);
1010    assert_eq!(t3[0], (0..7, TokenKind::Comment));
1011    assert_eq!("done */ let x"[t3[1].clone().0].to_string(), "let");
1012    assert_eq!(state, LineState::default());
1013  }
1014
1015  #[test]
1016  fn rust_block_comments_nest() {
1017    let mut state = LineState::default();
1018    Language::Rust.line("/* a /* b */ still", &mut state);
1019    assert_ne!(state, LineState::default());
1020    let t = Language::Rust.line("c */ code", &mut state);
1021    assert_eq!(t[0], (0..4, TokenKind::Comment));
1022    assert_eq!(state, LineState::default());
1023  }
1024
1025  #[test]
1026  fn sql_keywords_are_case_insensitive() {
1027    assert_eq!(
1028      kind_of(Language::Sql, "SELECT * FROM users;", "SELECT"),
1029      TokenKind::Keyword
1030    );
1031    assert_eq!(
1032      kind_of(Language::Sql, "select * from users;", "select"),
1033      TokenKind::Keyword
1034    );
1035    assert_eq!(
1036      kind_of(Language::Sql, "id INT PRIMARY KEY", "INT"),
1037      TokenKind::Type
1038    );
1039    assert_eq!(
1040      kind_of(Language::Sql, "count(*)", "count"),
1041      TokenKind::Function
1042    );
1043  }
1044
1045  #[test]
1046  fn sql_comments_and_strings() {
1047    assert_eq!(
1048      kind_of(Language::Sql, "x -- note", "-- note"),
1049      TokenKind::Comment
1050    );
1051    assert_eq!(
1052      kind_of(Language::Sql, "name = 'it''s'", "'it''s'"),
1053      TokenKind::StringLit
1054    );
1055    let mut state = LineState::default();
1056    Language::Sql.line("/* multi", &mut state);
1057    let t = Language::Sql.line("line */ SELECT", &mut state);
1058    assert_eq!(t[0], (0..7, TokenKind::Comment));
1059    assert_eq!(state, LineState::default());
1060  }
1061
1062  #[test]
1063  fn sql_blocks_do_not_nest() {
1064    let mut state = LineState::default();
1065    Language::Sql.line("/* a /* b */ tail", &mut state);
1066    // The first `*/` closes the whole comment.
1067    assert_eq!(state, LineState::default());
1068  }
1069
1070  #[test]
1071  fn json_tokens() {
1072    let line = r#"{"key": [1.5, true, null, "value"]}"#;
1073    assert_eq!(
1074      kind_of(Language::Json, line, r#""key""#),
1075      TokenKind::StringLit
1076    );
1077    assert_eq!(kind_of(Language::Json, line, "1.5"), TokenKind::Number);
1078    assert_eq!(kind_of(Language::Json, line, "true"), TokenKind::Keyword);
1079    assert_eq!(kind_of(Language::Json, line, "null"), TokenKind::Keyword);
1080  }
1081
1082  #[test]
1083  fn multibyte_idents_and_strings() {
1084    let line = "let café = \"日本語\"; // été";
1085    assert_eq!(kind_of(Language::Rust, line, "café"), TokenKind::Ident);
1086    assert_eq!(
1087      kind_of(Language::Rust, line, "\"日本語\""),
1088      TokenKind::StringLit
1089    );
1090    assert_eq!(kind_of(Language::Rust, line, "// été"), TokenKind::Comment);
1091  }
1092
1093  #[test]
1094  fn unterminated_string_stops_at_line_end() {
1095    let line = "let s = \"open";
1096    assert_eq!(
1097      kind_of(Language::Rust, line, "\"open"),
1098      TokenKind::StringLit
1099    );
1100    // ...and does not leak into the next line.
1101    let mut state = LineState::default();
1102    Language::Rust.line(line, &mut state);
1103    assert_eq!(state, LineState::default());
1104  }
1105
1106  #[test]
1107  fn punctuation_coalesces() {
1108    let tokens = kinds(Language::Rust, "a->b");
1109    assert_eq!(
1110      tokens,
1111      vec![
1112        ("a".to_string(), TokenKind::Ident),
1113        ("->".to_string(), TokenKind::Punct),
1114        ("b".to_string(), TokenKind::Ident),
1115      ]
1116    );
1117  }
1118
1119  #[test]
1120  fn empty_line_inside_block_comment() {
1121    let mut state = LineState::default();
1122    Language::Rust.line("/* open", &mut state);
1123    let t = Language::Rust.line("", &mut state);
1124    assert!(t.is_empty());
1125    assert_ne!(state, LineState::default());
1126  }
1127}
guise/editor/highlight.rs

guise/editor/
highlight.rs
