guise-ui 1.5.2

//! Pluggable, line-based syntax highlighting for the editor.
//!
//! A [`Highlighter`] tokenizes one line at a time into byte-range
//! [`TokenKind`] spans, threading a [`LineState`] through consecutive lines
//! so block comments carry across them. [`Language`] ships small
//! keyword/scanner tokenizers for Rust, SQL, and JSON, and [`token_color`]
//! maps kinds onto the active theme. Ranges are **byte** offsets into the
//! line (aligned to char boundaries), ready for gpui `TextRun` lengths.
//!
//! ```ignore
//! use guise::editor::{Highlighter, Language, LineState};
//!
//! let mut state = LineState::default();
//! for line in source.lines() {
//!     let tokens = Language::Rust.line(line, &mut state);
//!     // tokens: Vec<(Range<usize>, TokenKind)>
//! }
//! ```

use std::ops::Range;

use gpui::Hsla;

use crate::theme::{ColorName, Theme};

/// What a token is, for coloring. See [`token_color`].
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum TokenKind {
  Keyword,
  Ident,
  Number,
  StringLit,
  Comment,
  Punct,
  Type,
  Function,
}

impl TokenKind {
  /// Every kind, in [`index`](Self::index) order — lets a renderer resolve
  /// the theme palette once into an array.
  pub const ALL: [TokenKind; 8] = [
    TokenKind::Keyword,
    TokenKind::Ident,
    TokenKind::Number,
    TokenKind::StringLit,
    TokenKind::Comment,
    TokenKind::Punct,
    TokenKind::Type,
    TokenKind::Function,
  ];

  /// Stable index into [`ALL`](Self::ALL)-sized lookup tables.
  pub fn index(self) -> usize {
    self as usize
  }
}

/// Tokenizer state carried from one line to the next — currently the open
/// block-comment depth. Start each document with `LineState::default()`.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct LineState {
  block_depth: u32,
}

/// Tokenizes one line at a time. `state` carries block-comment continuation
/// across lines; feed lines in document order.
pub trait Highlighter {
  /// Tokenize `text` (a single line, no `\n`). Returned ranges are byte
  /// offsets into `text`, ascending and non-overlapping; uncovered gaps
  /// are unstyled.
  fn line(&self, text: &str, state: &mut LineState) -> Vec<(Range<usize>, TokenKind)>;
}

/// Whole-document highlighter — the seam for parse-tree backends (the
/// `treesitter` feature's `TreeSitterHighlighter`). Where [`Highlighter`]
/// re-scans line by line, an implementation parses the full document once
/// per edit and serves per-line tokens from that parse. The editor calls
/// [`update`](Self::update) only when the text changed, never per frame.
pub trait DocumentHighlighter {
  /// The document changed; reparse. `text` is the full buffer.
  fn update(&mut self, text: &str);

  /// Tokens for line `i` of the text last passed to
  /// [`update`](Self::update): ascending, non-overlapping byte ranges into
  /// that line, shaped like [`Highlighter::line`] output. Empty for
  /// out-of-range lines.
  fn tokens(&self, line: usize) -> &[(Range<usize>, TokenKind)];
}

/// Built-in languages with keyword/scanner tokenizers.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub enum Language {
  /// No highlighting — every line is plain text.
  #[default]
  None,
  Rust,
  Sql,
  Json,
  Toml,
  Python,
  JavaScript,
  TypeScript,
  Go,
  C,
  /// Line-structure highlighting: headings, quotes, fences, `code` spans.
  Markdown,
}

impl Highlighter for Language {
  fn line(&self, text: &str, state: &mut LineState) -> Vec<(Range<usize>, TokenKind)> {
    match self {
      Language::None => Vec::new(),
      Language::Rust => tokenize(&RUST, text, state),
      Language::Sql => tokenize(&SQL, text, state),
      Language::Json => tokenize(&JSON, text, state),
      Language::Toml => tokenize(&TOML, text, state),
      Language::Python => tokenize(&PYTHON, text, state),
      Language::JavaScript => tokenize(&JAVASCRIPT, text, state),
      Language::TypeScript => tokenize(&TYPESCRIPT, text, state),
      Language::Go => tokenize(&GO, text, state),
      Language::C => tokenize(&C, text, state),
      Language::Markdown => markdown_line(text, state),
    }
  }
}

/// The theme color for a token kind (light/dark aware). Comments use the
/// dimmed text color; identifiers and punctuation the normal text color.
pub fn token_color(kind: TokenKind, t: &Theme) -> Hsla {
  let shade = if t.scheme.is_dark() { 4 } else { 7 };
  match kind {
    TokenKind::Keyword => t.color(ColorName::Violet, shade).hsla(),
    TokenKind::StringLit => t.color(ColorName::Green, shade).hsla(),
    TokenKind::Number => t.color(ColorName::Orange, shade).hsla(),
    TokenKind::Type => t.color(ColorName::Teal, shade).hsla(),
    TokenKind::Function => t.color(ColorName::Blue, shade).hsla(),
    TokenKind::Comment => t.dimmed().hsla(),
    TokenKind::Ident | TokenKind::Punct => t.text().hsla(),
  }
}

// ---- scanner ---------------------------------------------------------------

/// How a quoted string escapes its own quote char.
#[derive(Clone, Copy)]
enum Escape {
  /// `\"` (Rust, JSON).
  Backslash,
  /// `''` (SQL).
  Doubled,
}

/// A language description the shared scanner runs over.
struct Syntax {
  line_comment: Option<&'static str>,
  block_comment: Option<(&'static str, &'static str)>,
  /// Whether block comments nest (Rust) or not (SQL).
  nested_blocks: bool,
  strings: &'static [(char, Escape)],
  keywords: &'static [&'static str],
  types: &'static [&'static str],
  /// Match keywords/types case-insensitively (SQL).
  case_insensitive: bool,
  /// Idents starting uppercase are types (Rust).
  uppercase_types: bool,
}

const RUST: Syntax = Syntax {
  line_comment: Some("//"),
  block_comment: Some(("/*", "*/")),
  nested_blocks: true,
  strings: &[('"', Escape::Backslash)],
  keywords: &[
    "as", "async", "await", "break", "const", "continue", "crate", "dyn", "else", "enum", "extern",
    "false", "fn", "for", "if", "impl", "in", "let", "loop", "match", "mod", "move", "mut", "pub",
    "ref", "return", "self", "Self", "static", "struct", "super", "trait", "true", "type", "union",
    "unsafe", "use", "where", "while",
  ],
  types: &[],
  case_insensitive: false,
  uppercase_types: true,
};

const SQL: Syntax = Syntax {
  line_comment: Some("--"),
  block_comment: Some(("/*", "*/")),
  nested_blocks: false,
  strings: &[('\'', Escape::Doubled)],
  keywords: &[
    "add",
    "all",
    "alter",
    "and",
    "as",
    "asc",
    "begin",
    "between",
    "by",
    "case",
    "cast",
    "check",
    "column",
    "commit",
    "constraint",
    "create",
    "cross",
    "default",
    "delete",
    "desc",
    "distinct",
    "drop",
    "else",
    "end",
    "exists",
    "false",
    "foreign",
    "from",
    "full",
    "group",
    "having",
    "if",
    "in",
    "index",
    "inner",
    "insert",
    "into",
    "is",
    "join",
    "key",
    "left",
    "like",
    "limit",
    "not",
    "null",
    "offset",
    "on",
    "or",
    "order",
    "outer",
    "primary",
    "references",
    "replace",
    "returning",
    "right",
    "rollback",
    "select",
    "set",
    "table",
    "then",
    "transaction",
    "true",
    "union",
    "unique",
    "update",
    "values",
    "view",
    "when",
    "where",
    "with",
  ],
  types: &[
    "bigint",
    "blob",
    "bool",
    "boolean",
    "bytea",
    "char",
    "date",
    "decimal",
    "double",
    "float",
    "int",
    "integer",
    "interval",
    "json",
    "jsonb",
    "numeric",
    "real",
    "serial",
    "smallint",
    "text",
    "time",
    "timestamp",
    "timestamptz",
    "uuid",
    "varchar",
  ],
  case_insensitive: true,
  uppercase_types: false,
};

const JSON: Syntax = Syntax {
  line_comment: None,
  block_comment: None,
  nested_blocks: false,
  strings: &[('"', Escape::Backslash)],
  keywords: &["false", "null", "true"],
  types: &[],
  case_insensitive: false,
  uppercase_types: false,
};

const TOML: Syntax = Syntax {
  line_comment: Some("#"),
  block_comment: None,
  nested_blocks: false,
  strings: &[('"', Escape::Backslash), ('\'', Escape::Backslash)],
  keywords: &["false", "true"],
  types: &[],
  case_insensitive: false,
  uppercase_types: false,
};

const PYTHON: Syntax = Syntax {
  line_comment: Some("#"),
  block_comment: None,
  nested_blocks: false,
  strings: &[('"', Escape::Backslash), ('\'', Escape::Backslash)],
  keywords: &[
    "False", "None", "True", "and", "as", "assert", "async", "await", "break", "class", "continue",
    "def", "del", "elif", "else", "except", "finally", "for", "from", "global", "if", "import",
    "in", "is", "lambda", "match", "nonlocal", "not", "or", "pass", "raise", "return", "try",
    "while", "with", "yield",
  ],
  types: &[
    "bool", "bytes", "dict", "float", "int", "list", "set", "str", "tuple",
  ],
  case_insensitive: false,
  uppercase_types: true,
};

const JAVASCRIPT: Syntax = Syntax {
  line_comment: Some("//"),
  block_comment: Some(("/*", "*/")),
  nested_blocks: false,
  strings: &[
    ('"', Escape::Backslash),
    ('\'', Escape::Backslash),
    ('`', Escape::Backslash),
  ],
  keywords: &[
    "async",
    "await",
    "break",
    "case",
    "catch",
    "class",
    "const",
    "continue",
    "debugger",
    "default",
    "delete",
    "do",
    "else",
    "export",
    "extends",
    "false",
    "finally",
    "for",
    "function",
    "if",
    "import",
    "in",
    "instanceof",
    "let",
    "new",
    "null",
    "of",
    "return",
    "static",
    "super",
    "switch",
    "this",
    "throw",
    "true",
    "try",
    "typeof",
    "undefined",
    "var",
    "void",
    "while",
    "with",
    "yield",
  ],
  types: &[],
  case_insensitive: false,
  uppercase_types: true,
};

const TYPESCRIPT: Syntax = Syntax {
  line_comment: Some("//"),
  block_comment: Some(("/*", "*/")),
  nested_blocks: false,
  strings: &[
    ('"', Escape::Backslash),
    ('\'', Escape::Backslash),
    ('`', Escape::Backslash),
  ],
  keywords: &[
    "abstract",
    "as",
    "async",
    "await",
    "break",
    "case",
    "catch",
    "class",
    "const",
    "continue",
    "debugger",
    "declare",
    "default",
    "delete",
    "do",
    "else",
    "enum",
    "export",
    "extends",
    "false",
    "finally",
    "for",
    "function",
    "if",
    "implements",
    "import",
    "in",
    "infer",
    "instanceof",
    "interface",
    "is",
    "keyof",
    "let",
    "namespace",
    "new",
    "null",
    "of",
    "readonly",
    "return",
    "satisfies",
    "static",
    "super",
    "switch",
    "this",
    "throw",
    "true",
    "try",
    "type",
    "typeof",
    "undefined",
    "var",
    "void",
    "while",
    "with",
    "yield",
  ],
  types: &[
    "any", "bigint", "boolean", "never", "number", "object", "string", "symbol", "unknown", "void",
  ],
  case_insensitive: false,
  uppercase_types: true,
};

const GO: Syntax = Syntax {
  line_comment: Some("//"),
  block_comment: Some(("/*", "*/")),
  nested_blocks: false,
  strings: &[('"', Escape::Backslash), ('`', Escape::Backslash)],
  keywords: &[
    "break",
    "case",
    "chan",
    "const",
    "continue",
    "default",
    "defer",
    "else",
    "fallthrough",
    "false",
    "for",
    "func",
    "go",
    "goto",
    "if",
    "import",
    "interface",
    "iota",
    "map",
    "nil",
    "package",
    "range",
    "return",
    "select",
    "struct",
    "switch",
    "true",
    "type",
    "var",
  ],
  types: &[
    "any",
    "bool",
    "byte",
    "complex128",
    "complex64",
    "error",
    "float32",
    "float64",
    "int",
    "int16",
    "int32",
    "int64",
    "int8",
    "rune",
    "string",
    "uint",
    "uint16",
    "uint32",
    "uint64",
    "uint8",
    "uintptr",
  ],
  case_insensitive: false,
  uppercase_types: true,
};

const C: Syntax = Syntax {
  line_comment: Some("//"),
  block_comment: Some(("/*", "*/")),
  nested_blocks: false,
  strings: &[('"', Escape::Backslash), ('\'', Escape::Backslash)],
  keywords: &[
    "break", "case", "const", "continue", "default", "do", "else", "enum", "extern", "for", "goto",
    "if", "inline", "register", "restrict", "return", "sizeof", "static", "struct", "switch",
    "typedef", "union", "volatile", "while",
  ],
  types: &[
    "bool", "char", "double", "float", "int", "long", "short", "signed", "size_t", "unsigned",
    "void",
  ],
  case_insensitive: false,
  uppercase_types: false,
};

/// Markdown is line-structural, not keyword-based, so it gets its own
/// tokenizer. `LineState::block_depth` doubles as the "inside a code fence"
/// flag (1 = fenced).
fn markdown_line(text: &str, state: &mut LineState) -> Vec<(Range<usize>, TokenKind)> {
  let trimmed = text.trim_start();
  let indent = text.len() - trimmed.len();

  if trimmed.starts_with("```") || trimmed.starts_with("~~~") {
    state.block_depth = if state.block_depth > 0 { 0 } else { 1 };
    return vec![(indent..text.len(), TokenKind::Punct)];
  }
  if state.block_depth > 0 {
    if text.is_empty() {
      return Vec::new();
    }
    return vec![(0..text.len(), TokenKind::StringLit)];
  }
  if trimmed.starts_with('#') {
    return vec![(indent..text.len(), TokenKind::Keyword)];
  }
  if trimmed.starts_with('>') {
    return vec![(indent..text.len(), TokenKind::Comment)];
  }

  let mut out = Vec::new();
  // List bullet: "- ", "* ", "+ ", or "1. " — mark just the marker.
  if trimmed.starts_with("- ") || trimmed.starts_with("* ") || trimmed.starts_with("+ ") {
    out.push((indent..indent + 1, TokenKind::Punct));
  } else {
    let digits = trimmed.chars().take_while(char::is_ascii_digit).count();
    if digits > 0 && trimmed[digits..].starts_with(". ") {
      out.push((indent..indent + digits + 1, TokenKind::Punct));
    }
  }
  // Inline `code` spans (ticks included). Unmatched ticks stay plain.
  let mut open: Option<usize> = None;
  for (b, c) in text.char_indices() {
    if c == '`' {
      match open.take() {
        Some(start) => out.push((start..b + 1, TokenKind::StringLit)),
        None => open = Some(b),
      }
    }
  }
  out.sort_by_key(|(range, _)| range.start);
  out
}

/// Run `syntax` over one line. Works on `char_indices` so every emitted
/// range is char-boundary aligned (multibyte-safe).
fn tokenize(syntax: &Syntax, text: &str, state: &mut LineState) -> Vec<(Range<usize>, TokenKind)> {
  let chars: Vec<(usize, char)> = text.char_indices().collect();
  let n = chars.len();
  let byte_at = |i: usize| chars.get(i).map(|&(b, _)| b).unwrap_or(text.len());
  let mut out: Vec<(Range<usize>, TokenKind)> = Vec::new();
  let mut i = 0;

  // A block comment left open by a previous line swallows the line start.
  if state.block_depth > 0 {
    match syntax.block_comment {
      Some((open, close)) => {
        let (end, depth) = scan_block(
          &chars,
          0,
          open,
          close,
          syntax.nested_blocks,
          state.block_depth,
        );
        state.block_depth = depth;
        if byte_at(end) > 0 {
          out.push((0..byte_at(end), TokenKind::Comment));
        }
        i = end;
      }
      // Stale state from another language: ignore it.
      None => state.block_depth = 0,
    }
  }

  while i < n {
    let (b, c) = chars[i];
    if c.is_whitespace() {
      i += 1;
      continue;
    }
    if let Some(lc) = syntax.line_comment {
      if starts_with_at(&chars, i, lc) {
        out.push((b..text.len(), TokenKind::Comment));
        break;
      }
    }
    if let Some((open, close)) = syntax.block_comment {
      if starts_with_at(&chars, i, open) {
        let after_open = i + open.chars().count();
        let (end, depth) = scan_block(&chars, after_open, open, close, syntax.nested_blocks, 1);
        state.block_depth = depth;
        out.push((b..byte_at(end), TokenKind::Comment));
        i = end;
        continue;
      }
    }
    if let Some(&(_, esc)) = syntax.strings.iter().find(|&&(q, _)| q == c) {
      let end = scan_string(&chars, i + 1, c, esc);
      out.push((b..byte_at(end), TokenKind::StringLit));
      i = end;
      continue;
    }
    if c.is_ascii_digit() {
      let end = scan_number(&chars, i);
      out.push((b..byte_at(end), TokenKind::Number));
      i = end;
      continue;
    }
    if c.is_alphabetic() || c == '_' {
      let end = scan_ident(&chars, i);
      let word = &text[b..byte_at(end)];
      out.push((b..byte_at(end), classify_word(syntax, word, &chars, end)));
      i = end;
      continue;
    }
    out.push((b..byte_at(i + 1), TokenKind::Punct));
    i += 1;
  }

  coalesce(out)
}

/// Does the char sequence at `i` spell out `pat`?
fn starts_with_at(chars: &[(usize, char)], i: usize, pat: &str) -> bool {
  let mut j = i;
  for p in pat.chars() {
    match chars.get(j) {
      Some(&(_, c)) if c == p => j += 1,
      _ => return false,
    }
  }
  true
}

/// Scan a block-comment body from `i` at `depth` (>= 1 means inside).
/// Returns the char index just past the final close, and the depth still
/// open at the line end (0 = closed).
fn scan_block(
  chars: &[(usize, char)],
  mut i: usize,
  open: &str,
  close: &str,
  nested: bool,
  mut depth: u32,
) -> (usize, u32) {
  let n = chars.len();
  while i < n {
    if nested && starts_with_at(chars, i, open) {
      depth += 1;
      i += open.chars().count();
    } else if starts_with_at(chars, i, close) {
      depth -= 1;
      i += close.chars().count();
      if depth == 0 {
        return (i, 0);
      }
    } else {
      i += 1;
    }
  }
  (n, depth)
}

/// Scan a string body from `i` (just past the opening quote). Returns the
/// char index just past the closing quote, or the line end if unterminated
/// (strings do not continue across lines).
fn scan_string(chars: &[(usize, char)], mut i: usize, quote: char, esc: Escape) -> usize {
  let n = chars.len();
  while i < n {
    let c = chars[i].1;
    match esc {
      Escape::Backslash if c == '\\' => {
        i += 2;
        continue;
      }
      Escape::Doubled if c == quote => {
        if i + 1 < n && chars[i + 1].1 == quote {
          i += 2;
          continue;
        }
        return i + 1;
      }
      _ if c == quote => return i + 1,
      _ => i += 1,
    }
  }
  n
}

/// Scan a number from `i` (a digit): integers, `0x`/`0b`/`0o` prefixes,
/// decimals, exponents, and trailing type suffixes (`1u8`, `2.5f64`).
fn scan_number(chars: &[(usize, char)], mut i: usize) -> usize {
  let n = chars.len();
  if chars[i].1 == '0' && i + 1 < n && matches!(chars[i + 1].1, 'x' | 'X' | 'b' | 'B' | 'o' | 'O') {
    i += 2;
    while i < n && (chars[i].1.is_ascii_alphanumeric() || chars[i].1 == '_') {
      i += 1;
    }
    return i;
  }
  while i < n && (chars[i].1.is_ascii_digit() || chars[i].1 == '_') {
    i += 1;
  }
  if i + 1 < n && chars[i].1 == '.' && chars[i + 1].1.is_ascii_digit() {
    i += 1;
    while i < n && (chars[i].1.is_ascii_digit() || chars[i].1 == '_') {
      i += 1;
    }
  }
  if i < n && matches!(chars[i].1, 'e' | 'E') {
    let mut j = i + 1;
    if j < n && matches!(chars[j].1, '+' | '-') {
      j += 1;
    }
    if j < n && chars[j].1.is_ascii_digit() {
      i = j;
      while i < n && chars[i].1.is_ascii_digit() {
        i += 1;
      }
    }
  }
  while i < n && (chars[i].1.is_ascii_alphanumeric() || chars[i].1 == '_') {
    i += 1;
  }
  i
}

/// Scan an identifier from `i` (a letter or `_`).
fn scan_ident(chars: &[(usize, char)], mut i: usize) -> usize {
  let n = chars.len();
  while i < n && (chars[i].1.is_alphanumeric() || chars[i].1 == '_') {
    i += 1;
  }
  i
}

/// Keyword / type / function-call / plain ident, in that priority. `end` is
/// the char index just past the word, for call-site lookahead.
fn classify_word(syntax: &Syntax, word: &str, chars: &[(usize, char)], end: usize) -> TokenKind {
  let in_set = |set: &[&str]| {
    if syntax.case_insensitive {
      set.iter().any(|k| k.eq_ignore_ascii_case(word))
    } else {
      set.contains(&word)
    }
  };
  if in_set(syntax.keywords) {
    return TokenKind::Keyword;
  }
  if in_set(syntax.types) {
    return TokenKind::Type;
  }
  if syntax.uppercase_types && word.chars().next().is_some_and(char::is_uppercase) {
    return TokenKind::Type;
  }
  // `name(` is a call; `name!(` a macro invocation.
  match chars.get(end).map(|&(_, c)| c) {
    Some('(') => TokenKind::Function,
    Some('!') if matches!(chars.get(end + 1), Some(&(_, '('))) => TokenKind::Function,
    _ => TokenKind::Ident,
  }
}

/// Merge adjacent tokens of the same kind with contiguous ranges, so a run
/// of punctuation becomes one span.
pub(crate) fn coalesce(tokens: Vec<(Range<usize>, TokenKind)>) -> Vec<(Range<usize>, TokenKind)> {
  let mut out: Vec<(Range<usize>, TokenKind)> = Vec::new();
  for (range, kind) in tokens {
    if let Some((last, last_kind)) = out.last_mut() {
      if *last_kind == kind && last.end == range.start {
        last.end = range.end;
        continue;
      }
    }
    out.push((range, kind));
  }
  out
}

#[cfg(test)]
mod tests {
  use super::*;

  fn kinds(lang: Language, line: &str) -> Vec<(String, TokenKind)> {
    let mut state = LineState::default();
    lang
      .line(line, &mut state)
      .into_iter()
      .map(|(r, k)| (line[r].to_string(), k))
      .collect()
  }

  fn kind_of(lang: Language, line: &str, word: &str) -> TokenKind {
    kinds(lang, line)
      .into_iter()
      .find(|(w, _)| w == word)
      .map(|(_, k)| k)
      .unwrap_or_else(|| panic!("token {word:?} not found in {line:?}"))
  }

  #[test]
  fn none_language_emits_nothing() {
    assert!(kinds(Language::None, "let x = 1;").is_empty());
  }

  #[test]
  fn new_languages_classify_keywords_strings_comments() {
    assert_eq!(
      kind_of(Language::Toml, "name = \"guise\" # crate", "\"guise\""),
      TokenKind::StringLit
    );
    assert_eq!(
      kind_of(Language::Toml, "flag = true", "true"),
      TokenKind::Keyword
    );
    assert_eq!(
      kind_of(Language::Python, "def run(): pass  # go", "def"),
      TokenKind::Keyword
    );
    assert_eq!(
      kind_of(Language::Python, "def run(): pass  # go", "# go"),
      TokenKind::Comment
    );
    assert_eq!(
      kind_of(Language::JavaScript, "const x = `hi`;", "const"),
      TokenKind::Keyword
    );
    assert_eq!(
      kind_of(Language::JavaScript, "const x = `hi`;", "`hi`"),
      TokenKind::StringLit
    );
    assert_eq!(
      kind_of(Language::TypeScript, "let n: number = 5;", "number"),
      TokenKind::Type
    );
    assert_eq!(
      kind_of(Language::TypeScript, "interface A {}", "interface"),
      TokenKind::Keyword
    );
    assert_eq!(
      kind_of(Language::Go, "func main() {}", "func"),
      TokenKind::Keyword
    );
    assert_eq!(
      kind_of(Language::Go, "var n int64", "int64"),
      TokenKind::Type
    );
    assert_eq!(
      kind_of(Language::C, "static int n = 0; // c", "static"),
      TokenKind::Keyword
    );
    assert_eq!(
      kind_of(Language::C, "static int n = 0; // c", "int"),
      TokenKind::Type
    );
  }

  #[test]
  fn markdown_structures_lines() {
    assert_eq!(
      kinds(Language::Markdown, "# Title"),
      vec![("# Title".into(), TokenKind::Keyword)]
    );
    assert_eq!(
      kinds(Language::Markdown, "> quoted"),
      vec![("> quoted".into(), TokenKind::Comment)]
    );
    let bullets = kinds(Language::Markdown, "- item with `code` span");
    assert_eq!(bullets[0], ("-".into(), TokenKind::Punct));
    assert_eq!(bullets[1], ("`code`".into(), TokenKind::StringLit));
    let ordered = kinds(Language::Markdown, "12. step");
    assert_eq!(ordered[0], ("12.".into(), TokenKind::Punct));
    // Unmatched ticks stay plain.
    assert!(kinds(Language::Markdown, "just a ` tick").is_empty());
  }

  #[test]
  fn markdown_fences_carry_state() {
    let mut state = LineState::default();
    let fence = Language::Markdown.line("```rust", &mut state);
    assert_eq!(fence[0].1, TokenKind::Punct);
    let inside = Language::Markdown.line("# not a heading", &mut state);
    assert_eq!(
      inside,
      vec![(0.."# not a heading".len(), TokenKind::StringLit)]
    );
    Language::Markdown.line("```", &mut state);
    let after = Language::Markdown.line("# heading again", &mut state);
    assert_eq!(after[0].1, TokenKind::Keyword);
  }

  #[test]
  fn ranges_are_ascending_and_in_bounds() {
    let line = "let s = \"héllo\"; // café";
    let mut state = LineState::default();
    let tokens = Language::Rust.line(line, &mut state);
    let mut at = 0;
    for (range, _) in &tokens {
      assert!(range.start >= at, "overlapping range");
      assert!(range.end <= line.len());
      assert!(line.is_char_boundary(range.start));
      assert!(line.is_char_boundary(range.end));
      at = range.end;
    }
  }

  #[test]
  fn rust_basics() {
    assert_eq!(
      kind_of(Language::Rust, "let x = 1;", "let"),
      TokenKind::Keyword
    );
    assert_eq!(kind_of(Language::Rust, "let x = 1;", "x"), TokenKind::Ident);
    assert_eq!(
      kind_of(Language::Rust, "let x = 10.5e3;", "10.5e3"),
      TokenKind::Number
    );
    assert_eq!(
      kind_of(Language::Rust, "let n = 0xff_u8;", "0xff_u8"),
      TokenKind::Number
    );
    assert_eq!(
      kind_of(
        Language::Rust,
        r#"let s = "hi \" there";"#,
        r#""hi \" there""#
      ),
      TokenKind::StringLit
    );
    assert_eq!(
      kind_of(Language::Rust, "let v: Vec<u8>;", "Vec"),
      TokenKind::Type
    );
    assert_eq!(
      kind_of(Language::Rust, "foo(1)", "foo"),
      TokenKind::Function
    );
    assert_eq!(
      kind_of(Language::Rust, "println!(\"x\")", "println"),
      TokenKind::Function
    );
    assert_eq!(
      kind_of(Language::Rust, "a + b // sum", "// sum"),
      TokenKind::Comment
    );
  }

  #[test]
  fn rust_block_comment_carries_state() {
    let mut state = LineState::default();
    let t1 = Language::Rust.line("start /* open", &mut state);
    assert_eq!(
      t1.last()
        .map(|(r, k)| ("start /* open"[r.clone()].to_string(), *k)),
      Some(("/* open".to_string(), TokenKind::Comment))
    );
    let t2 = Language::Rust.line("all comment", &mut state);
    assert_eq!(t2, vec![(0..11, TokenKind::Comment)]);
    let t3 = Language::Rust.line("done */ let x", &mut state);
    assert_eq!(t3[0], (0..7, TokenKind::Comment));
    assert_eq!("done */ let x"[t3[1].clone().0].to_string(), "let");
    assert_eq!(state, LineState::default());
  }

  #[test]
  fn rust_block_comments_nest() {
    let mut state = LineState::default();
    Language::Rust.line("/* a /* b */ still", &mut state);
    assert_ne!(state, LineState::default());
    let t = Language::Rust.line("c */ code", &mut state);
    assert_eq!(t[0], (0..4, TokenKind::Comment));
    assert_eq!(state, LineState::default());
  }

  #[test]
  fn sql_keywords_are_case_insensitive() {
    assert_eq!(
      kind_of(Language::Sql, "SELECT * FROM users;", "SELECT"),
      TokenKind::Keyword
    );
    assert_eq!(
      kind_of(Language::Sql, "select * from users;", "select"),
      TokenKind::Keyword
    );
    assert_eq!(
      kind_of(Language::Sql, "id INT PRIMARY KEY", "INT"),
      TokenKind::Type
    );
    assert_eq!(
      kind_of(Language::Sql, "count(*)", "count"),
      TokenKind::Function
    );
  }

  #[test]
  fn sql_comments_and_strings() {
    assert_eq!(
      kind_of(Language::Sql, "x -- note", "-- note"),
      TokenKind::Comment
    );
    assert_eq!(
      kind_of(Language::Sql, "name = 'it''s'", "'it''s'"),
      TokenKind::StringLit
    );
    let mut state = LineState::default();
    Language::Sql.line("/* multi", &mut state);
    let t = Language::Sql.line("line */ SELECT", &mut state);
    assert_eq!(t[0], (0..7, TokenKind::Comment));
    assert_eq!(state, LineState::default());
  }

  #[test]
  fn sql_blocks_do_not_nest() {
    let mut state = LineState::default();
    Language::Sql.line("/* a /* b */ tail", &mut state);
    // The first `*/` closes the whole comment.
    assert_eq!(state, LineState::default());
  }

  #[test]
  fn json_tokens() {
    let line = r#"{"key": [1.5, true, null, "value"]}"#;
    assert_eq!(
      kind_of(Language::Json, line, r#""key""#),
      TokenKind::StringLit
    );
    assert_eq!(kind_of(Language::Json, line, "1.5"), TokenKind::Number);
    assert_eq!(kind_of(Language::Json, line, "true"), TokenKind::Keyword);
    assert_eq!(kind_of(Language::Json, line, "null"), TokenKind::Keyword);
  }

  #[test]
  fn multibyte_idents_and_strings() {
    let line = "let café = \"日本語\"; // été";
    assert_eq!(kind_of(Language::Rust, line, "café"), TokenKind::Ident);
    assert_eq!(
      kind_of(Language::Rust, line, "\"日本語\""),
      TokenKind::StringLit
    );
    assert_eq!(kind_of(Language::Rust, line, "// été"), TokenKind::Comment);
  }

  #[test]
  fn unterminated_string_stops_at_line_end() {
    let line = "let s = \"open";
    assert_eq!(
      kind_of(Language::Rust, line, "\"open"),
      TokenKind::StringLit
    );
    // ...and does not leak into the next line.
    let mut state = LineState::default();
    Language::Rust.line(line, &mut state);
    assert_eq!(state, LineState::default());
  }

  #[test]
  fn punctuation_coalesces() {
    let tokens = kinds(Language::Rust, "a->b");
    assert_eq!(
      tokens,
      vec![
        ("a".to_string(), TokenKind::Ident),
        ("->".to_string(), TokenKind::Punct),
        ("b".to_string(), TokenKind::Ident),
      ]
    );
  }

  #[test]
  fn empty_line_inside_block_comment() {
    let mut state = LineState::default();
    Language::Rust.line("/* open", &mut state);
    let t = Language::Rust.line("", &mut state);
    assert!(t.is_empty());
    assert_ne!(state, LineState::default());
  }
}