use crate::lsp::{Encoding, SemanticToken};
use crate::snippets::Language;
#[cfg(test)]
mod tests;
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum TokenKind {
Keyword,
Function,
Type,
String,
Constant,
Comment,
Macro,
Namespace,
}
impl TokenKind {
pub fn from_name(name: &str) -> Option<Self> {
Some(match name {
"keyword" | "modifier" | "selfKeyword" | "selfTypeKeyword" => Self::Keyword,
"function" | "method" | "member" => Self::Function,
"type" | "class" | "struct" | "enum" | "interface" | "union" | "typeParameter"
| "typeAlias" | "builtinType" | "concept" => Self::Type,
"string" | "char" | "character" | "regexp" | "escapeSequence" | "formatSpecifier" => {
Self::String
}
"number" | "boolean" | "enumMember" | "const" | "constParameter" => Self::Constant,
"comment" => Self::Comment,
"macro" | "decorator" | "attribute" | "derive" | "builtinAttribute" | "lifetime"
| "label" => Self::Macro,
"namespace" | "module" => Self::Namespace,
_ => return None,
})
}
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct Token {
pub start: usize,
pub end: usize,
pub kind: Option<TokenKind>,
}
#[derive(Default)]
pub struct Highlights {
lines: Vec<Vec<Token>>,
}
impl Highlights {
pub fn new(tokens: &[SemanticToken], lines: &[String], encoding: Encoding) -> Self {
let mut table = vec![Vec::new(); lines.len()];
for token in tokens {
let Some(line) = lines.get(token.line) else {
continue;
};
let start = encoding.to_chars(line, token.start);
let end = encoding.to_chars(line, token.start + token.length);
if start < end {
table[token.line].push(Token {
start,
end,
kind: token.kind,
});
}
}
Self { lines: table }
}
pub fn on(&self, y: usize) -> &[Token] {
self.lines.get(y).map_or(&[], Vec::as_slice)
}
pub fn inserted(&mut self, x: usize, y: usize) {
for token in self.lines.get_mut(y).into_iter().flatten() {
if token.start >= x {
token.start += 1;
token.end += 1;
} else if token.end >= x {
token.end += 1;
}
}
}
pub fn deleted(&mut self, x: usize, y: usize) {
if let Some(tokens) = self.lines.get_mut(y) {
for token in tokens.iter_mut() {
if token.start > x {
token.start -= 1;
token.end -= 1;
} else if token.end > x {
token.end -= 1;
}
}
tokens.retain(|token| token.start < token.end);
}
}
pub fn split(&mut self, x: usize, y: usize) {
let Some(tokens) = self.lines.get_mut(y) else {
return;
};
let mut below = Vec::new();
for token in tokens.iter_mut() {
if token.end > x {
below.push(Token {
start: token.start.saturating_sub(x),
end: token.end - x,
kind: token.kind,
});
token.end = token.end.min(x);
}
}
tokens.retain(|token| token.start < token.end);
self.lines.insert(y + 1, below);
}
pub fn joined(&mut self, y: usize, len: usize) {
if y + 1 < self.lines.len() {
let below = self.lines.remove(y + 1);
self.lines[y].extend(below.into_iter().map(|token| Token {
start: token.start + len,
end: token.end + len,
kind: token.kind,
}));
}
}
}
#[derive(Clone, Copy, Debug, Default, PartialEq)]
pub enum LineState {
#[default]
Code,
Comment,
Text(Delimiter),
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum Delimiter {
Double,
Backtick,
Triple(char),
Raw(usize),
Block(usize),
}
struct Syntax {
keywords: &'static [&'static str],
constants: &'static [&'static str],
types: &'static [&'static str],
line_comment: &'static str,
block_comments: bool,
}
impl Syntax {
fn of(language: Language) -> &'static Self {
match language {
Language::Rust => &Self {
keywords: &[
"as", "async", "await", "break", "const", "continue", "crate", "dyn", "else",
"enum", "extern", "fn", "for", "if", "impl", "in", "let", "loop", "match",
"mod", "move", "mut", "pub", "ref", "return", "self", "Self", "static",
"struct", "super", "trait", "type", "unsafe", "use", "where", "while",
],
constants: &["true", "false"],
types: &[
"bool", "char", "str", "u8", "u16", "u32", "u64", "u128", "usize", "i8",
"i16", "i32", "i64", "i128", "isize", "f32", "f64",
],
line_comment: "//",
block_comments: true,
},
Language::Python => &Self {
keywords: &[
"and", "as", "assert", "async", "await", "break", "class", "continue", "def",
"del", "elif", "else", "except", "finally", "for", "from", "global", "if",
"import", "in", "is", "lambda", "nonlocal", "not", "or", "pass", "raise",
"return", "self", "try", "while", "with", "yield",
],
constants: &["True", "False", "None"],
types: &[
"bool", "bytes", "dict", "float", "int", "list", "object", "set", "str",
"tuple",
],
line_comment: "#",
block_comments: false,
},
Language::JavaScript => &Self {
keywords: &[
"abstract", "as", "async", "await", "break", "case", "catch", "class",
"const", "continue", "debugger", "declare", "default", "delete", "do",
"else", "enum", "export", "extends", "finally", "for", "from", "function",
"if", "implements", "import", "in", "instanceof", "interface", "let",
"namespace", "new", "of", "private", "protected", "public", "readonly",
"return", "static", "super", "switch", "this", "throw", "try", "type",
"typeof", "var", "void", "while", "with", "yield",
],
constants: &["true", "false", "null", "undefined", "NaN"],
types: &["any", "boolean", "never", "number", "object", "string", "unknown"],
line_comment: "//",
block_comments: true,
},
Language::C => &Self {
keywords: &[
"auto", "break", "case", "catch", "class", "const", "constexpr", "continue",
"default", "delete", "do", "else", "enum", "extern", "for", "goto", "if",
"inline", "namespace", "new", "operator", "override", "private",
"protected", "public", "register", "return", "sizeof", "static", "struct",
"switch", "template", "this", "throw", "try", "typedef", "typename",
"union", "using", "virtual", "volatile", "while",
],
constants: &["true", "false", "NULL", "nullptr"],
types: &[
"bool", "char", "double", "float", "int", "long", "short", "signed",
"size_t", "unsigned", "void",
],
line_comment: "//",
block_comments: true,
},
Language::Yaml => &Self {
keywords: &[],
constants: &["true", "false", "null", "yes", "no", "on", "off", "~"],
types: &[],
line_comment: "#",
block_comments: false,
},
Language::Go => &Self {
keywords: &[
"break", "case", "chan", "const", "continue", "default", "defer", "else",
"fallthrough", "for", "func", "go", "goto", "if", "import", "interface",
"map", "package", "range", "return", "select", "struct", "switch", "type",
"var",
],
constants: &["true", "false", "nil", "iota"],
types: &[
"any", "bool", "byte", "error", "float32", "float64", "int", "int8",
"int16", "int32", "int64", "rune", "string", "uint", "uint8", "uint16",
"uint32", "uint64",
],
line_comment: "//",
block_comments: true,
},
}
}
}
pub fn line_comment(language: Language) -> &'static str {
Syntax::of(language).line_comment
}
fn is_word(c: char) -> bool {
c.is_alphanumeric() || c == '_'
}
fn closing(chars: &[char], from: usize, state: LineState, language: Language) -> Option<usize> {
let at = |i: usize, c: char| chars.get(i) == Some(&c);
let escapes = match state {
LineState::Text(Delimiter::Raw(_)) => false,
LineState::Text(Delimiter::Backtick) => language != Language::Go,
_ => true,
};
let mut i = from;
while i < chars.len() {
let end = match state {
LineState::Code => return Some(from),
LineState::Comment => (at(i, '*') && at(i + 1, '/')).then_some(i + 2),
LineState::Text(Delimiter::Double) => at(i, '"').then_some(i + 1),
LineState::Text(Delimiter::Backtick) => at(i, '`').then_some(i + 1),
LineState::Text(Delimiter::Triple(quote)) => {
(0..3).all(|k| at(i + k, quote)).then_some(i + 3)
}
LineState::Text(Delimiter::Raw(hashes)) => {
(at(i, '"') && (1..=hashes).all(|k| at(i + k, '#'))).then_some(i + 1 + hashes)
}
LineState::Text(Delimiter::Block(_)) => None,
};
if end.is_some() {
return end;
}
if escapes && state != LineState::Comment && chars[i] == '\\' {
i += 1;
}
i += 1;
}
None
}
fn quoted_end(chars: &[char], start: usize, quote: char) -> usize {
let mut i = start + 1;
while i < chars.len() {
match chars[i] {
'\\' => i += 1,
c if c == quote => return i + 1,
_ => {}
}
i += 1;
}
chars.len()
}
pub fn lex_line(
line: &str,
language: Language,
state: LineState,
out: &mut Vec<Token>,
) -> LineState {
if language == Language::Yaml {
return lex_yaml(line, state, out);
}
let syntax = Syntax::of(language);
let chars: Vec<char> = line.chars().collect();
let len = chars.len();
let at = |i: usize, c: char| chars.get(i) == Some(&c);
let mut push = |start: usize, end: usize, kind: TokenKind| {
if start < end {
out.push(Token {
start,
end,
kind: Some(kind),
});
}
};
let mut open = state;
let mut open_start = 0;
let mut i = 0;
loop {
if open != LineState::Code {
let kind = match open {
LineState::Comment => TokenKind::Comment,
_ => TokenKind::String,
};
match closing(&chars, i, open, language) {
Some(end) => {
push(open_start, end, kind);
i = end;
open = LineState::Code;
}
None => {
push(open_start, len, kind);
return open;
}
}
}
if i >= len {
return LineState::Code;
}
let c = chars[i];
if c.is_whitespace() {
i += 1;
} else if syntax.line_comment.chars().enumerate().all(|(k, c)| at(i + k, c)) {
push(i, len, TokenKind::Comment);
return LineState::Code;
} else if syntax.block_comments && c == '/' && at(i + 1, '*') {
(open, open_start) = (LineState::Comment, i);
i += 2;
} else if c == '"' || c == '\'' || c == '`' {
let multi_line = match (language, c) {
(Language::Rust, '"') => Some((Delimiter::Double, 1)),
(Language::Python, _) if c != '`' && at(i + 1, c) && at(i + 2, c) => {
Some((Delimiter::Triple(c), 3))
}
(Language::JavaScript | Language::Go, '`') => Some((Delimiter::Backtick, 1)),
_ => None,
};
if let Some((delimiter, width)) = multi_line {
(open, open_start) = (LineState::Text(delimiter), i);
i += width;
} else if c == '`' {
i += 1;
} else if language == Language::Rust && !at(i + 1, '\\') && !at(i + 2, '\'') {
let end = (i + 1..len).find(|j| !is_word(chars[*j])).unwrap_or(len);
push(i, end, TokenKind::Macro);
i = end.max(i + 1);
} else {
let end = quoted_end(&chars, i, c);
push(i, end, TokenKind::String);
i = end;
}
} else if c.is_ascii_digit() {
let mut end = i + 1;
while end < len
&& (is_word(chars[end])
|| (chars[end] == '.' && chars.get(end + 1).is_some_and(char::is_ascii_digit)))
{
end += 1;
}
push(i, end, TokenKind::Constant);
i = end;
} else if is_word(c) {
let mut end = (i..len).find(|j| !is_word(chars[*j])).unwrap_or(len);
let word: String = chars[i..end].iter().collect();
let raw_hashes = (end..len).take_while(|j| chars[*j] == '#').count();
if language == Language::Rust
&& (word == "r" || word == "br")
&& at(end + raw_hashes, '"')
{
(open, open_start) = (LineState::Text(Delimiter::Raw(raw_hashes)), i);
i = end + raw_hashes + 1;
continue;
}
let kind = if syntax.keywords.contains(&word.as_str()) {
Some(TokenKind::Keyword)
} else if syntax.constants.contains(&word.as_str()) {
Some(TokenKind::Constant)
} else if syntax.types.contains(&word.as_str()) {
Some(TokenKind::Type)
} else if language == Language::Rust && at(end, '!') && !at(end + 1, '=') {
end += 1;
Some(TokenKind::Macro)
} else if at(end, '(') {
Some(TokenKind::Function)
} else if c.is_uppercase() {
let shouting = word.len() > 1 && !word.chars().any(char::is_lowercase);
Some(if shouting { TokenKind::Constant } else { TokenKind::Type })
} else {
None
};
if let Some(kind) = kind {
push(i, end, kind);
}
i = end;
} else if (c == '@' && language == Language::Python)
|| (c == '#' && language == Language::C)
{
let end = (i + 1..len).find(|j| !is_word(chars[*j])).unwrap_or(len);
push(i, end, TokenKind::Macro);
i = end.max(i + 1);
} else if c == '#' && language == Language::Rust && (at(i + 1, '[') || at(i + 1, '!')) {
let end = (i..len).find(|j| chars[*j] == ']').map_or(len, |j| j + 1);
push(i, end, TokenKind::Macro);
i = end;
} else {
i += 1;
}
}
}
fn lex_yaml(line: &str, state: LineState, out: &mut Vec<Token>) -> LineState {
let chars: Vec<char> = line.chars().collect();
let len = chars.len();
let mut push = |start: usize, end: usize, kind: TokenKind| {
if start < end {
out.push(Token {
start,
end,
kind: Some(kind),
});
}
};
let space = |i: usize| chars.get(i).is_none_or(|c| c.is_whitespace());
let indent = chars.iter().take_while(|c| **c == ' ').count();
if let LineState::Text(Delimiter::Block(parent)) = state
&& (indent > parent || indent == len)
{
push(indent, len, TokenKind::String);
return state;
}
let mut i = indent;
while i < len && chars[i] == '-' && space(i + 1) {
i += 1;
while i < len && chars[i] == ' ' {
i += 1;
}
}
let rest: String = chars[i..].iter().collect();
if i == 0 && (rest.trim_end() == "---" || rest.trim_end() == "...") {
push(0, 3, TokenKind::Keyword);
return LineState::Code;
}
let mut depth = 0usize;
let mut at_start = true;
let mut seen_key = false;
while i < len {
let c = chars[i];
if c == ' ' {
i += 1;
continue;
}
if c == '#' && (i == 0 || chars[i - 1] == ' ') {
push(i, len, TokenKind::Comment);
return LineState::Code;
}
match c {
'[' | '{' => depth += 1,
']' | '}' => depth = depth.saturating_sub(1),
_ => {}
}
if matches!(c, '[' | '{' | ']' | '}' | ',') || (c == ':' && space(i + 1)) {
i += 1;
at_start = true;
continue;
}
let quoted = c == '"' || c == '\'';
let mut end = if quoted { quoted_end(&chars, i, c) } else { i };
while !quoted && end < len {
let stop = match chars[end] {
'#' => chars[end - 1] == ' ',
':' => space(end + 1) && (depth > 0 || !seen_key),
',' | ']' | '}' => depth > 0,
_ => false,
};
if stop {
break;
}
end += 1;
}
while !quoted && end > i && chars[end - 1] == ' ' {
end -= 1;
}
let word: String = chars[i..end].iter().collect();
let after = (end..len).find(|j| chars[*j] != ' ').unwrap_or(len);
let is_key = chars.get(after) == Some(&':')
&& space(after + 1)
&& (depth > 0 || !seen_key);
let block = matches!(c, '|' | '>') && word.chars().skip(1).all(|c| "+-0123456789".contains(c));
if is_key {
push(i, end, TokenKind::Function);
seen_key = true;
} else if block && at_start {
push(i, end, TokenKind::Keyword);
return LineState::Text(Delimiter::Block(indent));
} else if quoted {
push(i, end, TokenKind::String);
} else if c == '&' || c == '*' {
push(i, end, TokenKind::Macro);
} else if c == '!' {
let tag_end = (i..end).find(|j| chars[*j] == ' ').unwrap_or(end);
push(i, tag_end, TokenKind::Type);
push(tag_end, end, TokenKind::String);
} else if Syntax::of(Language::Yaml).constants.contains(&word.to_lowercase().as_str())
|| word.parse::<f64>().is_ok()
|| word.starts_with("0x")
{
push(i, end, TokenKind::Constant);
} else {
push(i, end, TokenKind::String);
let starts = |at: usize, text: &str| text.chars().enumerate().all(|(k, c)| chars.get(at + k) == Some(&c));
let mut from = i;
while let Some(open) = (from..end).find(|at| starts(*at, "${{")) {
let close = (open..end).find(|at| starts(*at, "}}")).map_or(end, |at| at + 2);
push(open, close, TokenKind::Macro);
from = close;
}
}
i = end.max(i + 1);
at_start = false;
}
LineState::Code
}
pub fn line_states(lines: &[String], language: Language) -> Vec<LineState> {
let mut states = vec![LineState::Code; lines.len()];
rescan(&mut states, lines, language, 0, lines.len());
states
}
pub fn rescan(
states: &mut [LineState],
lines: &[String],
language: Language,
first: usize,
last_changed: usize,
) {
let mut scratch = Vec::new();
for y in first..lines.len().saturating_sub(1) {
scratch.clear();
let next = lex_line(&lines[y], language, states[y], &mut scratch);
if y >= last_changed && states[y + 1] == next {
break;
}
states[y + 1] = next;
}
}