use crate::lsp::{Encoding, SemanticToken};
use crate::snippets::Language;
#[cfg(test)]
mod tests;
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum TokenKind {
Keyword,
Function,
Type,
String,
Constant,
Comment,
Macro,
Namespace,
}
impl TokenKind {
pub fn from_name(name: &str) -> Option<Self> {
Some(match name {
"keyword" | "modifier" | "selfKeyword" | "selfTypeKeyword" => Self::Keyword,
"function" | "method" | "member" => Self::Function,
"type" | "class" | "struct" | "enum" | "interface" | "union" | "typeParameter"
| "typeAlias" | "builtinType" | "concept" => Self::Type,
"string" | "char" | "character" | "regexp" | "escapeSequence" | "formatSpecifier" => {
Self::String
}
"number" | "boolean" | "enumMember" | "const" | "constParameter" => Self::Constant,
"comment" => Self::Comment,
"macro" | "decorator" | "attribute" | "derive" | "builtinAttribute" | "lifetime"
| "label" => Self::Macro,
"namespace" | "module" => Self::Namespace,
_ => return None,
})
}
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct Token {
pub start: usize,
pub end: usize,
pub kind: Option<TokenKind>,
}
#[derive(Default)]
pub struct Highlights {
lines: Vec<Vec<Token>>,
}
impl Highlights {
pub fn new(tokens: &[SemanticToken], lines: &[String], encoding: Encoding) -> Self {
let mut table = vec![Vec::new(); lines.len()];
for token in tokens {
let Some(line) = lines.get(token.line) else {
continue;
};
let start = encoding.to_chars(line, token.start);
let end = encoding.to_chars(line, token.start + token.length);
if start < end {
table[token.line].push(Token {
start,
end,
kind: token.kind,
});
}
}
Self { lines: table }
}
pub fn on(&self, y: usize) -> &[Token] {
self.lines.get(y).map_or(&[], Vec::as_slice)
}
pub fn inserted(&mut self, x: usize, y: usize) {
for token in self.lines.get_mut(y).into_iter().flatten() {
if token.start >= x {
token.start += 1;
token.end += 1;
} else if token.end >= x {
token.end += 1;
}
}
}
pub fn deleted(&mut self, x: usize, y: usize) {
if let Some(tokens) = self.lines.get_mut(y) {
for token in tokens.iter_mut() {
if token.start > x {
token.start -= 1;
token.end -= 1;
} else if token.end > x {
token.end -= 1;
}
}
tokens.retain(|token| token.start < token.end);
}
}
pub fn split(&mut self, x: usize, y: usize) {
let Some(tokens) = self.lines.get_mut(y) else {
return;
};
let mut below = Vec::new();
for token in tokens.iter_mut() {
if token.end > x {
below.push(Token {
start: token.start.saturating_sub(x),
end: token.end - x,
kind: token.kind,
});
token.end = token.end.min(x);
}
}
tokens.retain(|token| token.start < token.end);
self.lines.insert(y + 1, below);
}
pub fn joined(&mut self, y: usize, len: usize) {
if y + 1 < self.lines.len() {
let below = self.lines.remove(y + 1);
self.lines[y].extend(below.into_iter().map(|token| Token {
start: token.start + len,
end: token.end + len,
kind: token.kind,
}));
}
}
}
#[derive(Clone, Copy, Debug, Default, PartialEq)]
pub enum LineState {
#[default]
Code,
Comment,
Text(Delimiter),
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum Delimiter {
Double,
Backtick,
Triple(char),
Raw(usize),
}
struct Syntax {
keywords: &'static [&'static str],
constants: &'static [&'static str],
types: &'static [&'static str],
line_comment: &'static str,
block_comments: bool,
}
impl Syntax {
fn of(language: Language) -> &'static Self {
match language {
Language::Rust => &Self {
keywords: &[
"as", "async", "await", "break", "const", "continue", "crate", "dyn", "else",
"enum", "extern", "fn", "for", "if", "impl", "in", "let", "loop", "match",
"mod", "move", "mut", "pub", "ref", "return", "self", "Self", "static",
"struct", "super", "trait", "type", "unsafe", "use", "where", "while",
],
constants: &["true", "false"],
types: &[
"bool", "char", "str", "u8", "u16", "u32", "u64", "u128", "usize", "i8",
"i16", "i32", "i64", "i128", "isize", "f32", "f64",
],
line_comment: "//",
block_comments: true,
},
Language::Python => &Self {
keywords: &[
"and", "as", "assert", "async", "await", "break", "class", "continue", "def",
"del", "elif", "else", "except", "finally", "for", "from", "global", "if",
"import", "in", "is", "lambda", "nonlocal", "not", "or", "pass", "raise",
"return", "self", "try", "while", "with", "yield",
],
constants: &["True", "False", "None"],
types: &[
"bool", "bytes", "dict", "float", "int", "list", "object", "set", "str",
"tuple",
],
line_comment: "#",
block_comments: false,
},
Language::JavaScript => &Self {
keywords: &[
"abstract", "as", "async", "await", "break", "case", "catch", "class",
"const", "continue", "debugger", "declare", "default", "delete", "do",
"else", "enum", "export", "extends", "finally", "for", "from", "function",
"if", "implements", "import", "in", "instanceof", "interface", "let",
"namespace", "new", "of", "private", "protected", "public", "readonly",
"return", "static", "super", "switch", "this", "throw", "try", "type",
"typeof", "var", "void", "while", "with", "yield",
],
constants: &["true", "false", "null", "undefined", "NaN"],
types: &["any", "boolean", "never", "number", "object", "string", "unknown"],
line_comment: "//",
block_comments: true,
},
Language::C => &Self {
keywords: &[
"auto", "break", "case", "catch", "class", "const", "constexpr", "continue",
"default", "delete", "do", "else", "enum", "extern", "for", "goto", "if",
"inline", "namespace", "new", "operator", "override", "private",
"protected", "public", "register", "return", "sizeof", "static", "struct",
"switch", "template", "this", "throw", "try", "typedef", "typename",
"union", "using", "virtual", "volatile", "while",
],
constants: &["true", "false", "NULL", "nullptr"],
types: &[
"bool", "char", "double", "float", "int", "long", "short", "signed",
"size_t", "unsigned", "void",
],
line_comment: "//",
block_comments: true,
},
Language::Go => &Self {
keywords: &[
"break", "case", "chan", "const", "continue", "default", "defer", "else",
"fallthrough", "for", "func", "go", "goto", "if", "import", "interface",
"map", "package", "range", "return", "select", "struct", "switch", "type",
"var",
],
constants: &["true", "false", "nil", "iota"],
types: &[
"any", "bool", "byte", "error", "float32", "float64", "int", "int8",
"int16", "int32", "int64", "rune", "string", "uint", "uint8", "uint16",
"uint32", "uint64",
],
line_comment: "//",
block_comments: true,
},
}
}
}
fn is_word(c: char) -> bool {
c.is_alphanumeric() || c == '_'
}
fn closing(chars: &[char], from: usize, state: LineState, language: Language) -> Option<usize> {
let at = |i: usize, c: char| chars.get(i) == Some(&c);
let escapes = match state {
LineState::Text(Delimiter::Raw(_)) => false,
LineState::Text(Delimiter::Backtick) => language != Language::Go,
_ => true,
};
let mut i = from;
while i < chars.len() {
let end = match state {
LineState::Code => return Some(from),
LineState::Comment => (at(i, '*') && at(i + 1, '/')).then_some(i + 2),
LineState::Text(Delimiter::Double) => at(i, '"').then_some(i + 1),
LineState::Text(Delimiter::Backtick) => at(i, '`').then_some(i + 1),
LineState::Text(Delimiter::Triple(quote)) => {
(0..3).all(|k| at(i + k, quote)).then_some(i + 3)
}
LineState::Text(Delimiter::Raw(hashes)) => {
(at(i, '"') && (1..=hashes).all(|k| at(i + k, '#'))).then_some(i + 1 + hashes)
}
};
if end.is_some() {
return end;
}
if escapes && state != LineState::Comment && chars[i] == '\\' {
i += 1;
}
i += 1;
}
None
}
fn quoted_end(chars: &[char], start: usize, quote: char) -> usize {
let mut i = start + 1;
while i < chars.len() {
match chars[i] {
'\\' => i += 1,
c if c == quote => return i + 1,
_ => {}
}
i += 1;
}
chars.len()
}
pub fn lex_line(
line: &str,
language: Language,
state: LineState,
out: &mut Vec<Token>,
) -> LineState {
let syntax = Syntax::of(language);
let chars: Vec<char> = line.chars().collect();
let len = chars.len();
let at = |i: usize, c: char| chars.get(i) == Some(&c);
let mut push = |start: usize, end: usize, kind: TokenKind| {
if start < end {
out.push(Token {
start,
end,
kind: Some(kind),
});
}
};
let mut open = state;
let mut open_start = 0;
let mut i = 0;
loop {
if open != LineState::Code {
let kind = match open {
LineState::Comment => TokenKind::Comment,
_ => TokenKind::String,
};
match closing(&chars, i, open, language) {
Some(end) => {
push(open_start, end, kind);
i = end;
open = LineState::Code;
}
None => {
push(open_start, len, kind);
return open;
}
}
}
if i >= len {
return LineState::Code;
}
let c = chars[i];
if c.is_whitespace() {
i += 1;
} else if syntax.line_comment.chars().enumerate().all(|(k, c)| at(i + k, c)) {
push(i, len, TokenKind::Comment);
return LineState::Code;
} else if syntax.block_comments && c == '/' && at(i + 1, '*') {
(open, open_start) = (LineState::Comment, i);
i += 2;
} else if c == '"' || c == '\'' || c == '`' {
let multi_line = match (language, c) {
(Language::Rust, '"') => Some((Delimiter::Double, 1)),
(Language::Python, _) if c != '`' && at(i + 1, c) && at(i + 2, c) => {
Some((Delimiter::Triple(c), 3))
}
(Language::JavaScript | Language::Go, '`') => Some((Delimiter::Backtick, 1)),
_ => None,
};
if let Some((delimiter, width)) = multi_line {
(open, open_start) = (LineState::Text(delimiter), i);
i += width;
} else if c == '`' {
i += 1;
} else if language == Language::Rust && !at(i + 1, '\\') && !at(i + 2, '\'') {
let end = (i + 1..len).find(|j| !is_word(chars[*j])).unwrap_or(len);
push(i, end, TokenKind::Macro);
i = end.max(i + 1);
} else {
let end = quoted_end(&chars, i, c);
push(i, end, TokenKind::String);
i = end;
}
} else if c.is_ascii_digit() {
let mut end = i + 1;
while end < len
&& (is_word(chars[end])
|| (chars[end] == '.' && chars.get(end + 1).is_some_and(char::is_ascii_digit)))
{
end += 1;
}
push(i, end, TokenKind::Constant);
i = end;
} else if is_word(c) {
let mut end = (i..len).find(|j| !is_word(chars[*j])).unwrap_or(len);
let word: String = chars[i..end].iter().collect();
let raw_hashes = (end..len).take_while(|j| chars[*j] == '#').count();
if language == Language::Rust
&& (word == "r" || word == "br")
&& at(end + raw_hashes, '"')
{
(open, open_start) = (LineState::Text(Delimiter::Raw(raw_hashes)), i);
i = end + raw_hashes + 1;
continue;
}
let kind = if syntax.keywords.contains(&word.as_str()) {
Some(TokenKind::Keyword)
} else if syntax.constants.contains(&word.as_str()) {
Some(TokenKind::Constant)
} else if syntax.types.contains(&word.as_str()) {
Some(TokenKind::Type)
} else if language == Language::Rust && at(end, '!') && !at(end + 1, '=') {
end += 1;
Some(TokenKind::Macro)
} else if at(end, '(') {
Some(TokenKind::Function)
} else if c.is_uppercase() {
let shouting = word.len() > 1 && !word.chars().any(char::is_lowercase);
Some(if shouting { TokenKind::Constant } else { TokenKind::Type })
} else {
None
};
if let Some(kind) = kind {
push(i, end, kind);
}
i = end;
} else if (c == '@' && language == Language::Python)
|| (c == '#' && language == Language::C)
{
let end = (i + 1..len).find(|j| !is_word(chars[*j])).unwrap_or(len);
push(i, end, TokenKind::Macro);
i = end.max(i + 1);
} else if c == '#' && language == Language::Rust && (at(i + 1, '[') || at(i + 1, '!')) {
let end = (i..len).find(|j| chars[*j] == ']').map_or(len, |j| j + 1);
push(i, end, TokenKind::Macro);
i = end;
} else {
i += 1;
}
}
}
pub fn line_states(lines: &[String], language: Language) -> Vec<LineState> {
let mut states = vec![LineState::Code; lines.len()];
rescan(&mut states, lines, language, 0, lines.len());
states
}
pub fn rescan(
states: &mut [LineState],
lines: &[String],
language: Language,
first: usize,
last_changed: usize,
) {
let mut scratch = Vec::new();
for y in first..lines.len().saturating_sub(1) {
scratch.clear();
let next = lex_line(&lines[y], language, states[y], &mut scratch);
if y >= last_changed && states[y + 1] == next {
break;
}
states[y + 1] = next;
}
}