pub const TOKEN_OTHER: u8 = 0;
pub const TOKEN_KEYWORD: u8 = 1;
pub const TOKEN_STRING: u8 = 2;
pub const TOKEN_COMMENT: u8 = 3;
pub const TOKEN_NUMBER: u8 = 4;
pub const TOKEN_FUNCTION: u8 = 5;
pub const TOKEN_TYPE: u8 = 6;
pub const TOKEN_OPERATOR: u8 = 7;
pub const TOKEN_PUNCTUATION: u8 = 8;
pub const TOKEN_VARIABLE: u8 = 9;
pub const TOKEN_CONSTANT: u8 = 10;
pub const TOKEN_BUILTIN: u8 = 11;
pub const TOKEN_ENTITY: u8 = 12;
pub const TOKEN_MARKUP: u8 = 13;
pub const TOKEN_REGEX: u8 = 14;
pub const TOKEN_TYPE_COUNT: u8 = 15;
pub static TOKEN_NAMES: &[&str; 15] = &[
"other", "keyword", "string", "comment", "number",
"function", "type", "operator", "punctuation", "variable",
"constant", "builtin", "entity", "markup", "regex",
];
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct HighlightToken {
pub token_type: u8,
pub len: u16,
}
#[cfg(feature = "highlight")]
pub fn highlight_code(code: &str, language: &str) -> Option<Vec<HighlightToken>> {
use std::sync::OnceLock;
use syntect::parsing::{ParseState, ScopeStack, ScopeStackOp, SyntaxSet};
static SYNTAX_SET: OnceLock<SyntaxSet> = OnceLock::new();
let ss = SYNTAX_SET.get_or_init(|| {
SyntaxSet::load_defaults_newlines()
});
let syntax = ss.find_syntax_by_token(language)?;
let mut parse_state = ParseState::new(syntax);
let mut scope_stack = ScopeStack::new();
let mut tokens: Vec<HighlightToken> = Vec::new();
let mut pos: usize;
let lines: Vec<&str> = code.lines().collect();
if lines.is_empty() {
return Some(Vec::new());
}
let line_count = lines.len();
for (line_idx, line) in lines.iter().enumerate() {
let ops = match parse_state.parse_line(line, ss) {
Ok(ops) => ops,
Err(_) => return None,
};
pos = 0;
for (i, op) in &ops {
let i = *i;
if i > pos {
let text_len = i - pos;
let tt = map_scopes(&scope_stack);
merge_or_push(&mut tokens, tt, text_len as u32);
}
match op {
ScopeStackOp::Push(scope) => scope_stack.push(*scope),
ScopeStackOp::Pop(n) => {
for _ in 0..*n {
scope_stack.pop();
}
}
_ => {}
}
pos = i;
}
if pos < line.len() {
let text_len = line.len() - pos;
let tt = map_scopes(&scope_stack);
merge_or_push(&mut tokens, tt, text_len as u32);
}
if line_idx + 1 < line_count {
tokens.push(HighlightToken { token_type: TOKEN_OTHER, len: 1 });
}
}
if code.ends_with('\n') && !tokens.is_empty() {
tokens.push(HighlightToken { token_type: TOKEN_OTHER, len: 1 });
}
let total: usize = tokens.iter().map(|t| t.len as usize).sum();
if total != code.len() {
return None;
}
Some(tokens)
}
#[cfg(not(feature = "highlight"))]
pub fn highlight_code(_code: &str, _language: &str) -> Option<Vec<HighlightToken>> {
None
}
#[cfg(feature = "highlight")]
fn map_scopes(scopes: &syntect::parsing::ScopeStack) -> u8 {
use std::fmt::Write;
thread_local! {
static SCRATCH: std::cell::RefCell<String> = std::cell::RefCell::new(String::new());
}
let slice = scopes.as_slice();
if slice.is_empty() {
return TOKEN_OTHER;
}
SCRATCH.with(|cell| {
let mut combined = cell.borrow_mut();
combined.clear();
for scope in slice {
let _ = write!(&mut *combined, "{} ", scope);
}
combined.pop(); let combined = combined.as_str();
if combined.contains("comment") {
return TOKEN_COMMENT;
}
if combined.contains("string.regexp") {
return TOKEN_REGEX;
}
if combined.contains("string") {
return TOKEN_STRING;
}
if combined.contains("keyword") && !combined.contains("operator") {
return TOKEN_KEYWORD;
}
if combined.contains("constant.numeric") {
return TOKEN_NUMBER;
}
if combined.contains("entity.name.function")
|| combined.contains("support.function")
|| combined.contains("meta.function")
{
return TOKEN_FUNCTION;
}
if combined.contains("entity.name.type")
|| combined.contains("support.type")
|| combined.contains("storage.type")
{
return TOKEN_TYPE;
}
if combined.contains("keyword.operator")
|| combined.contains("punctuation.definition")
{
return TOKEN_OPERATOR;
}
if combined.contains("punctuation")
|| combined.contains("meta.brace")
|| combined.contains("meta.delimiter")
|| combined.contains("meta.group")
{
return TOKEN_PUNCTUATION;
}
if combined.contains("variable") || combined.contains("meta") {
return TOKEN_VARIABLE;
}
if combined.contains("constant") {
return TOKEN_CONSTANT;
}
if combined.contains("support") {
return TOKEN_BUILTIN;
}
if combined.contains("entity") {
return TOKEN_ENTITY;
}
if combined.contains("markup") {
return TOKEN_MARKUP;
}
TOKEN_OTHER
})
}
#[cfg(feature = "highlight")]
fn merge_or_push(tokens: &mut Vec<HighlightToken>, tt: u8, mut len: u32) {
while len > 0 {
if let Some(last) = tokens.last_mut() {
if last.token_type == tt && last.len < u16::MAX {
let room = (u16::MAX - last.len) as u32;
let add = room.min(len);
last.len += add as u16;
len -= add;
continue;
}
}
let take = len.min(u16::MAX as u32) as u16;
tokens.push(HighlightToken { token_type: tt, len: take });
len -= take as u32;
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_highlight_rust() {
let code = "fn main() {\n println!(\"Hello\");\n}\n";
let tokens = highlight_code(code, "rust");
if let Some(ref tokens) = tokens {
let total: usize = tokens.iter().map(|t| t.len as usize).sum();
assert_eq!(total, code.len(), "Token lengths must sum to code length");
let types: std::collections::HashSet<u8> = tokens.iter().map(|t| t.token_type).collect();
assert!(types.len() >= 2, "Should have multiple token types, got {:?}", tokens);
}
}
#[test]
fn test_highlight_python() {
let code = "def hello():\n return 'world'\n";
let tokens = highlight_code(code, "python");
if let Some(tokens) = tokens {
let total: usize = tokens.iter().map(|t| t.len as usize).sum();
assert_eq!(total, code.len());
}
}
#[test]
fn test_highlight_unknown_language() {
let tokens = highlight_code("text", "nonexistent_lang_xyz");
assert_eq!(tokens, None);
}
#[test]
fn test_highlight_plain_text() {
let tokens = highlight_code("just plain text", "text");
if let Some(tokens) = tokens {
let total: usize = tokens.iter().map(|t| t.len as usize).sum();
assert_eq!(total, "just plain text".len());
}
}
#[test]
fn test_empty_code() {
let tokens = highlight_code("", "rust");
if let Some(tokens) = tokens {
assert!(tokens.is_empty());
}
}
#[cfg(feature = "highlight")]
#[test]
fn merge_or_push_splits_long_runs() {
let mut tokens = Vec::new();
merge_or_push(&mut tokens, TOKEN_STRING, 200_000);
let total: usize = tokens.iter().map(|t| t.len as usize).sum();
assert_eq!(total, 200_000);
assert!(tokens.len() >= 4, "long run must split: {}", tokens.len());
}
#[cfg(feature = "highlight")]
#[test]
fn big_code_highlight_total_matches() {
let code = format!("// {}\n", "x".repeat(100_000));
if let Some(tokens) = highlight_code(&code, "rust") {
let total: usize = tokens.iter().map(|t| t.len as usize).sum();
assert_eq!(total, code.len(), "token lengths must sum to code length");
}
}
}