use std::{
collections::HashMap,
fmt,
sync::{
Arc, LazyLock,
atomic::{AtomicU64, Ordering},
},
};
use parking_lot::RwLock;
use super::{BufferId, CodeEdit, CodeToken, LineState, cache};
use crate::core::Signal;
pub trait Tokenizer: Send + Sync + 'static {
fn tokens(&self, code: &str) -> Vec<CodeToken>;
fn tokens_from(&self, lines: &[&str], start: &LineState) -> Option<Vec<(Vec<CodeToken>, LineState)>> {
let _ = (lines, start);
None
}
fn tokens_in(&self, buffer: BufferId, code: &str) -> Vec<CodeToken> {
let _ = buffer;
self.tokens(code)
}
fn edited(&self, buffer: BufferId, edit: &CodeEdit) {
let _ = (buffer, edit);
}
fn dropped(&self, buffer: BufferId) {
let _ = buffer;
}
}
impl<F> Tokenizer for F
where
F: Fn(&str) -> Vec<CodeToken> + Send + Sync + 'static,
{
fn tokens(&self, code: &str) -> Vec<CodeToken> {
self(code)
}
}
static NEXT_TOKENIZER: AtomicU64 = AtomicU64::new(1);
#[derive(Clone)]
pub struct CodeTokenizer {
id: u64,
tokenizer: Arc<dyn Tokenizer>,
generation: Signal<u64>,
}
#[derive(Clone)]
pub struct TokenizerHandle {
id: u64,
generation: Signal<u64>,
}
impl TokenizerHandle {
pub fn invalidate(&self) {
cache::forget(self.id);
self
.generation
.update(|generation| *generation = generation.wrapping_add(1));
}
pub fn id(&self) -> u64 {
self.id
}
}
impl fmt::Debug for TokenizerHandle {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_tuple("TokenizerHandle").field(&self.id).finish()
}
}
impl CodeTokenizer {
pub fn new(tokenizer: impl Tokenizer) -> Self {
Self {
id: NEXT_TOKENIZER.fetch_add(1, Ordering::Relaxed),
tokenizer: Arc::new(tokenizer),
generation: Signal::new(0),
}
}
pub fn tokens(&self, code: &str) -> Arc<[CodeToken]> {
let generation = self.generation.get();
cache::tokens(self.id, generation, code, || {
super::normalized(code, self.tokenizer.tokens(code))
})
}
pub fn handle(&self) -> TokenizerHandle {
TokenizerHandle {
id: self.id,
generation: self.generation.clone(),
}
}
pub(crate) fn generation(&self) -> u64 {
self.generation.get_untracked()
}
pub(crate) fn track(&self) {
let _ = self.generation.get();
}
pub(crate) fn tokens_from(&self, lines: &[&str], start: &LineState) -> Option<Vec<(Vec<CodeToken>, LineState)>> {
self.tokenizer.tokens_from(lines, start)
}
pub(crate) fn tokens_in(&self, buffer: BufferId, code: &str) -> Vec<CodeToken> {
super::normalized(code, self.tokenizer.tokens_in(buffer, code))
}
pub(crate) fn edited(&self, buffer: BufferId, edit: &CodeEdit) {
self.tokenizer.edited(buffer, edit);
}
pub(crate) fn dropped(&self, buffer: BufferId) {
self.tokenizer.dropped(buffer);
}
pub fn id(&self) -> u64 {
self.id
}
}
impl<H: Tokenizer> From<H> for CodeTokenizer {
fn from(tokenizer: H) -> Self {
Self::new(tokenizer)
}
}
impl PartialEq for CodeTokenizer {
fn eq(&self, other: &Self) -> bool {
self.id == other.id
}
}
impl fmt::Debug for CodeTokenizer {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_tuple("CodeTokenizer").field(&self.id).finish()
}
}
impl crate::app::component::DevtoolsInspectable for CodeTokenizer {}
struct Language {
name: Arc<str>,
aliases: Vec<Arc<str>>,
tokenizer: CodeTokenizer,
}
#[derive(Default)]
struct Registry {
languages: HashMap<Arc<str>, Language>,
names: HashMap<Arc<str>, Arc<str>>,
changes: HashMap<Arc<str>, Signal<u64>>,
}
static REGISTRY: LazyLock<RwLock<Registry>> = LazyLock::new(|| {
let mut registry = Registry::default();
for language in super::languages::all() {
let name = language.name();
let aliases: Vec<Arc<str>> = language.aliases_of().to_vec();
registry.insert(&name, &aliases, CodeTokenizer::new(language));
}
RwLock::new(registry)
});
fn key(language: &str) -> Arc<str> {
let language = language.trim().trim_start_matches(['{', '.']);
let end = language
.find(|ch: char| ch.is_whitespace() || matches!(ch, ',' | '}' | '{'))
.unwrap_or(language.len());
Arc::from(language[..end].to_lowercase())
}
impl Registry {
fn insert(
&mut self,
name: &str,
aliases: &[Arc<str>],
tokenizer: CodeTokenizer,
) -> (Vec<Arc<str>>, Option<CodeTokenizer>) {
let name = key(name);
let own = self.names.get(&name).cloned().unwrap_or_else(|| name.clone());
let language = self.languages.entry(own.clone()).or_insert_with(|| Language {
name: own.clone(),
aliases: Vec::new(),
tokenizer: tokenizer.clone(),
});
let replaced = (language.tokenizer != tokenizer).then(|| std::mem::replace(&mut language.tokenizer, tokenizer));
let mut moved = Vec::new();
for alias in std::iter::once(&name).chain(aliases) {
let alias = key(alias);
if alias != language.name && !language.aliases.contains(&alias) {
language.aliases.push(alias.clone());
}
moved.push(alias);
}
let mut changed: Vec<Arc<str>> = std::iter::once(language.name.clone())
.chain(language.aliases.iter().cloned())
.collect();
for alias in moved {
if let Some(other) = self.names.insert(alias.clone(), own.clone())
&& other != own
&& let Some(other) = self.languages.get_mut(&other)
{
other.aliases.retain(|held| *held != alias);
}
if !changed.contains(&alias) {
changed.push(alias);
}
}
self.names.insert(own.clone(), own);
(changed, replaced)
}
}
fn tell(names: Vec<Arc<str>>) {
let signals: Vec<Signal<u64>> = {
let registry = REGISTRY.read();
names
.iter()
.filter_map(|name| registry.changes.get(name).cloned())
.collect()
};
for signal in signals {
signal.update(|version| *version = version.wrapping_add(1));
}
}
pub fn register_tokenizer(language: &str, tokenizer: impl Into<CodeTokenizer>) -> TokenizerHandle {
register_tokenizer_with_aliases(language, &[], tokenizer)
}
pub fn register_tokenizer_with_aliases(
language: &str,
aliases: &[&str],
tokenizer: impl Into<CodeTokenizer>,
) -> TokenizerHandle {
let tokenizer = tokenizer.into();
let handle = tokenizer.handle();
let aliases: Vec<Arc<str>> = aliases.iter().map(|alias| Arc::from(*alias)).collect();
let (changed, replaced) = REGISTRY.write().insert(language, &aliases, tokenizer);
if let Some(replaced) = replaced {
cache::forget(replaced.id());
}
tell(changed);
handle
}
pub fn remove_tokenizer(language: &str) -> bool {
let name = key(language);
let removed = {
let mut registry = REGISTRY.write();
let Some(own) = registry.names.get(&name).cloned() else {
return false;
};
let Some(language) = registry.languages.remove(&own) else {
return false;
};
registry.names.retain(|_, language| *language != own);
language
};
cache::forget(removed.tokenizer.id());
tell(std::iter::once(removed.name).chain(removed.aliases).collect());
true
}
pub fn tokenizer(language: &str) -> Option<CodeTokenizer> {
let name = key(language);
let (found, change) = {
let registry = REGISTRY.read();
let found = registry
.names
.get(&name)
.and_then(|own| registry.languages.get(own))
.map(|language| language.tokenizer.clone());
(found, registry.changes.get(&name).cloned())
};
let change = change.unwrap_or_else(|| {
REGISTRY
.write()
.changes
.entry(name)
.or_insert_with(|| Signal::new(0))
.clone()
});
let _ = change.get();
found
}
pub fn knows(language: &str) -> bool {
let name = key(language);
REGISTRY.read().names.contains_key(&name)
}
pub fn languages() -> Vec<(Arc<str>, Vec<Arc<str>>)> {
let registry = REGISTRY.read();
let mut all: Vec<_> = registry
.languages
.values()
.map(|language| (language.name.clone(), language.aliases.clone()))
.collect();
all.sort_by(|a, b| a.0.cmp(&b.0));
all
}
pub fn tokens(code: &str, language: &str) -> Arc<[CodeToken]> {
match tokenizer(language) {
Some(tokenizer) if !code.is_empty() => tokenizer.tokens(code),
_ => Arc::from([]),
}
}