use std::{any::Any, fmt, sync::Arc};
use super::{CodeToken, CodeTokenizer};
#[derive(Clone, Default)]
pub struct LineState(Option<Arc<dyn StateValue>>);
trait StateValue: Any + Send + Sync {
fn same(&self, other: &dyn StateValue) -> bool;
fn as_any(&self) -> &dyn Any;
}
impl<T: Clone + PartialEq + Send + Sync + 'static> StateValue for T {
fn same(&self, other: &dyn StateValue) -> bool {
other.as_any().downcast_ref::<T>() == Some(self)
}
fn as_any(&self) -> &dyn Any {
self
}
}
impl LineState {
pub fn start() -> Self {
Self(None)
}
pub fn new<T: Clone + PartialEq + Send + Sync + 'static>(value: T) -> Self {
Self(Some(Arc::new(value)))
}
pub fn get<T: 'static>(&self) -> Option<&T> {
self.0.as_ref().and_then(|value| value.as_any().downcast_ref::<T>())
}
}
impl PartialEq for LineState {
fn eq(&self, other: &Self) -> bool {
match (&self.0, &other.0) {
(None, None) => true,
(Some(one), Some(other)) => one.same(other.as_ref()),
_ => false,
}
}
}
impl fmt::Debug for LineState {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(if self.0.is_some() {
"LineState(..)"
} else {
"LineState::start"
})
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct BufferId(pub u64);
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)]
pub struct CodePoint {
pub row: usize,
pub column: usize,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)]
pub struct CodeEdit {
pub start_byte: usize,
pub old_end_byte: usize,
pub new_end_byte: usize,
pub start_point: CodePoint,
pub old_end_point: CodePoint,
pub new_end_point: CodePoint,
}
impl CodeEdit {
pub fn between(old: &str, new: &str) -> Option<Self> {
if old == new {
return None;
}
let mut start = old
.bytes()
.zip(new.bytes())
.take_while(|(one, other)| one == other)
.count();
while !old.is_char_boundary(start) || !new.is_char_boundary(start) {
start -= 1;
}
let most = old.len().min(new.len()) - start;
let mut same_end = old
.bytes()
.rev()
.zip(new.bytes().rev())
.take(most)
.take_while(|(one, other)| one == other)
.count();
while !old.is_char_boundary(old.len() - same_end) || !new.is_char_boundary(new.len() - same_end) {
same_end -= 1;
}
let (old_end, new_end) = (old.len() - same_end, new.len() - same_end);
Some(Self {
start_byte: start,
old_end_byte: old_end,
new_end_byte: new_end,
start_point: point(new, start),
old_end_point: point(old, old_end),
new_end_point: point(new, new_end),
})
}
}
fn point(text: &str, at: usize) -> CodePoint {
let before = &text[..at];
let row = before.bytes().filter(|byte| *byte == b'\n').count();
let column = before.rfind('\n').map_or(at, |newline| at - newline - 1);
CodePoint { row, column }
}
const BATCH: usize = 32;
#[derive(Clone)]
struct KeptLine {
start: LineState,
tokens: Arc<[CodeToken]>,
}
#[derive(Default)]
pub(crate) struct LineCache {
tokenizer: Option<(u64, u64)>,
code: Arc<str>,
lines: Vec<KeptLine>,
all: Option<Arc<[CodeToken]>>,
#[cfg(test)]
pub(crate) tokenized: usize,
}
impl LineCache {
pub(crate) fn tokens(&mut self, tokenizer: &CodeTokenizer, code: &str) -> Arc<[CodeToken]> {
self.tokens_with(tokenizer, code, |tokenizer, code| tokenizer.tokens(code).to_vec())
}
pub(crate) fn tokens_with(
&mut self,
tokenizer: &CodeTokenizer,
code: &str,
whole: impl FnOnce(&CodeTokenizer, &str) -> Vec<CodeToken>,
) -> Arc<[CodeToken]> {
let mut whole = Some(whole);
let key = (tokenizer.id(), tokenizer.generation());
if self.tokenizer != Some(key) {
*self = Self {
tokenizer: Some(key),
#[cfg(test)]
tokenized: self.tokenized,
..Self::default()
};
} else if let Some(all) = &self.all
&& &*self.code == code
{
return all.clone();
}
let new: Vec<&str> = code.split('\n').collect();
let (first, tail) = if self.lines.is_empty() || self.all.is_none() {
(0, 0)
} else {
let start = self
.code
.bytes()
.zip(code.bytes())
.take_while(|(one, other)| one == other)
.count();
let most = self.code.len().min(code.len()) - start;
let same_end = self
.code
.bytes()
.rev()
.zip(code.bytes().rev())
.take(most)
.take_while(|(one, other)| one == other)
.count();
let first = code[..start].bytes().filter(|byte| *byte == b'\n').count();
let tail = code[code.len() - same_end..]
.bytes()
.filter(|byte| *byte == b'\n')
.count();
(first, tail.min(new.len() - first - 1).min(self.lines.len() - first - 1))
};
let mut lines: Vec<KeptLine> = self.lines[..first.min(self.lines.len())].to_vec();
let mut state = self
.lines
.get(first)
.map_or_else(LineState::start, |line| line.start.clone());
let mut at = first;
let mut resumed = None;
'lines: while at < new.len() {
let batch = &new[at..(at + BATCH).min(new.len())];
let Some(found) = tokenizer.tokens_from(batch, &state) else {
return self.whole(code, whole.take().map_or_else(Vec::new, |whole| whole(tokenizer, code)));
};
if found.len() != batch.len() {
return self.whole(code, whole.take().map_or_else(Vec::new, |whole| whole(tokenizer, code)));
}
#[cfg(test)]
{
self.tokenized += batch.len();
}
for (line, (tokens, end)) in batch.iter().zip(found) {
lines.push(KeptLine {
start: state,
tokens: super::normalized(line, tokens).into(),
});
state = end;
at += 1;
if tail > 0 && at >= new.len() - tail {
let old = at + self.lines.len() - new.len();
if old < self.lines.len() && self.lines[old].start == state {
resumed = Some(old);
break 'lines;
}
}
}
}
if let Some(old) = resumed {
lines.extend_from_slice(&self.lines[old..]);
}
let mut all = Vec::new();
let mut offset = 0;
for (line, kept) in new.iter().zip(&lines) {
all.extend(kept.tokens.iter().map(|token| CodeToken {
range: token.range.start + offset..token.range.end + offset,
kind: token.kind,
}));
offset += line.len() + 1;
}
let all: Arc<[CodeToken]> = all.into();
self.lines = lines;
self.code = Arc::from(code);
self.all = Some(all.clone());
all
}
fn whole(&mut self, code: &str, all: Vec<CodeToken>) -> Arc<[CodeToken]> {
let all: Arc<[CodeToken]> = super::normalized(code, all).into();
self.lines.clear();
self.code = Arc::from(code);
self.all = Some(all.clone());
all
}
}