use incremental_lang::{Grammar, Span};
use jsonx_lib::Value;
use crate::{
document::Document,
json::{to_value, write_str, write_u64},
};
#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)]
pub struct SemanticTokensLegend {
pub token_types: Vec<String>,
pub token_modifiers: Vec<String>,
}
impl SemanticTokensLegend {
#[must_use]
pub fn standard() -> Self {
Self::new(
&[
"namespace",
"type",
"class",
"enum",
"interface",
"struct",
"typeParameter",
"parameter",
"variable",
"property",
"enumMember",
"event",
"function",
"method",
"macro",
"keyword",
"modifier",
"comment",
"string",
"number",
"regexp",
"operator",
"decorator",
],
&[
"declaration",
"definition",
"readonly",
"static",
"deprecated",
"abstract",
"async",
"modification",
"documentation",
"defaultLibrary",
],
)
}
#[must_use]
pub fn new(token_types: &[&str], token_modifiers: &[&str]) -> Self {
Self {
token_types: token_types.iter().map(|&name| name.to_owned()).collect(),
token_modifiers: token_modifiers
.iter()
.map(|&name| name.to_owned())
.collect(),
}
}
#[must_use]
pub fn token_type(&self, name: &str) -> Option<u32> {
self.token_types
.iter()
.position(|t| t == name)
.and_then(|i| u32::try_from(i).ok())
}
#[must_use]
pub fn modifier(&self, name: &str) -> Option<u32> {
let index = self.token_modifiers.iter().position(|m| m == name)?;
1u32.checked_shl(u32::try_from(index).ok()?)
}
#[must_use]
pub fn to_json(&self) -> Value<'static> {
let mut out = String::new();
self.write(&mut out);
to_value(&out)
}
pub(crate) fn write(&self, out: &mut String) {
out.push_str("{\"tokenTypes\":");
write_names(out, &self.token_types);
out.push_str(",\"tokenModifiers\":");
write_names(out, &self.token_modifiers);
out.push('}');
}
}
fn write_names(out: &mut String, names: &[String]) {
out.push('[');
for (i, name) in names.iter().enumerate() {
if i > 0 {
out.push(',');
}
write_str(out, name);
}
out.push(']');
}
#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)]
pub struct SemanticTokens {
data: Vec<u32>,
result_id: Option<String>,
}
impl SemanticTokens {
#[must_use]
pub fn from_data(data: Vec<u32>) -> Self {
Self {
data,
result_id: None,
}
}
#[must_use]
pub fn data(&self) -> &[u32] {
&self.data
}
#[must_use]
pub fn len(&self) -> usize {
self.data.len() / 5
}
#[must_use]
pub fn is_empty(&self) -> bool {
self.data.is_empty()
}
#[must_use]
pub fn result_id(&self) -> Option<&str> {
self.result_id.as_deref()
}
pub(crate) fn with_result_id(mut self, id: String) -> Self {
self.result_id = Some(id);
self
}
pub(crate) fn into_data(self) -> Vec<u32> {
self.data
}
#[must_use]
pub fn to_json(&self) -> Value<'static> {
let mut out = String::new();
self.write(&mut out);
to_value(&out)
}
pub(crate) fn write(&self, out: &mut String) {
out.push('{');
if let Some(id) = &self.result_id {
out.push_str("\"resultId\":");
write_str(out, id);
out.push(',');
}
out.push_str("\"data\":");
write_data(out, &self.data);
out.push('}');
}
}
fn write_data(out: &mut String, data: &[u32]) {
out.push('[');
for (i, &n) in data.iter().enumerate() {
if i > 0 {
out.push(',');
}
write_u64(out, u64::from(n));
}
out.push(']');
}
pub(crate) fn diff(previous: &[u32], next: &[u32]) -> Option<(usize, usize, Vec<u32>)> {
if previous == next {
return None;
}
let prefix = previous
.iter()
.zip(next)
.take_while(|(a, b)| a == b)
.count();
let start = prefix - prefix % 5;
let room = previous.len().min(next.len()) - start;
let suffix = previous
.iter()
.rev()
.zip(next.iter().rev())
.take(room)
.take_while(|(a, b)| a == b)
.count();
let suffix = suffix - suffix % 5;
let delete = previous.len() - start - suffix;
let replacement = next[start..next.len() - suffix].to_vec();
Some((start, delete, replacement))
}
pub(crate) fn write_delta(
out: &mut String,
result_id: &str,
edit: Option<(usize, usize, Vec<u32>)>,
) {
out.push_str("{\"resultId\":");
write_str(out, result_id);
out.push_str(",\"edits\":[");
if let Some((start, delete, data)) = edit {
out.push_str("{\"start\":");
write_u64(out, start as u64);
out.push_str(",\"deleteCount\":");
write_u64(out, delete as u64);
if !data.is_empty() {
out.push_str(",\"data\":");
write_data(out, &data);
}
out.push('}');
}
out.push_str("]}");
}
pub struct SemanticTokensBuilder<'d, G: Grammar> {
document: &'d Document<G>,
tokens: Vec<Pushed>,
}
#[derive(Clone, Copy, Debug)]
struct Pushed {
start: u32,
end: u32,
token_type: u32,
modifiers: u32,
}
impl<'d, G: Grammar> SemanticTokensBuilder<'d, G> {
#[must_use]
pub fn new(document: &'d Document<G>) -> Self {
Self {
document,
tokens: Vec::new(),
}
}
pub fn push(&mut self, span: Span, token_type: u32, modifiers: u32) {
self.tokens.push(Pushed {
start: span.start().to_u32(),
end: span.end().to_u32(),
token_type,
modifiers,
});
}
#[must_use]
pub fn len(&self) -> usize {
self.tokens.len()
}
#[must_use]
pub fn is_empty(&self) -> bool {
self.tokens.is_empty()
}
#[must_use]
pub fn build(self) -> SemanticTokens {
let document = self.document;
let text = document.text();
let mut tokens = self.tokens;
tokens.sort_by_key(|t| (t.start, t.end));
let mut kept: Vec<Pushed> = Vec::with_capacity(tokens.len());
let mut covered = 0u32;
for mut token in tokens {
token.start = floor_boundary(text, token.start);
token.end = floor_boundary(text, token.end);
if token.start >= token.end || token.start < covered {
continue;
}
covered = token.end;
kept.push(token);
}
let starts: Vec<u32> = kept.iter().map(|t| t.start).collect();
let (mut order, mut positions) = (Vec::new(), Vec::new());
document.positions(&starts, &mut order, &mut positions);
let bytes = text.as_bytes();
let mut data = Vec::with_capacity(kept.len() * 5);
let (mut prev_line, mut prev_column) = (0u32, 0u32);
let mut emit = |line: u32, column: u32, length: u32, token: &Pushed| {
let delta_line = line.saturating_sub(prev_line);
let delta_start = if delta_line == 0 {
column.saturating_sub(prev_column)
} else {
column
};
data.extend_from_slice(&[
delta_line,
delta_start,
length,
token.token_type,
token.modifiers,
]);
(prev_line, prev_column) = (line, column);
};
for (token, position) in kept.iter().zip(&positions) {
let (end, mut at) = (token.end as usize, token.start as usize);
let (mut line, mut column) = (position.line, position.character);
loop {
let line_end = crate::lines::find_line_break(&bytes[at..end]).map(|i| at + i);
let segment_end = line_end.unwrap_or(end);
if segment_end > at {
let length = document.width_of(at, segment_end);
emit(line, column, length, token);
}
let Some(break_at) = line_end else { break };
let mut next = break_at + 1;
if bytes[break_at] == b'\r' && bytes.get(next) == Some(&b'\n') {
next += 1;
}
if next >= end {
break;
}
(at, line, column) = (next, line.saturating_add(1), 0);
}
}
SemanticTokens {
data,
result_id: None,
}
}
}
fn floor_boundary(text: &str, offset: u32) -> u32 {
let mut at = (offset as usize).min(text.len());
while !text.is_char_boundary(at) {
at -= 1;
}
at as u32
}
#[cfg(test)]
mod tests {
use proptest::prelude::*;
use super::*;
use crate::{position::Encoding, testing::Chars};
#[allow(clippy::expect_used)]
fn doc(text: &str, encoding: Encoding) -> Document<Chars> {
Document::open(Chars, "file:///t", 1, text.to_owned(), encoding)
.expect("the test document opens")
}
fn span(a: u32, b: u32) -> Span {
Span::new(a, b)
}
#[test]
fn test_tokens_are_delta_encoded_across_lines() {
let d = doc("ab cd\nef", Encoding::Utf16);
let mut b = SemanticTokensBuilder::new(&d);
b.push(span(3, 5), 2, 1);
b.push(span(0, 2), 1, 0);
b.push(span(6, 8), 3, 0);
let t = b.build();
assert_eq!(t.data(), [0, 0, 2, 1, 0, 0, 3, 2, 2, 1, 1, 0, 2, 3, 0]);
assert_eq!(t.len(), 3);
}
#[test]
fn test_utf16_columns_and_lengths_count_surrogate_pairs() {
let d = doc("😀x", Encoding::Utf16);
let mut b = SemanticTokensBuilder::new(&d);
b.push(span(0, 4), 0, 0);
b.push(span(4, 5), 1, 0);
assert_eq!(b.build().data(), [0, 0, 2, 0, 0, 0, 2, 1, 1, 0]);
let d = doc("😀x", Encoding::Utf8);
let mut b = SemanticTokensBuilder::new(&d);
b.push(span(0, 4), 0, 0);
b.push(span(4, 5), 1, 0);
assert_eq!(b.build().data(), [0, 0, 4, 0, 0, 0, 4, 1, 1, 0]);
}
#[test]
fn test_multi_line_tokens_split_per_line() {
let d = doc("a/*x\r\ny\n*/b", Encoding::Utf16);
let mut b = SemanticTokensBuilder::new(&d);
b.push(span(1, 10), 7, 0);
let t = b.build();
assert_eq!(
t.data(),
[
0, 1, 3, 7, 0, 1, 0, 1, 7, 0, 1, 0, 2, 7, 0, ]
);
}
#[test]
fn test_overlapping_empty_and_out_of_range_tokens_are_dropped_or_clamped() {
let d = doc("abcdef", Encoding::Utf16);
let mut b = SemanticTokensBuilder::new(&d);
b.push(span(0, 4), 1, 0);
b.push(span(2, 5), 2, 0); b.push(span(4, 4), 3, 0); b.push(span(4, 99), 4, 0); let t = b.build();
assert_eq!(t.data(), [0, 0, 4, 1, 0, 0, 4, 2, 4, 0]);
assert!(SemanticTokensBuilder::new(&d).build().is_empty());
}
#[test]
fn test_token_boundaries_inside_a_character_round_down() {
let d = doc("é", Encoding::Utf16);
let mut b = SemanticTokensBuilder::new(&d);
b.push(span(1, 2), 0, 0); assert_eq!(b.build().data(), [0, 0, 1, 0, 0]);
}
#[test]
fn test_diff_is_one_token_aligned_edit() {
let a: Vec<u32> = (0..15).collect();
assert_eq!(diff(&a, &a), None);
let mut b = a.clone();
b[7] = 99;
assert_eq!(diff(&a, &b), Some((5, 5, b[5..10].to_vec())));
let mut c = a.clone();
c.extend([1, 2, 3, 4, 5]);
assert_eq!(diff(&a, &c), Some((15, 0, vec![1, 2, 3, 4, 5])));
assert_eq!(diff(&a, &[]), Some((0, 15, vec![])));
for (old, new) in [(&a, &b), (&a, &c)] {
if let Some((start, delete, data)) = diff(old, new) {
let mut applied = old.clone();
let _removed: Vec<u32> = applied.splice(start..start + delete, data).collect();
assert_eq!(&applied, new);
}
}
}
#[test]
fn test_write_delta_and_full_shapes() {
let mut out = String::new();
write_delta(&mut out, "7", Some((5, 5, vec![1, 2, 3, 4, 5])));
assert_eq!(
out,
r#"{"resultId":"7","edits":[{"start":5,"deleteCount":5,"data":[1,2,3,4,5]}]}"#
);
out.clear();
write_delta(&mut out, "8", None);
assert_eq!(out, r#"{"resultId":"8","edits":[]}"#);
out.clear();
write_delta(&mut out, "9", Some((0, 5, vec![])));
assert_eq!(
out,
r#"{"resultId":"9","edits":[{"start":0,"deleteCount":5}]}"#
);
let full = SemanticTokens::from_data(vec![1, 2, 3, 4, 5]).with_result_id("3".into());
assert_eq!(
full.to_json().to_string(),
r#"{"resultId":"3","data":[1,2,3,4,5]}"#
);
}
#[test]
fn test_legend_lookups_and_json() {
let legend = SemanticTokensLegend::standard();
assert_eq!(legend.token_types.len(), 23);
assert_eq!(legend.token_modifiers.len(), 10);
assert_eq!(legend.modifier("defaultLibrary"), Some(1 << 9));
let json = SemanticTokensLegend::new(&["a"], &["m"]).to_json();
assert_eq!(
json.to_string(),
r#"{"tokenTypes":["a"],"tokenModifiers":["m"]}"#
);
let many = SemanticTokensLegend {
token_types: vec![],
token_modifiers: (0..33).map(|i| i.to_string()).collect(),
};
assert_eq!(many.modifier("31"), Some(1 << 31));
assert_eq!(many.modifier("32"), None);
}
fn tokens_strategy() -> impl Strategy<Value = Vec<u32>> {
prop::collection::vec(prop::collection::vec(0u32..3, 5), 0..8)
.prop_map(|tokens| tokens.concat())
}
proptest! {
#![proptest_config(ProptestConfig::with_cases(1024))]
#[test]
fn prop_diff_applied_to_the_previous_gives_the_next(
previous in tokens_strategy(),
next in tokens_strategy(),
) {
match diff(&previous, &next) {
None => prop_assert_eq!(&previous, &next),
Some((start, delete, data)) => {
prop_assert_eq!(start % 5, 0);
prop_assert_eq!(delete % 5, 0);
prop_assert_eq!(data.len() % 5, 0);
prop_assert!(start + delete <= previous.len());
let mut applied = previous.clone();
let _removed: Vec<u32> = applied.splice(start..start + delete, data).collect();
prop_assert_eq!(&applied, &next);
prop_assert_ne!(&previous, &next);
}
}
}
}
}