use lsp_types::{Position, PositionEncodingKind};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum PositionEncoding {
Utf8,
#[default]
Utf16,
}
impl PositionEncoding {
pub fn to_kind(self) -> PositionEncodingKind {
match self {
PositionEncoding::Utf8 => PositionEncodingKind::UTF8,
PositionEncoding::Utf16 => PositionEncodingKind::UTF16,
}
}
fn metric(self) -> Metric {
match self {
PositionEncoding::Utf8 => Metric::Utf8,
PositionEncoding::Utf16 => Metric::Utf16,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct LineCol {
pub line: usize,
pub column: usize,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct WideChar {
start: u32,
end: u32,
}
impl WideChar {
fn len(self) -> u32 {
self.end - self.start
}
fn len_utf16(self) -> u32 {
if self.len() == 4 { 2 } else { 1 }
}
}
#[derive(Clone, Copy)]
enum Metric {
Utf8,
Utf16,
CodePoint,
}
impl Metric {
fn wide_units(self, w: WideChar) -> u32 {
match self {
Metric::Utf8 => w.len(),
Metric::Utf16 => w.len_utf16(),
Metric::CodePoint => 1,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct LineIndex {
line_starts: Vec<usize>,
wide_chars: Vec<WideChar>,
len: usize,
}
impl LineIndex {
pub fn new(text: &str) -> Self {
debug_assert!(
text.len() <= u32::MAX as usize,
"LineIndex records wide chars as u32 offsets"
);
let bytes = text.as_bytes();
let mut line_starts = Vec::with_capacity(bytes.len() / 40 + 1);
line_starts.push(0);
line_starts.extend(memchr::memchr_iter(b'\n', bytes).map(|i| i + 1));
let wide_chars = if bytes.is_ascii() {
Vec::new()
} else {
text.char_indices()
.filter(|(_, ch)| ch.len_utf8() > 1)
.map(|(offset, ch)| WideChar {
start: offset as u32,
end: (offset + ch.len_utf8()) as u32,
})
.collect()
};
Self {
line_starts,
wide_chars,
len: text.len(),
}
}
pub fn apply_edit(&mut self, range: std::ops::Range<usize>, insert: &str) {
let (start, end) = (range.start, range.end);
debug_assert!(
start <= end && end <= self.len,
"edit {range:?} out of range"
);
debug_assert!(
self.is_char_boundary(start) && self.is_char_boundary(end),
"edit {range:?} splits a wide char"
);
let removed = end - start;
let inserted = insert.len();
let shift = removed != inserted;
let first = self.line_starts.partition_point(|&s| s <= start);
let last = self.line_starts.partition_point(|&s| s <= end);
if shift {
for s in &mut self.line_starts[last..] {
*s = *s - removed + inserted;
}
}
self.line_starts.splice(
first..last,
memchr::memchr_iter(b'\n', insert.as_bytes()).map(|i| start + i + 1),
);
let wfirst = self
.wide_chars
.partition_point(|w| (w.start as usize) < start);
let wlast = self
.wide_chars
.partition_point(|w| (w.start as usize) < end);
if shift {
for w in &mut self.wide_chars[wlast..] {
w.start = (w.start as usize - removed + inserted) as u32;
w.end = (w.end as usize - removed + inserted) as u32;
}
}
if insert.is_ascii() {
self.wide_chars.splice(wfirst..wlast, std::iter::empty());
} else {
let new: Vec<WideChar> = insert
.char_indices()
.filter(|(_, ch)| ch.len_utf8() > 1)
.map(|(i, ch)| WideChar {
start: (start + i) as u32,
end: (start + i + ch.len_utf8()) as u32,
})
.collect();
self.wide_chars.splice(wfirst..wlast, new);
}
self.len = self.len - removed + inserted;
}
fn is_char_boundary(&self, offset: usize) -> bool {
!self
.wide_chars_from(offset.saturating_sub(3))
.iter()
.take_while(|w| (w.start as usize) < offset)
.any(|w| (w.end as usize) > offset)
}
pub fn byte_to_lc(&self, offset: usize) -> LineCol {
let clamped = offset.min(self.len);
let line = self.line_index_for(clamped);
LineCol {
line: line + 1,
column: self.col_in(self.line_starts[line], clamped, Metric::CodePoint) as usize + 1,
}
}
pub fn byte_to_position(&self, offset: usize, encoding: PositionEncoding) -> Position {
let clamped = offset.min(self.len);
let line = self.line_index_for(clamped);
let character = self.col_in(self.line_starts[line], clamped, encoding.metric());
Position::new(line as u32, character)
}
pub fn position_to_byte(&self, position: Position, encoding: PositionEncoding) -> usize {
let line = position.line as usize;
if line >= self.line_starts.len() {
return self.len;
}
self.byte_at_col(line, position.character, encoding.metric())
}
pub fn byte_to_line(&self, offset: usize) -> u32 {
self.line_index_for(offset.min(self.len)) as u32
}
pub fn line_count(&self) -> usize {
self.line_starts.len()
}
pub fn line_start(&self, line: usize) -> usize {
self.line_starts.get(line).copied().unwrap_or(self.len)
}
fn wide_chars_from(&self, from: usize) -> &[WideChar] {
let i = self
.wide_chars
.partition_point(|w| (w.start as usize) < from);
&self.wide_chars[i..]
}
fn col_in(&self, line_start: usize, offset: usize, metric: Metric) -> u32 {
let rel = (offset - line_start) as u32;
if self.wide_chars.is_empty() {
return rel;
}
let mut shortfall = 0u32;
for w in self.wide_chars_from(line_start) {
if w.end as usize > offset {
break;
}
shortfall += w.len() - metric.wide_units(*w);
}
rel - shortfall
}
fn byte_at_col(&self, line: usize, target_col: u32, metric: Metric) -> usize {
let line_start = self.line_starts[line];
let line_end = self.line_starts.get(line + 1).copied().unwrap_or(self.len);
if self.wide_chars.is_empty() {
return (line_start + target_col as usize).min(line_end);
}
let mut col = 0u32;
let mut byte = line_start;
for w in self.wide_chars_from(line_start) {
let w_start = w.start as usize;
if w_start >= line_end {
break;
}
let ascii = (w_start - byte) as u32;
if col + ascii >= target_col {
return byte + (target_col - col) as usize;
}
col += ascii;
byte = w_start;
let units = metric.wide_units(*w);
if col + units > target_col {
return byte;
}
col += units;
byte = w.end as usize;
}
(byte + (target_col - col) as usize).min(line_end)
}
fn line_index_for(&self, offset: usize) -> usize {
match self.line_starts.binary_search(&offset) {
Ok(idx) => idx,
Err(idx) => idx.saturating_sub(1),
}
}
#[cfg(test)]
fn assert_canonical(&self) {
assert_eq!(self.line_starts.first(), Some(&0), "missing leading zero");
assert!(
self.line_starts.windows(2).all(|w| w[0] < w[1]),
"line starts not strictly increasing: {:?}",
self.line_starts
);
assert!(
self.line_starts.iter().all(|&s| s <= self.len),
"line start past the end: {:?} len {}",
self.line_starts,
self.len
);
assert!(
self.wide_chars
.windows(2)
.all(|w| w[0].end <= w[1].start && w[0].start < w[0].end),
"wide chars overlap or are unsorted: {:?}",
self.wide_chars
);
assert!(
self.wide_chars
.iter()
.all(|w| (2..=4).contains(&w.len()) && w.end as usize <= self.len),
"wide char out of range: {:?} len {}",
self.wide_chars,
self.len
);
}
}
#[cfg(test)]
mod tests {
use super::*;
use PositionEncoding::{Utf8, Utf16};
#[test]
fn empty_string() {
let idx = LineIndex::new("");
assert_eq!(idx.byte_to_lc(0), LineCol { line: 1, column: 1 });
assert_eq!(idx.byte_to_position(0, Utf16), Position::new(0, 0));
assert_eq!(idx.byte_to_position(0, Utf8), Position::new(0, 0));
}
#[test]
fn single_line() {
let idx = LineIndex::new("abc");
assert_eq!(idx.byte_to_lc(0).column, 1);
assert_eq!(idx.byte_to_lc(2).column, 3);
assert_eq!(idx.byte_to_lc(3).column, 4);
assert_eq!(idx.byte_to_position(2, Utf16), Position::new(0, 2));
assert_eq!(idx.byte_to_position(2, Utf8), Position::new(0, 2));
}
#[test]
fn multi_line() {
let idx = LineIndex::new("ab\ncd\nef");
assert_eq!(idx.byte_to_lc(0), LineCol { line: 1, column: 1 });
assert_eq!(idx.byte_to_lc(2), LineCol { line: 1, column: 3 }); assert_eq!(idx.byte_to_lc(3), LineCol { line: 2, column: 1 });
assert_eq!(idx.byte_to_lc(6), LineCol { line: 3, column: 1 });
assert_eq!(idx.byte_to_position(6, Utf16), Position::new(2, 0));
assert_eq!(idx.byte_to_position(6, Utf8), Position::new(2, 0));
}
#[test]
fn utf8_multibyte() {
let idx = LineIndex::new("\u{00e1}b\nc");
assert_eq!(idx.byte_to_lc(2), LineCol { line: 1, column: 2 });
assert_eq!(idx.byte_to_position(2, Utf16), Position::new(0, 1));
assert_eq!(idx.byte_to_position(2, Utf8), Position::new(0, 2));
assert_eq!(idx.byte_to_lc(3), LineCol { line: 1, column: 3 });
assert_eq!(idx.byte_to_position(3, Utf16), Position::new(0, 2));
assert_eq!(idx.byte_to_position(3, Utf8), Position::new(0, 3));
}
#[test]
fn utf16_surrogate_pair() {
let idx = LineIndex::new("\u{1F600}x");
assert_eq!(idx.byte_to_lc(4), LineCol { line: 1, column: 2 });
assert_eq!(idx.byte_to_position(4, Utf16), Position::new(0, 2));
assert_eq!(idx.byte_to_position(4, Utf8), Position::new(0, 4));
assert_eq!(idx.byte_to_position(5, Utf16), Position::new(0, 3));
assert_eq!(idx.byte_to_position(5, Utf8), Position::new(0, 5));
}
#[test]
fn offset_past_end_clamps() {
let idx = LineIndex::new("abc");
assert_eq!(idx.byte_to_lc(100), LineCol { line: 1, column: 4 });
}
#[test]
fn trailing_newline() {
let idx = LineIndex::new("ab\n");
assert_eq!(idx.byte_to_lc(3), LineCol { line: 2, column: 1 });
}
#[test]
fn line_start_clamps_past_the_end() {
let idx = LineIndex::new("ab\ncd");
assert_eq!(idx.line_start(0), 0);
assert_eq!(idx.line_start(1), 3);
assert_eq!(idx.line_start(2), 5);
assert_eq!(idx.line_start(99), 5);
}
#[test]
fn wide_chars_on_late_lines() {
let text = "a\n\n\n\u{00e1}b\n\n\u{1F600}\u{00e1}c\nd";
let idx = LineIndex::new(text);
assert_eq!(idx.byte_to_position(6, Utf16), Position::new(3, 1));
assert_eq!(idx.byte_to_position(6, Utf8), Position::new(3, 2));
assert_eq!(idx.position_to_byte(Position::new(3, 1), Utf16), 6);
assert_eq!(idx.byte_to_position(15, Utf16), Position::new(5, 3));
assert_eq!(idx.byte_to_position(15, Utf8), Position::new(5, 6));
assert_eq!(idx.position_to_byte(Position::new(5, 3), Utf16), 15);
assert_eq!(idx.byte_to_position(17, Utf16), Position::new(6, 0));
assert_eq!(idx.byte_to_lc(17), LineCol { line: 7, column: 1 });
}
#[test]
fn position_to_byte_round_trips_both_encodings() {
let texts = [
"ab\ncde\nf\u{00e1}g\n\u{1F600}h",
"\u{00e1}\u{1F600}\u{00e1}\n\nx\u{1F600}y\u{00e1}z\n\u{00e1}\u{00e1}\n",
"\u{1F600}\u{1F600}\n\u{00e1}\u{1F600}",
];
for text in texts {
let idx = LineIndex::new(text);
for encoding in [Utf8, Utf16] {
for offset in 0..=text.len() {
if !text.is_char_boundary(offset) {
continue;
}
let pos = idx.byte_to_position(offset, encoding);
assert_eq!(
idx.position_to_byte(pos, encoding),
offset,
"text {text:?} offset {offset} encoding {encoding:?}"
);
}
}
}
}
#[test]
fn position_to_byte_handles_wide_chars_and_overshoot() {
let idx = LineIndex::new("\u{1F600}x\ny");
assert_eq!(idx.position_to_byte(Position::new(0, 0), Utf16), 0);
assert_eq!(idx.position_to_byte(Position::new(0, 2), Utf16), 4); assert_eq!(idx.position_to_byte(Position::new(1, 0), Utf16), 6); assert_eq!(idx.position_to_byte(Position::new(0, 1), Utf16), 0);
assert_eq!(idx.position_to_byte(Position::new(0, 99), Utf16), 6);
assert_eq!(idx.position_to_byte(Position::new(9, 0), Utf16), 7);
assert_eq!(idx.position_to_byte(Position::new(0, 4), Utf8), 4); assert_eq!(idx.position_to_byte(Position::new(0, 5), Utf8), 5); }
#[test]
fn to_kind_maps_to_lsp() {
assert_eq!(Utf8.to_kind(), PositionEncodingKind::UTF8);
assert_eq!(Utf16.to_kind(), PositionEncodingKind::UTF16);
}
#[track_caller]
fn assert_patch_matches_rebuild(base: &str, range: std::ops::Range<usize>, insert: &str) {
let mut spliced = base.to_string();
spliced.replace_range(range.clone(), insert);
let mut patched = LineIndex::new(base);
patched.apply_edit(range.clone(), insert);
patched.assert_canonical();
assert_eq!(
patched,
LineIndex::new(&spliced),
"base {base:?} range {range:?} insert {insert:?}"
);
}
const BASES: [&str; 9] = [
"",
"a",
"\n",
"\n\n\n",
"ab\ncd\nef",
"ab\ncd\nef\n",
"\u{00e1}b\nc\u{1F600}\nd",
"a\r\nb\r\n",
"\u{1F600}\n\u{1F600}",
];
const INSERTS: [&str; 11] = [
"",
"x",
"\n",
"\nx",
"x\n",
"\n\n",
"xy",
"\u{00e1}",
"\u{1F600}",
"a\u{00e1}\nb\u{1F600}\n",
"\r\n",
];
#[test]
fn apply_edit_matches_rebuild_exhaustively() {
for base in BASES {
let bounds: Vec<usize> = (0..=base.len())
.filter(|&o| base.is_char_boundary(o))
.collect();
for (i, &start) in bounds.iter().enumerate() {
for &end in &bounds[i..] {
for insert in INSERTS {
assert_patch_matches_rebuild(base, start..end, insert);
}
}
}
}
}
#[test]
fn apply_edit_sequences_match_rebuild() {
type Recipe = fn(&str) -> (std::ops::Range<usize>, &'static str);
let recipes: [Recipe; 6] = [
|_| (0..0, "\n"),
|t| (t.len()..t.len(), "x"),
|t| (0..t.find('\n').map_or(0, |i| i + 1), ""),
|t| {
let mut at = t.len();
while at > 0 && !t.is_char_boundary(at - 1) {
at -= 1;
}
(at.saturating_sub(1)..t.len(), "")
},
|t| {
let mut at = t.len() / 2;
while !t.is_char_boundary(at) {
at -= 1;
}
(at..at, "\u{00e1}")
},
|t| {
let mut at = t.len() / 2;
while !t.is_char_boundary(at) {
at -= 1;
}
(at..at, "p\nq\n")
},
];
for base in ["ab\ncd\nef\n", "\u{1F600}x\n\u{00e1}", ""] {
for a in recipes {
for b in recipes {
for c in recipes {
let mut text = base.to_string();
let mut index = LineIndex::new(&text);
for step in [a, b, c] {
let (range, insert) = step(&text);
index.apply_edit(range.clone(), insert);
text.replace_range(range, insert);
index.assert_canonical();
assert_eq!(index, LineIndex::new(&text), "after {text:?}");
}
}
}
}
}
}
#[test]
fn apply_edit_no_op_leaves_the_index_untouched() {
let before = LineIndex::new("ab\ncd\n");
let mut after = before.clone();
after.apply_edit(3..3, "");
assert_eq!(before, after);
}
#[test]
fn apply_edit_pure_insert_and_pure_delete() {
assert_patch_matches_rebuild("ab\ncd", 2..2, "XY");
assert_patch_matches_rebuild("ab\ncd", 1..3, "");
}
#[test]
fn apply_edit_at_the_buffer_end() {
assert_patch_matches_rebuild("ab\ncd", 5..5, "e");
assert_patch_matches_rebuild("ab\n", 3..3, "c");
}
#[test]
fn apply_edit_deleting_the_trailing_newline() {
assert_patch_matches_rebuild("ab\ncd\n", 5..6, "");
}
#[test]
fn apply_edit_inserting_a_newline_at_offset_zero() {
assert_patch_matches_rebuild("ab\ncd", 0..0, "\n");
}
#[test]
fn apply_edit_deleting_from_zero_across_a_newline() {
assert_patch_matches_rebuild("ab\ncd\nef", 0..4, "");
}
#[test]
fn apply_edit_spanning_several_newlines() {
assert_patch_matches_rebuild("a\nb\nc\nd\ne", 1..7, "Z");
}
#[test]
fn apply_edit_multi_line_paste() {
assert_patch_matches_rebuild("ab\ncd", 2..2, "1\n2\n3\n4");
}
#[test]
fn apply_edit_swapping_wide_chars_and_ascii() {
assert_patch_matches_rebuild("a\u{1F600}b\n\u{00e1}c", 1..5, "z");
assert_patch_matches_rebuild("azb\n\u{00e1}c", 1..2, "\u{1F600}");
}
#[test]
fn apply_edit_keeps_crlf_line_starts() {
assert_patch_matches_rebuild("a\r\nb", 1..1, "\r\n");
}
}