use std::{collections::BTreeMap, convert::Infallible, ops::Range};
use crate::{
fmt::encoding::{BytePreservingEncoder, Encoder, LossyEncoder, Replacement},
latin_1::Latin1Str,
syntax::node::SyntaxNode,
text::{char_encoding::CharEncoding, char_iter::CharIter},
tokens::TriviaPiece,
};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct SourceLoc {
pub line: usize,
pub col: usize,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub struct EncodedOffset(usize);
impl EncodedOffset {
pub(crate) fn new(inner: usize) -> EncodedOffset {
EncodedOffset(inner)
}
pub fn raw(&self) -> usize {
self.0
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub struct EncodedSpan {
start: usize,
end: usize,
}
impl EncodedSpan {
pub(crate) fn new(start: usize, end: usize) -> EncodedSpan {
EncodedSpan { start, end }
}
pub fn start(&self) -> EncodedOffset {
EncodedOffset::new(self.start)
}
pub fn end(&self) -> EncodedOffset {
EncodedOffset::new(self.end)
}
}
#[derive(Debug)]
struct WideChar {
offset: usize,
byte_len: usize,
target_len: usize,
}
impl WideChar {
fn end(&self) -> usize {
self.offset + self.byte_len
}
}
pub struct SourceLocConverter {
line_starts: Vec<usize>,
text_len: usize,
wide_char_lines: BTreeMap<usize, Vec<WideChar>>,
}
impl SourceLocConverter {
fn build<C: CharEncoding, Err>(
root: &SyntaxNode,
mut handle_piece: impl FnMut(
&TriviaPiece,
usize,
&mut Vec<usize>,
&mut BTreeMap<usize, Vec<WideChar>>,
) -> Result<(), Err>,
) -> Result<SourceLocConverter, Err> {
let mut line_starts = vec![0usize];
let mut wide_char_lines = BTreeMap::new();
let mut cursor = 0usize;
let mut tok = root.first_token();
while let Some(t) = tok {
for piece in t.leading_trivia() {
handle_piece(piece, cursor, &mut line_starts, &mut wide_char_lines)?;
cursor += piece.byte_len();
}
record_text::<C>(t.text(), cursor, &mut line_starts, &mut wide_char_lines);
cursor += t.text().len();
tok = t.next_token();
}
Ok(SourceLocConverter {
line_starts,
text_len: cursor,
wide_char_lines,
})
}
pub fn new<E: BytePreservingEncoder, C: CharEncoding>(
root: &SyntaxNode,
) -> Result<SourceLocConverter, E::Err>
where
for<'a> E::Str<'a>: CharIter,
{
Self::build::<C, _>(root, |piece, cursor, line_starts, wide_char_lines| {
record_piece::<E, C>(piece, cursor, line_starts, wide_char_lines)
})
}
pub fn new_lossy<E: LossyEncoder, C: CharEncoding>(root: &SyntaxNode) -> SourceLocConverter
where
for<'a> E::Str<'a>: CharIter,
{
Self::build::<C, Infallible>(root, |piece, cursor, line_starts, wide_char_lines| {
record_piece_lossy::<E, C>(piece, cursor, line_starts, wide_char_lines);
Ok(())
})
.unwrap()
}
pub fn source_loc(&self, offset: usize) -> SourceLoc {
let offset = offset.min(self.text_len);
let i = match self.line_starts.binary_search(&offset) {
Ok(i) => i,
Err(i) => i - 1,
};
let line_start = self.line_starts[i];
let byte_col = offset - line_start;
let mut target_col: usize = 0;
let mut prev_end: usize = 0;
if let Some(wides) = self.wide_char_lines.get(&i) {
for wide in wides {
if wide.offset >= byte_col {
break;
}
target_col += wide.offset - prev_end;
if wide.end() <= byte_col {
target_col += wide.target_len;
prev_end = wide.end();
} else {
return SourceLoc {
line: i,
col: target_col,
};
}
}
}
target_col += byte_col - prev_end;
SourceLoc {
line: i,
col: target_col,
}
}
pub fn convert_byte_offset(&self, offset: usize) -> EncodedOffset {
let mut delta = 0usize;
for (line, wide_chars) in &self.wide_char_lines {
let line_start = self.line_starts[*line];
for wide in wide_chars {
if wide.offset + line_start >= offset {
break;
}
delta += wide.target_len - wide.byte_len;
}
}
EncodedOffset::new(offset + delta)
}
pub fn convert_byte_span(&self, span: &Range<usize>) -> EncodedSpan {
EncodedSpan::new(
self.convert_byte_offset(span.start).raw(),
self.convert_byte_offset(span.end).raw(),
)
}
pub fn line_start(&self, index: usize) -> EncodedOffset {
EncodedOffset::new(self.line_starts[index])
}
}
fn record_text<C: CharEncoding>(
text: &Latin1Str,
cursor: usize,
line_starts: &mut Vec<usize>,
wide_char_lines: &mut BTreeMap<usize, Vec<WideChar>>,
) {
record_str::<C>(text, &[], line_starts, wide_char_lines, cursor);
}
fn record_piece<E: Encoder, C: CharEncoding>(
piece: &TriviaPiece,
cursor: usize,
line_starts: &mut Vec<usize>,
wide_char_lines: &mut BTreeMap<usize, Vec<WideChar>>,
) -> Result<(), E::Err>
where
for<'a> E::Str<'a>: CharIter,
{
match piece {
TriviaPiece::BlockComment(c) | TriviaPiece::LineComment(c) => {
record_str::<C>(c.encode::<E>()?, &[], line_starts, wide_char_lines, cursor);
}
other => {
record_non_comment_trivia::<C>(other, cursor, line_starts, wide_char_lines);
}
}
Ok(())
}
fn record_piece_lossy<E: LossyEncoder, C: CharEncoding>(
piece: &TriviaPiece,
cursor: usize,
line_starts: &mut Vec<usize>,
wide_char_lines: &mut BTreeMap<usize, Vec<WideChar>>,
) where
for<'a> E::Str<'a>: CharIter,
{
match piece {
TriviaPiece::BlockComment(c) | TriviaPiece::LineComment(c) => {
let (encoded, replacements) = c.encode_lossy::<E>();
record_str::<C>(encoded, &replacements, line_starts, wide_char_lines, cursor);
}
other => {
record_non_comment_trivia::<C>(other, cursor, line_starts, wide_char_lines);
}
}
}
fn record_non_comment_trivia<C: CharEncoding>(
piece: &TriviaPiece,
cursor: usize,
line_starts: &mut Vec<usize>,
wide_char_lines: &mut BTreeMap<usize, Vec<WideChar>>,
) {
match piece {
TriviaPiece::LineFeeds(n)
| TriviaPiece::CarriageReturns(n)
| TriviaPiece::FormFeeds(n)
| TriviaPiece::VerticalTabs(n) => {
for i in 0..*n {
line_starts.push(cursor + i + 1);
}
}
TriviaPiece::CarriageReturnLineFeeds(n) => {
for i in 0..*n {
line_starts.push(cursor + (i + 1) * 2);
}
}
TriviaPiece::BlockComment(_) | TriviaPiece::LineComment(_) => {
unreachable!("Handled by specific branch");
}
TriviaPiece::NonBreakingSpaces(n) => {
let target_len = C::char_len('\u{00A0}');
if target_len != 1 {
let line = line_starts.len() - 1;
let line_start = line_starts[line];
for i in 0..*n {
let abs_offset = cursor + i;
wide_char_lines.entry(line).or_default().push(WideChar {
offset: abs_offset - line_start,
byte_len: 1,
target_len,
});
}
}
}
_ => {}
}
}
fn record_str<C: CharEncoding>(
encoded: impl CharIter,
replacements: &[Replacement],
line_starts: &mut Vec<usize>,
wide_char_lines: &mut BTreeMap<usize, Vec<WideChar>>,
cursor: usize,
) {
let mut line = line_starts.len() - 1;
let mut itr = encoded.iter_chars_indices().peekable();
let byte_len = encoded.byte_count();
let mut repl_idx = 0;
let mut delta: usize = 0;
while let Some((enc_pos, ch)) = itr.next() {
let source_pos = enc_pos - delta;
if ch == '\n' {
line_starts.push(cursor + source_pos + 1);
line += 1;
continue;
}
if ch == '\r' {
if itr.peek().is_some_and(|(_, ch)| *ch == '\n') {
let _ = itr.next();
line_starts.push(cursor + source_pos + 2);
} else {
line_starts.push(cursor + source_pos + 1);
}
line += 1;
continue;
}
let enc_next = itr.peek().map(|(p, _)| *p).unwrap_or(byte_len);
let enc_byte_width = enc_next - enc_pos;
let target_width = C::char_len(ch);
let at_replacement =
repl_idx < replacements.len() && source_pos == replacements[repl_idx].source_offset;
if at_replacement {
let r = &replacements[repl_idx];
let source_byte_width = r.source_bytes;
if source_byte_width != target_width {
let abs_offset = cursor + source_pos;
let line_start = line_starts[line];
wide_char_lines.entry(line).or_default().push(WideChar {
offset: abs_offset - line_start,
byte_len: source_byte_width,
target_len: target_width,
});
}
delta += r.target_bytes - r.source_bytes;
repl_idx += 1;
} else {
if enc_byte_width != 1 || enc_byte_width != target_width {
let abs_offset = cursor + source_pos;
let line_start = line_starts[line];
wide_char_lines.entry(line).or_default().push(WideChar {
offset: abs_offset - line_start,
byte_len: enc_byte_width,
target_len: target_width,
});
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::fmt::encoding::{Latin1Encoder, LossyUtf8Encoder, Utf8Encoder};
use crate::parser::parse;
use crate::text::char_encoding::{Utf16, Utf32, Utf8};
use crate::tokens::TokenStream;
fn converter<E: BytePreservingEncoder, C: CharEncoding>(
input: impl Into<TokenStream>,
) -> SourceLocConverter
where
for<'a> E::Str<'a>: CharIter,
E::Err: std::fmt::Debug,
{
let (design_file, _) = parse(input);
SourceLocConverter::new::<E, C>(&design_file.0).unwrap()
}
fn lossy_converter<E: LossyEncoder, C: CharEncoding>(
input: impl Into<TokenStream>,
) -> SourceLocConverter
where
for<'a> E::Str<'a>: CharIter,
{
let (design_file, _) = parse(input);
SourceLocConverter::new_lossy::<E, C>(&design_file.0)
}
fn utf8_utf16(input: &str) -> SourceLocConverter {
converter::<Utf8Encoder, Utf16>(input)
}
fn loc(conv: &SourceLocConverter, offset: usize) -> (usize, usize) {
let l = conv.source_loc(offset);
(l.line, l.col)
}
#[test]
fn single_line_ascii() {
let conv = utf8_utf16("entity e is end;");
assert_eq!(loc(&conv, 0), (0, 0));
assert_eq!(loc(&conv, 7), (0, 7));
}
#[test]
fn multi_line_ascii() {
let input = "entity e is\nend;\n";
let conv = utf8_utf16(input);
assert_eq!(loc(&conv, 0), (0, 0));
assert_eq!(loc(&conv, 11), (0, 11));
assert_eq!(loc(&conv, 12), (1, 0));
assert_eq!(loc(&conv, 15), (1, 3));
}
#[test]
fn crlf_line_endings() {
let input = "entity e is\r\nend;\r\n";
let conv = utf8_utf16(input);
assert_eq!(loc(&conv, 13), (1, 0));
assert_eq!(loc(&conv, 16), (1, 3));
}
#[test]
fn line_comment_with_umlaut() {
let input = "-- ä\nentity e is end;";
let conv = utf8_utf16(input);
assert_eq!(loc(&conv, 2), (0, 2));
assert_eq!(loc(&conv, 3), (0, 3));
assert_eq!(loc(&conv, 4), (0, 3));
assert_eq!(loc(&conv, 6), (1, 0));
}
#[test]
fn line_comment_with_emoji() {
let input = "-- 💣\nentity e is end;";
let conv = utf8_utf16(input);
assert_eq!(loc(&conv, 8), (1, 0));
assert_eq!(loc(&conv, 3), (0, 3));
assert_eq!(loc(&conv, 5), (0, 3));
}
#[test]
fn block_comment_multiline() {
let input = "/* line1\nline2 */\nentity e is end;";
let conv = utf8_utf16(input);
assert_eq!(loc(&conv, 9), (1, 0));
assert_eq!(loc(&conv, 18), (2, 0));
}
#[test]
fn block_comment_multiline_carriage_return() {
let input = "/* line1\rline2 */\rentity e is end;";
let conv = utf8_utf16(input);
assert_eq!(loc(&conv, 9), (1, 0));
assert_eq!(loc(&conv, 18), (2, 0));
}
#[test]
fn block_comment_multiline_carriage_return_line_feed() {
let input = "/* line1\r\nline2 */\r\nentity e is end;";
let conv = utf8_utf16(input);
assert_eq!(loc(&conv, 10), (1, 0));
assert_eq!(loc(&conv, 20), (2, 0));
}
#[test]
fn offset_beyond_end_is_clamped() {
let input = "entity e is end;";
let conv = utf8_utf16(input);
let clamped = conv.source_loc(9999);
let end = conv.source_loc(input.len());
assert_eq!(clamped.line, end.line);
assert_eq!(clamped.col, end.col);
}
#[test]
fn offset_zero() {
let input = "entity e is end;";
let conv = utf8_utf16(input);
assert_eq!(loc(&conv, 0), (0, 0));
}
#[test]
fn utf32_target_counts_code_points() {
let input = "-- 💣x\nentity e is end;";
let conv = converter::<Utf8Encoder, Utf32>(input);
assert_eq!(loc(&conv, 3), (0, 3));
assert_eq!(loc(&conv, 7), (0, 4));
}
#[test]
fn utf8_target_matches_byte_offset() {
let input = "-- ä💣\nentity e is end;";
let conv = converter::<Utf8Encoder, Utf8>(input);
assert_eq!(loc(&conv, 3), (0, 3)); assert_eq!(loc(&conv, 5), (0, 5)); assert_eq!(loc(&conv, 9), (0, 9)); }
#[test]
fn latin1_source_utf16_target() {
let input = "-- hello\nentity e is end;";
let conv = converter::<Latin1Encoder, Utf16>(input);
assert_eq!(loc(&conv, 0), (0, 0));
assert_eq!(loc(&conv, 9), (1, 0));
}
#[test]
fn latin1_source_utf8_target_codepoint() {
let input: &[u8] = b"--\xE4\nentity e is end;";
let conv = converter::<Latin1Encoder, Utf8>(input);
assert_eq!(conv.convert_byte_offset(0).raw(), 0);
assert_eq!(conv.convert_byte_offset(2).raw(), 2); assert_eq!(conv.convert_byte_offset(3).raw(), 4); assert_eq!(conv.convert_byte_offset(4).raw(), 5); }
#[test]
fn latin1_source_utf8_target_codepoint_twice() {
let input: &[u8] = b"--\xE4\xE4\nentity e is end;";
let conv = converter::<Latin1Encoder, Utf8>(input);
assert_eq!(conv.convert_byte_offset(0).raw(), 0);
assert_eq!(conv.convert_byte_offset(2).raw(), 2); assert_eq!(conv.convert_byte_offset(3).raw(), 4); assert_eq!(conv.convert_byte_offset(4).raw(), 6); assert_eq!(conv.convert_byte_offset(5).raw(), 7); }
#[test]
fn lossy_utf8_to_utf8() {
let input: &[u8] = b"/*\xE4\xE4*/entity e is end;";
let conv = lossy_converter::<LossyUtf8Encoder, Utf8>(input);
assert_eq!(conv.convert_byte_offset(0).raw(), 0); assert_eq!(conv.convert_byte_offset(1).raw(), 1); assert_eq!(conv.convert_byte_offset(2).raw(), 2); assert_eq!(conv.convert_byte_offset(3).raw(), 5); assert_eq!(conv.convert_byte_offset(4).raw(), 8); assert_eq!(conv.convert_byte_offset(5).raw(), 9); assert_eq!(conv.convert_byte_offset(6).raw(), 10); }
#[test]
fn multiple_wide_chars_same_line() {
let input = "-- äö\nentity e is end;";
let conv = utf8_utf16(input);
assert_eq!(loc(&conv, 3), (0, 3));
assert_eq!(loc(&conv, 5), (0, 4));
assert_eq!(loc(&conv, 8), (1, 0));
}
#[test]
fn empty_input() {
let conv = utf8_utf16("");
assert_eq!(loc(&conv, 0), (0, 0));
}
#[test]
fn non_breaking_space_latin1_to_utf8() {
let input: &[u8] = b"\xA0entity e is end;";
let conv = converter::<Latin1Encoder, Utf8>(input);
assert_eq!(conv.convert_byte_offset(0).raw(), 0); assert_eq!(conv.convert_byte_offset(1).raw(), 2); }
#[test]
fn non_breaking_space_latin1_to_utf16() {
let input: &[u8] = b"\xA0entity e is end;";
let conv = converter::<Latin1Encoder, Utf16>(input);
assert_eq!(loc(&conv, 0), (0, 0));
assert_eq!(loc(&conv, 1), (0, 1));
}
#[test]
fn multiple_non_breaking_spaces_latin1_to_utf8() {
let input: &[u8] = b"\xA0\xA0entity e is end;";
let conv = converter::<Latin1Encoder, Utf8>(input);
assert_eq!(conv.convert_byte_offset(0).raw(), 0); assert_eq!(conv.convert_byte_offset(1).raw(), 2); assert_eq!(conv.convert_byte_offset(2).raw(), 4); }
}