use std::{fmt, path::Path, sync::Arc};
#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
#[repr(transparent)]
pub struct SourceId(u32);
impl SourceId {
#[must_use]
pub const fn new(value: u32) -> Self {
Self(value)
}
#[must_use]
pub const fn get(self) -> u32 {
self.0
}
}
impl From<u32> for SourceId {
fn from(value: u32) -> Self {
Self::new(value)
}
}
#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
pub struct SourceIdentity {
source_id: SourceId,
path: Arc<Path>,
}
impl SourceIdentity {
pub(crate) fn new(source_id: SourceId, path: Arc<Path>) -> Self {
Self { source_id, path }
}
#[must_use]
pub const fn source_id(&self) -> SourceId {
self.source_id
}
#[must_use]
pub fn path(&self) -> &Path {
&self.path
}
}
#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
#[repr(transparent)]
pub struct Utf16Pos(usize);
impl Utf16Pos {
pub const ZERO: Self = Self(0);
#[must_use]
pub const fn new(offset: usize) -> Self {
Self(offset)
}
#[must_use]
pub const fn get(self) -> usize {
self.0
}
}
impl From<usize> for Utf16Pos {
fn from(offset: usize) -> Self {
Self::new(offset)
}
}
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub struct TextRange {
start: Utf16Pos,
end: Utf16Pos,
}
impl TextRange {
pub const fn new(start: Utf16Pos, end: Utf16Pos) -> Result<Self, SourcePositionError> {
if start.get() > end.get() {
return Err(SourcePositionError::RangeStartAfterEnd { start, end });
}
Ok(Self { start, end })
}
#[must_use]
pub const fn start(self) -> Utf16Pos {
self.start
}
#[must_use]
pub const fn end(self) -> Utf16Pos {
self.end
}
#[must_use]
pub const fn is_empty(self) -> bool {
self.start.get() == self.end.get()
}
#[must_use]
pub const fn len(self) -> usize {
self.end.get() - self.start.get()
}
}
#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
pub enum ScriptKind {
JavaScript,
JavaScriptReact,
TypeScript,
TypeScriptReact,
Json,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum SourcePositionError {
ByteOffsetOutOfBounds { offset: usize, len: usize },
ByteOffsetInsideCodePoint { offset: usize },
Utf16PositionOutOfBounds { position: Utf16Pos, len: Utf16Pos },
Utf16PositionInsideSurrogatePair { position: Utf16Pos },
RangeStartAfterEnd { start: Utf16Pos, end: Utf16Pos },
}
impl fmt::Display for SourcePositionError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match *self {
Self::ByteOffsetOutOfBounds { offset, len } => {
write!(
formatter,
"byte offset {offset} exceeds source length {len}"
)
}
Self::ByteOffsetInsideCodePoint { offset } => {
write!(formatter, "byte offset {offset} splits a UTF-8 code point")
}
Self::Utf16PositionOutOfBounds { position, len } => write!(
formatter,
"UTF-16 position {} exceeds source length {}",
position.get(),
len.get()
),
Self::Utf16PositionInsideSurrogatePair { position } => write!(
formatter,
"UTF-16 position {} splits a surrogate pair",
position.get()
),
Self::RangeStartAfterEnd { start, end } => write!(
formatter,
"range start {} follows range end {}",
start.get(),
end.get()
),
}
}
}
impl std::error::Error for SourcePositionError {}
#[derive(Clone, Copy, Debug)]
struct BoundaryCheckpoint {
byte: usize,
utf16: Utf16Pos,
}
#[derive(Clone, Debug)]
pub struct SourceText {
text: Arc<str>,
checkpoints: Arc<[BoundaryCheckpoint]>,
line_starts: Arc<[Utf16Pos]>,
utf16_len: Utf16Pos,
}
impl SourceText {
#[must_use]
pub fn new(text: impl Into<Arc<str>>) -> Self {
Self::from_arc(text.into())
}
#[must_use]
pub fn from_arc(text: Arc<str>) -> Self {
let mut checkpoints = vec![BoundaryCheckpoint {
byte: 0,
utf16: Utf16Pos::ZERO,
}];
let mut line_starts = vec![Utf16Pos::ZERO];
let mut utf16_offset = 0;
let mut characters = text.char_indices().peekable();
while let Some((byte_start, character)) = characters.next() {
utf16_offset += character.len_utf16();
if !character.is_ascii() {
checkpoints.push(BoundaryCheckpoint {
byte: byte_start + character.len_utf8(),
utf16: Utf16Pos::new(utf16_offset),
});
}
let ends_line = character == '\n'
|| (character == '\r' && !matches!(characters.peek(), Some(&(_, '\n'))))
|| character == '\u{2028}'
|| character == '\u{2029}';
if ends_line {
line_starts.push(Utf16Pos::new(utf16_offset));
}
}
Self {
text,
checkpoints: Arc::from(checkpoints),
line_starts: Arc::from(line_starts),
utf16_len: Utf16Pos::new(utf16_offset),
}
}
#[must_use]
pub fn as_str(&self) -> &str {
self.text.as_ref()
}
#[must_use]
pub const fn len_utf16(&self) -> Utf16Pos {
self.utf16_len
}
#[must_use]
pub fn is_empty(&self) -> bool {
self.text.is_empty()
}
pub fn byte_to_utf16(&self, byte_offset: usize) -> Result<Utf16Pos, SourcePositionError> {
if byte_offset > self.text.len() {
return Err(SourcePositionError::ByteOffsetOutOfBounds {
offset: byte_offset,
len: self.text.len(),
});
}
if !self.text.is_char_boundary(byte_offset) {
return Err(SourcePositionError::ByteOffsetInsideCodePoint {
offset: byte_offset,
});
}
let checkpoint_index = self
.checkpoints
.partition_point(|checkpoint| checkpoint.byte <= byte_offset)
.saturating_sub(1);
let checkpoint = &self.checkpoints[checkpoint_index];
Ok(Utf16Pos::new(
checkpoint.utf16.get() + (byte_offset - checkpoint.byte),
))
}
pub fn utf16_to_byte(&self, position: Utf16Pos) -> Result<usize, SourcePositionError> {
if position > self.utf16_len {
return Err(SourcePositionError::Utf16PositionOutOfBounds {
position,
len: self.utf16_len,
});
}
let checkpoint_index = self
.checkpoints
.partition_point(|checkpoint| checkpoint.utf16 <= position)
.saturating_sub(1);
let checkpoint = &self.checkpoints[checkpoint_index];
let byte_offset = checkpoint.byte + (position.get() - checkpoint.utf16.get());
if !self.text.is_char_boundary(byte_offset) {
return Err(SourcePositionError::Utf16PositionInsideSurrogatePair { position });
}
Ok(byte_offset)
}
pub fn range(&self, start: Utf16Pos, end: Utf16Pos) -> Result<TextRange, SourcePositionError> {
self.utf16_to_byte(start)?;
self.utf16_to_byte(end)?;
TextRange::new(start, end)
}
pub fn line_column(&self, position: Utf16Pos) -> Result<(usize, usize), SourcePositionError> {
self.utf16_to_byte(position)?;
let line_index = self
.line_starts
.partition_point(|line_start| *line_start <= position)
.saturating_sub(1);
let line_start = self.line_starts[line_index];
Ok((line_index, position.get() - line_start.get()))
}
}
#[cfg(test)]
mod tests {
use super::{SourcePositionError, SourceText, TextRange, Utf16Pos};
#[test]
fn ascii_boundaries_round_trip() {
let source = SourceText::new("hello");
assert_eq!(source.len_utf16(), Utf16Pos::new(5));
for offset in 0..=5 {
assert_eq!(source.byte_to_utf16(offset), Ok(Utf16Pos::new(offset)));
assert_eq!(source.utf16_to_byte(Utf16Pos::new(offset)), Ok(offset));
}
assert_eq!(source.line_column(Utf16Pos::new(5)), Ok((0, 5)));
}
#[test]
fn bmp_code_points_preserve_utf16_width_but_not_byte_width() {
let source = SourceText::new("aéä¸");
assert_eq!(source.byte_to_utf16(0), Ok(Utf16Pos::new(0)));
assert_eq!(source.byte_to_utf16(1), Ok(Utf16Pos::new(1)));
assert_eq!(source.byte_to_utf16(3), Ok(Utf16Pos::new(2)));
assert_eq!(source.byte_to_utf16(6), Ok(Utf16Pos::new(3)));
assert_eq!(source.utf16_to_byte(Utf16Pos::new(2)), Ok(3));
assert_eq!(
source.byte_to_utf16(2),
Err(SourcePositionError::ByteOffsetInsideCodePoint { offset: 2 })
);
}
#[test]
fn astral_code_points_use_two_utf16_units() {
let source = SourceText::new("a😀b");
assert_eq!(source.byte_to_utf16(1), Ok(Utf16Pos::new(1)));
assert_eq!(source.byte_to_utf16(5), Ok(Utf16Pos::new(3)));
assert_eq!(source.byte_to_utf16(6), Ok(Utf16Pos::new(4)));
assert_eq!(source.utf16_to_byte(Utf16Pos::new(1)), Ok(1));
assert_eq!(source.utf16_to_byte(Utf16Pos::new(3)), Ok(5));
assert_eq!(source.utf16_to_byte(Utf16Pos::new(4)), Ok(6));
assert_eq!(
source.utf16_to_byte(Utf16Pos::new(2)),
Err(SourcePositionError::Utf16PositionInsideSurrogatePair {
position: Utf16Pos::new(2),
})
);
}
#[test]
fn combining_marks_each_advance_the_utf16_column() {
let source = SourceText::new("e\u{301}x");
assert_eq!(source.byte_to_utf16(1), Ok(Utf16Pos::new(1)));
assert_eq!(source.byte_to_utf16(3), Ok(Utf16Pos::new(2)));
assert_eq!(source.byte_to_utf16(4), Ok(Utf16Pos::new(3)));
assert_eq!(source.line_column(Utf16Pos::new(2)), Ok((0, 2)));
}
#[test]
fn crlf_is_one_line_break_and_columns_are_utf16_units() {
let source = SourceText::new("a\r\n😀\nb");
assert_eq!(source.len_utf16(), Utf16Pos::new(7));
assert_eq!(source.line_column(Utf16Pos::new(0)), Ok((0, 0)));
assert_eq!(source.line_column(Utf16Pos::new(2)), Ok((0, 2)));
assert_eq!(source.line_column(Utf16Pos::new(3)), Ok((1, 0)));
assert_eq!(source.line_column(Utf16Pos::new(5)), Ok((1, 2)));
assert_eq!(source.line_column(Utf16Pos::new(6)), Ok((2, 0)));
assert_eq!(source.line_column(Utf16Pos::new(7)), Ok((2, 1)));
}
#[test]
fn unicode_line_and_paragraph_separators_advance_line_and_column() {
let source = SourceText::new("a\u{2028}b\u{2029}c");
assert_eq!(source.len_utf16(), Utf16Pos::new(5));
assert_eq!(source.line_column(Utf16Pos::new(0)), Ok((0, 0)));
assert_eq!(source.line_column(Utf16Pos::new(1)), Ok((0, 1)));
assert_eq!(source.line_column(Utf16Pos::new(2)), Ok((1, 0)));
assert_eq!(source.line_column(Utf16Pos::new(3)), Ok((1, 1)));
assert_eq!(source.line_column(Utf16Pos::new(4)), Ok((2, 0)));
assert_eq!(source.line_column(Utf16Pos::new(5)), Ok((2, 1)));
}
#[test]
fn empty_and_end_positions_are_valid_boundaries() {
let empty = SourceText::new("");
assert!(empty.is_empty());
assert_eq!(empty.byte_to_utf16(0), Ok(Utf16Pos::ZERO));
assert_eq!(empty.utf16_to_byte(Utf16Pos::ZERO), Ok(0));
assert_eq!(empty.line_column(Utf16Pos::ZERO), Ok((0, 0)));
let source = SourceText::new("😀");
assert_eq!(source.byte_to_utf16(4), Ok(Utf16Pos::new(2)));
assert_eq!(source.utf16_to_byte(Utf16Pos::new(2)), Ok(4));
}
#[test]
fn invalid_byte_and_utf16_offsets_have_distinct_errors() {
let source = SourceText::new("😀");
assert_eq!(
source.byte_to_utf16(1),
Err(SourcePositionError::ByteOffsetInsideCodePoint { offset: 1 })
);
assert_eq!(
source.byte_to_utf16(5),
Err(SourcePositionError::ByteOffsetOutOfBounds { offset: 5, len: 4 })
);
assert_eq!(
source.utf16_to_byte(Utf16Pos::new(1)),
Err(SourcePositionError::Utf16PositionInsideSurrogatePair {
position: Utf16Pos::new(1),
})
);
assert_eq!(
source.utf16_to_byte(Utf16Pos::new(3)),
Err(SourcePositionError::Utf16PositionOutOfBounds {
position: Utf16Pos::new(3),
len: Utf16Pos::new(2),
})
);
}
#[test]
fn ranges_cannot_be_reversed_and_source_ranges_validate_boundaries() {
assert_eq!(
TextRange::new(Utf16Pos::new(3), Utf16Pos::new(1)),
Err(SourcePositionError::RangeStartAfterEnd {
start: Utf16Pos::new(3),
end: Utf16Pos::new(1),
})
);
let source = SourceText::new("a😀b");
let range = source
.range(Utf16Pos::new(1), Utf16Pos::new(3))
.expect("the emoji's outer boundaries are valid");
assert_eq!(range.start(), Utf16Pos::new(1));
assert_eq!(range.end(), Utf16Pos::new(3));
assert_eq!(range.len(), 2);
assert_eq!(
source.range(Utf16Pos::new(3), Utf16Pos::new(1)),
Err(SourcePositionError::RangeStartAfterEnd {
start: Utf16Pos::new(3),
end: Utf16Pos::new(1),
})
);
assert_eq!(
source.range(Utf16Pos::new(1), Utf16Pos::new(2)),
Err(SourcePositionError::Utf16PositionInsideSurrogatePair {
position: Utf16Pos::new(2),
})
);
}
}