textlane_core/models/
lines.rs

1use crate::models::position::{Point, Position, UnistPoint, UnistPosition};
2
3/// The line starts of a source, to turn byte points into unist's UTF-16 lines and columns.
4#[derive(Clone, Debug)]
5pub struct Lines<'a> {
6    source: &'a str,
7    starts: Vec<usize>,
8    ascii: bool,
9    // UTF-16 units before every `BLOCK`th byte, so a point never counts from the start
10    checkpoints: Vec<usize>,
11}
12
13const BLOCK: usize = 256;
14
15impl<'a> Lines<'a> {
16    pub fn new(source: &'a str) -> Self {
17        let bytes = source.as_bytes();
18        let mut starts = vec![0];
19        let mut index = 0;
20        while index < bytes.len() {
21            match bytes[index] {
22                b'\n' => starts.push(index + 1),
23                b'\r' if bytes.get(index + 1) != Some(&b'\n') => starts.push(index + 1),
24                _ => {}
25            }
26            index += 1;
27        }
28        let ascii = source.is_ascii();
29        let checkpoints = match ascii {
30            true => vec![],
31            false => {
32                let mut total = 0;
33                let mut checkpoints = Vec::with_capacity(bytes.len() / BLOCK + 1);
34                for block in bytes.chunks(BLOCK) {
35                    checkpoints.push(total);
36                    total += count_byte_units(block);
37                }
38                checkpoints
39            }
40        };
41        Self {
42            source,
43            starts,
44            ascii,
45            checkpoints,
46        }
47    }
48
49    /// The unist point of a byte point.
50    pub fn to_unist_point(&self, point: Point) -> UnistPoint {
51        let offset = point.offset().min(self.source.len());
52        let line = self.starts.partition_point(|start| *start <= offset);
53        let start = self.starts[line - 1];
54        // Inside a tab, however far into its columns, micromark counts the tab as passed by
55        // one column; past the end, as parse5 puts a token ended by EOF, counts on the last line
56        let extra = usize::from(point.virtual_spaces() > 0) + (point.offset() - offset);
57        UnistPoint {
58            line,
59            column: self.count_units_to(offset) - self.count_units_to(start) + 1 + extra,
60            offset: Some(self.count_units_to(offset) + extra),
61        }
62    }
63
64    /// The unist position of a position, which is unchanged when it is one already.
65    pub fn to_unist_position(&self, position: &Position<'_>) -> UnistPosition {
66        match position {
67            Position::Span { start, end } => UnistPosition {
68                start: self.to_unist_point(*start),
69                end: self.to_unist_point(*end),
70            },
71            Position::Unist(position) => (*position).clone(),
72        }
73    }
74
75    // UTF-16 units before byte `to`, from the checkpoint before it
76    fn count_units_to(&self, to: usize) -> usize {
77        if self.ascii {
78            return to;
79        }
80        let block = to / BLOCK;
81        let before = self.checkpoints.get(block).copied().unwrap_or_default();
82        let bytes = self.source.as_bytes();
83        before + count_byte_units(bytes.get(block * BLOCK..to).unwrap_or_default())
84    }
85}
86
87// Each UTF-8 lead byte starts one UTF-16 unit, or two for a four-byte character
88fn count_byte_units(bytes: &[u8]) -> usize {
89    bytes
90        .iter()
91        .map(|byte| match byte {
92            0x80..=0xBF => 0,
93            0xF0..=0xFF => 2,
94            _ => 1,
95        })
96        .sum()
97}
98
99#[cfg(test)]
100#[path = "lines_unit.rs"]
101mod unit;
textlane_core/models/lines.rs

textlane_core/models/
lines.rs
