cpd-tokenizer 0.1.5

Source code tokenizers for cpd
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

use cpd_core::models::Location;

pub(crate) struct LineIndex {
    newlines: Vec<usize>,
}

impl LineIndex {
    pub(crate) fn new(content: &[u8]) -> Self {
        let newlines = content
            .iter()
            .enumerate()
            .filter_map(|(i, &b)| (b == b'\n').then_some(i))
            .collect();
        Self { newlines }
    }

    pub(crate) fn location(&self, offset: usize) -> Location {
        let previous_newlines = self.newlines.partition_point(|&nl| nl < offset);
        let line_start = if previous_newlines == 0 {
            0
        } else {
            self.newlines[previous_newlines - 1] + 1
        };
        Location {
            line: previous_newlines as u32 + 1,
            column: (offset - line_start) as u32,
            offset: offset as u32,
        }
    }
}