sentience-tokenize 0.2.3

Tiny Rust zero-dep tokenizer (ident, number, string, parens, operators, keywords).
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

use crate::Span;

pub struct LineMap {
    starts: Vec<usize>,
}

impl LineMap {
    pub fn new(src: &str) -> Self {
        let mut starts = Vec::with_capacity(128);
        starts.push(0);
        for (i, b) in src.bytes().enumerate() {
            if b == b'\n' {
                starts.push(i + 1);
            }
        }
        Self { starts }
    }

    pub fn to_line_col(&self, byte: usize) -> (usize, usize) {
        let i = match self.starts.binary_search(&byte) {
            Ok(k) => k,
            Err(k) => k.saturating_sub(1),
        };
        (i + 1, byte.saturating_sub(self.starts[i]) + 1)
    }

    pub fn span_lines(&self, span: Span) -> ((usize, usize), (usize, usize)) {
        (self.to_line_col(span.start), self.to_line_col(span.end))
    }

    pub fn line_count(&self) -> usize {
        self.starts.len()
    }
}