toklen 0.2.0

A single-threaded, lightweight, and fast token counter.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56

use crate::pre_tokenized::PreTokenizedString;
use crate::pre_tokenized::PtSplit;

use super::Error;

/// Whitespace pre-tokenizer: splits on whitespace **runs**, unlike
/// [`WhitespaceSplit`](super::whitespace_split) which splits on each individual
/// whitespace character.
///
/// ```text
/// Whitespace:      "a  b" → ["a", "b"]       (2 spaces → 1 split)
/// WhitespaceSplit: "a  b" → ["a", "", "b"]   (each space → separate split)
/// ```
pub fn pre_tokenize(pts: &mut PreTokenizedString) -> Result<(), Error> {
    let buffer = pts.buffer.as_str();
    let old_splits = &pts.splits;
    let hint: usize = old_splits.len() << 1; // Vec::len <= isize::MAX
    let mut new_splits = Vec::with_capacity(hint);

    for split in old_splits {
        if split.token_id.is_some() {
            new_splits.push(split.clone());
            continue;
        }

        let text = &buffer[split.range.clone()];
        if text.is_empty() {
            continue;
        }

        let base = split.range.start;
        let mut seg_start: Option<usize> = None;

        for (i, ch) in text.char_indices() {
            if ch.is_whitespace() {
                // Close current non-whitespace segment (if any).
                if let Some(start) = seg_start.take() {
                    let range = (base + start)..(base + i);
                    new_splits.push(PtSplit::from_range(range));
                }
            } else if seg_start.is_none() {
                // Start a new non-whitespace segment.
                seg_start = Some(i);
            }
        }

        // Trailing segment.
        if let Some(start) = seg_start {
            let range = (base + start)..(base + text.len());
            new_splits.push(PtSplit::from_range(range));
        }
    }

    pts.splits = new_splits;
    Ok(())
}