toklen 0.2.0

A single-threaded, lightweight, and fast token counter.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59

use crate::pre_tokenized::PreTokenizedString;
use crate::pre_tokenized::PtSplit;

use super::Error;

/// WhitespaceSplit pre-tokenizer: splits on **each individual** whitespace
/// character, unlike [`Whitespace`](super::whitespace) which treats whitespace
/// runs as a single delimiter.
///
/// ```text
/// Whitespace:      "a  b" → ["a", "b"]       (run of 2 spaces → 1 split)
/// WhitespaceSplit: "a  b" → ["a", "", "b"]   (each space → separate split)
/// ```
///
/// Empty segments are discarded, so token **counts** are identical in both
/// cases — but the intermediate split boundaries differ, which matters when
/// chained with other pre-tokenizers in a [`Sequence`].
pub fn pre_tokenize(pts: &mut PreTokenizedString) -> Result<(), Error> {
    let buffer = pts.buffer.as_str();
    let old_splits = &pts.splits;
    let hint: usize = old_splits.len() << 1; // Vec::len <= isize::MAX
    let mut new_splits = Vec::with_capacity(hint);

    for split in old_splits {
        if split.token_id.is_some() {
            new_splits.push(split.clone());
            continue;
        }

        let text = &buffer[split.range.clone()];
        if text.is_empty() {
            continue;
        }

        let base = split.range.start;
        let mut seg_start = 0usize;

        for (i, ch) in text.char_indices() {
            if ch.is_whitespace() {
                // Close the segment before this whitespace char.
                if i > seg_start {
                    let range = (base + seg_start)..(base + i);
                    new_splits.push(PtSplit::from_range(range));
                }
                // Skip the whitespace char itself (start next segment after it).
                seg_start = i + ch.len_utf8();
            }
        }

        // Trailing segment after the last whitespace.
        if seg_start < text.len() {
            let range = (base + seg_start)..(base + text.len());
            new_splits.push(PtSplit::from_range(range));
        }
    }

    pts.splits = new_splits;
    Ok(())
}