tokenx 0.2.0

vocab-free heuristic LLM token-count estimation
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101

//! Chunking parity, mirroring the splitByTokens suite.

mod common;

use common::ENGLISH_TEXT;
use tokenx::{split_by_tokens, split_by_tokens_with, SplitByTokensOptions};

#[test]
fn split_into_chunks_is_lossless() {
    let chunks = split_by_tokens(ENGLISH_TEXT, 5);
    assert!(chunks.len() > 1);
    assert_eq!(chunks.concat(), ENGLISH_TEXT);
    assert_eq!(
        chunks,
        vec!["Hello, world! This", " is a short sentence", "."]
    );
}

#[test]
fn overlap_grows_chunk_count_and_chars() {
    let no_overlap = split_by_tokens(ENGLISH_TEXT, 5);
    let opts = SplitByTokensOptions {
        overlap: Some(2),
        ..Default::default()
    };
    let with_overlap = split_by_tokens_with(ENGLISH_TEXT, 5, &opts);

    assert!(with_overlap.len() >= no_overlap.len());

    let total_with: usize = with_overlap.iter().map(|c| c.encode_utf16().count()).sum();
    let total_without: usize = no_overlap.iter().map(|c| c.encode_utf16().count()).sum();
    assert!(total_with >= total_without);

    assert_eq!(
        with_overlap,
        vec![
            "Hello, world! This",
            "! This is a short",
            "a short sentence.",
            "sentence."
        ]
    );
}

#[test]
fn empty_or_zero_chunk_returns_empty() {
    // tokens_per_chunk is unsigned, so 0 is the only non-producing budget. A
    // negative budget cannot be expressed.
    assert!(split_by_tokens("", 5).is_empty());
    assert!(split_by_tokens("text", 0).is_empty());
}

#[test]
fn overlap_at_or_above_chunk_size_carries_whole_chunk() {
    // overlap >= tokens_per_chunk carries the whole chunk into the next
    // iteration, so progress comes only from new segments. There is no guard
    // against this, by design. On short text the loop still terminates. This
    // pins the exact output so a future guard cannot change it silently.
    let opts = SplitByTokensOptions {
        overlap: Some(5),
        ..Default::default()
    };
    let chunks = split_by_tokens_with("Hello, world! This is a short sentence.", 5, &opts);
    assert_eq!(
        chunks,
        vec![
            "Hello, world! This",
            "Hello, world! This ",
            "Hello, world! This is",
            ", world! This is ",
            ", world! This is a",
            "world! This is a ",
            "world! This is a short",
            "! This is a short ",
            "! This is a short sentence",
            "is a short sentence.",
            "a short sentence."
        ]
    );
}

#[test]
fn short_text_is_single_chunk() {
    assert_eq!(split_by_tokens("Hi there", 100), vec!["Hi there"]);
}

#[test]
fn split_with_options_changes_chunking_but_stays_lossless() {
    let default_chunks = split_by_tokens(ENGLISH_TEXT, 5);
    let opts = SplitByTokensOptions {
        default_chars_per_token: Some(3.0),
        ..Default::default()
    };
    let custom_chunks = split_by_tokens_with(ENGLISH_TEXT, 5, &opts);
    assert_ne!(custom_chunks, default_chunks);
    assert_eq!(custom_chunks.concat(), ENGLISH_TEXT);
    assert_eq!(
        custom_chunks,
        vec!["Hello, world", "! This is a", " short sentence", "."]
    );
}