chonkier 0.0.2

🦛 Chonkie, now in Rust 🦀: No-nonsense, ultra-fast, ultra-light chunking library
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16

/*
    A simple example of token chunking with Chonkier's TokenChunker.
*/
use chonkier::CharacterTokenizer;
use chonkier::TokenChunker;

fn main() {
    let chunker = TokenChunker::new(CharacterTokenizer::new(), 20, 5);
    let text = "The quick brown fox jumps over the lazy dog.";
    let chunks = chunker.chunk(&text);
    for chunk in chunks {
        println!("Chunk: {}", chunk.text);
        println!("Tokens: {}", chunk.token_count);
    }

}