chunk 0.10.2

The fastest semantic text chunking library — up to 1TB/s chunking throughput
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115

//! Benchmark pattern vs delimiter throughput

use chunk::chunk;
use std::time::Instant;

fn main() {
    // Create test data - mix of spaces and metaspaces
    let base_text = "The quick brown fox jumps over the lazy dog. ";
    let metaspace = "▁";

    // Create ~100MB of test data with spaces
    let repeat_count = 100_000_000 / base_text.len();
    let text_with_spaces: String = base_text.repeat(repeat_count);
    let text_bytes_spaces = text_with_spaces.as_bytes();

    // Create same size text with metaspaces instead of spaces
    let base_with_metaspace = base_text.replace(' ', metaspace);
    let text_with_metaspace: String = base_with_metaspace.repeat(repeat_count);
    let text_bytes_metaspace = text_with_metaspace.as_bytes();

    let size_mb_spaces = text_bytes_spaces.len() as f64 / 1_000_000.0;
    let size_mb_metaspace = text_bytes_metaspace.len() as f64 / 1_000_000.0;

    println!("Benchmark: Pattern vs Delimiter Throughput");
    println!("==========================================");
    println!(
        "Text with spaces:    {:.2} MB ({} bytes)",
        size_mb_spaces,
        text_bytes_spaces.len()
    );
    println!(
        "Text with metaspace: {:.2} MB ({} bytes)",
        size_mb_metaspace,
        text_bytes_metaspace.len()
    );
    println!();

    const ITERATIONS: usize = 10;

    // Test with different chunk sizes
    for &chunk_size in &[1024, 4096, 16384, 65536] {
        println!("=== Chunk size: {} bytes ===", chunk_size);
        bench_delimiter(text_bytes_spaces, size_mb_spaces, chunk_size, ITERATIONS);
        bench_pattern(
            text_bytes_metaspace,
            size_mb_metaspace,
            metaspace.as_bytes(),
            chunk_size,
            ITERATIONS,
        );
        println!();
    }
}

fn bench_delimiter(text: &[u8], size_mb: f64, chunk_size: usize, iterations: usize) {
    // Warmup
    let _: Vec<_> = chunk(text)
        .size(chunk_size)
        .delimiters(b" ")
        .prefix()
        .collect();

    let mut times = Vec::new();
    let mut num_chunks = 0;
    for _ in 0..iterations {
        let start = Instant::now();
        let chunks: Vec<_> = chunk(text)
            .size(chunk_size)
            .delimiters(b" ")
            .prefix()
            .collect();
        let elapsed = start.elapsed();
        times.push(elapsed.as_secs_f64());
        num_chunks = chunks.len();
    }
    let avg = times.iter().sum::<f64>() / iterations as f64;
    let throughput = size_mb / avg / 1000.0; // GB/s
    println!(
        "  Single-byte (space):    {:>7.2} GB/s  ({:>6} chunks, {:.3}ms avg)",
        throughput,
        num_chunks,
        avg * 1000.0
    );
}

fn bench_pattern(text: &[u8], size_mb: f64, pattern: &[u8], chunk_size: usize, iterations: usize) {
    // Warmup
    let _: Vec<_> = chunk(text)
        .size(chunk_size)
        .pattern(pattern)
        .prefix()
        .collect();

    let mut times = Vec::new();
    let mut num_chunks = 0;
    for _ in 0..iterations {
        let start = Instant::now();
        let chunks: Vec<_> = chunk(text)
            .size(chunk_size)
            .pattern(pattern)
            .prefix()
            .collect();
        let elapsed = start.elapsed();
        times.push(elapsed.as_secs_f64());
        num_chunks = chunks.len();
    }
    let avg = times.iter().sum::<f64>() / iterations as f64;
    let throughput = size_mb / avg / 1000.0; // GB/s
    println!(
        "  Multi-byte (metaspace): {:>7.2} GB/s  ({:>6} chunks, {:.3}ms avg)",
        throughput,
        num_chunks,
        avg * 1000.0
    );
}