Struct Chunker 

pub struct Chunker { /* private fields */ }

Text chunker with configurable strategy and parameters.

Example

use pdfvec::{Chunker, ChunkStrategy};

let chunker = Chunker::new(ChunkStrategy::Fixed)
    .chunk_size(512)
    .overlap(50);

let text = "Your document text here...";
let chunks: Vec<_> = chunker.chunks(text).collect();

Implementations

impl Chunker

pub fn new(strategy: ChunkStrategy) -> Self

Creates a new chunker with the specified strategy.

Example

use pdfvec::{Chunker, ChunkStrategy};

let chunker = Chunker::new(ChunkStrategy::Paragraph);

pub fn chunk_size(self, size: usize) -> Self

Sets the target chunk size in characters.

For ChunkStrategy::Fixed, this is the exact size (except for the last chunk). For other strategies, this is the target size for merging small segments.

Default: 512

Example

use pdfvec::{Chunker, ChunkStrategy};

let chunker = Chunker::new(ChunkStrategy::Fixed).chunk_size(1024);

pub fn overlap(self, chars: usize) -> Self

Sets the overlap between consecutive chunks in characters.

Overlap helps maintain context across chunk boundaries. Only applies to ChunkStrategy::Fixed.

Default: 0

Example

use pdfvec::{Chunker, ChunkStrategy};

let chunker = Chunker::new(ChunkStrategy::Fixed)
    .chunk_size(100)
    .overlap(20);

pub fn min_chunk_size(self, size: usize) -> Self

Sets the minimum chunk size for paragraph and sentence strategies.

Chunks smaller than this will be merged with adjacent chunks.

Default: 100

Example

use pdfvec::{Chunker, ChunkStrategy};

let chunker = Chunker::new(ChunkStrategy::Paragraph)
    .min_chunk_size(50);

pub fn chunks<'a>( &self, text: &'a str, ) -> Box<dyn Iterator<Item = Chunk<'a>> + 'a>

Returns an iterator over chunks of the input text.

Example

use pdfvec::{Chunker, ChunkStrategy};

let text = "Hello world. How are you?";
let chunker = Chunker::new(ChunkStrategy::Sentence);

for chunk in chunker.chunks(text) {
    println!("Chunk {}: {}", chunk.index(), chunk.text());
}

pub fn get_chunk_size(&self) -> usize

Returns the configured chunk size.

pub fn get_overlap(&self) -> usize

Returns the configured overlap.

pub fn strategy(&self) -> ChunkStrategy

Returns the configured strategy.

Trait Implementations

impl Clone for Chunker

fn clone(&self) -> Chunker

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for Chunker

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for Chunker

fn default() -> Self

Returns the “default value” for a type.