Trait TryChunkSizer

Source

pub trait TryChunkSizer {
    type Error;

    // Required method
    fn try_size(&self, chunk: &str) -> Result<usize, Self::Error>;
}

Expand description

Determines the size of a chunk, returning an error if sizing fails.

All ChunkSizer implementations automatically implement this trait with Infallible as their error type. This adapter preserves their behavior, including any panics; it does not catch panics or convert them into errors. A type implementing ChunkSizer cannot also provide a separate implementation of this trait. Use a newtype for different fallible behavior.

use text_splitter::{ChunkConfig, TextSplitter, TryChunkSizer};

struct CheckedCharacters;

impl TryChunkSizer for CheckedCharacters {
    type Error = &'static str;

    fn try_size(&self, chunk: &str) -> Result<usize, Self::Error> {
        if chunk.contains('\0') {
            Err("NUL is not supported")
        } else {
            Ok(chunk.chars().count())
        }
    }
}

let splitter = TextSplitter::new(ChunkConfig::new(10).with_sizer(CheckedCharacters));
let chunks = splitter.try_chunks("hello\0world").collect::<Result<Vec<_>, _>>();
assert_eq!(chunks, Err("NUL is not supported"));