Trait llm_chain::text_splitter::TextSplitter

pub trait TextSplitter<TokenType>: Tokenizer<TokenType>where
    TokenType: Clone,{
    // Provided method
    fn split_text(
        &self,
        doc: &str,
        max_tokens_per_chunk: usize,
        chunk_overlap: usize
    ) -> Result<Vec<String>, TokenizerError> { ... }
}

Provided Methods

fn split_text( &self, doc: &str, max_tokens_per_chunk: usize, chunk_overlap: usize ) -> Result<Vec<String>, TokenizerError>

Implementors

impl TextSplitter<String> for NaiveWhitespaceSplitter