Struct langchain_rust::text_splitter::TokenSplitter

pub struct TokenSplitter { /* private fields */ }

Implementations

impl TokenSplitter

pub fn new(options: SplitterOptions) -> TokenSplitter

pub fn get_tokenizer_from_str(&self, s: &str) -> Option<Tokenizer>

Trait Implementations

impl Clone for TokenSplitter

fn clone(&self) -> TokenSplitter

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for TokenSplitter

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for TokenSplitter

fn default() -> Self

Returns the “default value” for a type.

impl TextSplitter for TokenSplitter

fn split_text(&self, text: &str) -> Result<Vec<String>, Box<dyn Error>>

fn split_documents( &self, documents: &[Document] ) -> Result<Vec<Document>, Box<dyn Error>>

fn create_documents( &self, text: &[String], metadatas: &[HashMap<String, Value>] ) -> Result<Vec<Document>, Box<dyn Error>>