Struct TokenSplitter 

pub struct TokenSplitter { /* private fields */ }

Implementations

impl TokenSplitter

pub fn new(options: SplitterOptions) -> TokenSplitter

pub fn get_tokenizer_from_str(&self, s: &str) -> Option<Tokenizer>

👎Deprecated:

Use SplitterOptions::get_tokenizer_from_str instead

Trait Implementations

impl Clone for TokenSplitter

fn clone(&self) -> TokenSplitter

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for TokenSplitter

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for TokenSplitter

fn default() -> Self

Returns the “default value” for a type.

impl TextSplitter for TokenSplitter

fn split_text<'life0, 'life1, 'async_trait>( &'life0 self, text: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<Vec<String>, TextSplitterError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

fn split_documents<'life0, 'life1, 'async_trait>( &'life0 self, documents: &'life1 [Document], ) -> Pin<Box<dyn Future<Output = Result<Vec<Document>, TextSplitterError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

fn create_documents<'life0, 'life1, 'life2, 'async_trait>( &'life0 self, text: &'life1 [String], metadatas: &'life2 [HashMap<String, Value>], ) -> Pin<Box<dyn Future<Output = Result<Vec<Document>, TextSplitterError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait,