pub struct TokenSplitter { /* private fields */ }
Implementations§
Source§impl TokenSplitter
impl TokenSplitter
pub fn new(options: SplitterOptions) -> TokenSplitter
pub fn get_tokenizer_from_str(&self, s: &str) -> Option<Tokenizer>
👎Deprecated: Use
SplitterOptions::get_tokenizer_from_str
insteadTrait Implementations§
Source§impl Clone for TokenSplitter
impl Clone for TokenSplitter
Source§fn clone(&self) -> TokenSplitter
fn clone(&self) -> TokenSplitter
Returns a copy of the value. Read more
1.0.0 · Source§fn clone_from(&mut self, source: &Self)
fn clone_from(&mut self, source: &Self)
Performs copy-assignment from
source
. Read moreSource§impl Debug for TokenSplitter
impl Debug for TokenSplitter
Source§impl Default for TokenSplitter
impl Default for TokenSplitter
Source§impl TextSplitter for TokenSplitter
impl TextSplitter for TokenSplitter
fn split_text<'life0, 'life1, 'async_trait>(
&'life0 self,
text: &'life1 str,
) -> Pin<Box<dyn Future<Output = Result<Vec<String>, TextSplitterError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
fn split_documents<'life0, 'life1, 'async_trait>(
&'life0 self,
documents: &'life1 [Document],
) -> Pin<Box<dyn Future<Output = Result<Vec<Document>, TextSplitterError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
fn create_documents<'life0, 'life1, 'life2, 'async_trait>(
&'life0 self,
text: &'life1 [String],
metadatas: &'life2 [HashMap<String, Value>],
) -> Pin<Box<dyn Future<Output = Result<Vec<Document>, TextSplitterError>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
'life1: 'async_trait,
'life2: 'async_trait,
Auto Trait Implementations§
impl Freeze for TokenSplitter
impl RefUnwindSafe for TokenSplitter
impl Send for TokenSplitter
impl Sync for TokenSplitter
impl Unpin for TokenSplitter
impl UnwindSafe for TokenSplitter
Blanket Implementations§
Source§impl<T> BorrowMut<T> for Twhere
T: ?Sized,
impl<T> BorrowMut<T> for Twhere
T: ?Sized,
Source§fn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
Mutably borrows from an owned value. Read more
Source§impl<T> CloneToUninit for Twhere
T: Clone,
impl<T> CloneToUninit for Twhere
T: Clone,
Source§impl<T> Instrument for T
impl<T> Instrument for T
Source§fn instrument(self, span: Span) -> Instrumented<Self>
fn instrument(self, span: Span) -> Instrumented<Self>
Source§fn in_current_span(self) -> Instrumented<Self>
fn in_current_span(self) -> Instrumented<Self>
Source§impl<T> IntoEither for T
impl<T> IntoEither for T
Source§fn into_either(self, into_left: bool) -> Either<Self, Self>
fn into_either(self, into_left: bool) -> Either<Self, Self>
Converts
self
into a Left
variant of Either<Self, Self>
if into_left
is true
.
Converts self
into a Right
variant of Either<Self, Self>
otherwise. Read moreSource§fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
Converts
self
into a Left
variant of Either<Self, Self>
if into_left(&self)
returns true
.
Converts self
into a Right
variant of Either<Self, Self>
otherwise. Read more