pub struct BatchedStreamingTokenizer<T: Tokenizer> { /* private fields */ }Expand description
Batched streaming tokenizer for processing multiple streams
Implementations§
Source§impl<T: Tokenizer> BatchedStreamingTokenizer<T>
impl<T: Tokenizer> BatchedStreamingTokenizer<T>
Sourcepub fn with_streaming_params(
self,
buffer_size: usize,
overlap_size: usize,
) -> Self
pub fn with_streaming_params( self, buffer_size: usize, overlap_size: usize, ) -> Self
Set streaming parameters
Sourcepub fn with_max_chunk_length(self, max_length: usize) -> Self
pub fn with_max_chunk_length(self, max_length: usize) -> Self
Set max chunk length
Sourcepub fn process_text_batch(
&self,
texts: &[String],
) -> Result<Vec<Vec<TokenizedInput>>>
pub fn process_text_batch( &self, texts: &[String], ) -> Result<Vec<Vec<TokenizedInput>>>
Process multiple text streams in batches
Sourcepub fn batch_size(&self) -> usize
pub fn batch_size(&self) -> usize
Get batch size
Sourcepub fn streaming_tokenizer(&self) -> &StreamingTokenizer<T>
pub fn streaming_tokenizer(&self) -> &StreamingTokenizer<T>
Get the underlying streaming tokenizer
Auto Trait Implementations§
impl<T> Freeze for BatchedStreamingTokenizer<T>where
T: Freeze,
impl<T> RefUnwindSafe for BatchedStreamingTokenizer<T>where
T: RefUnwindSafe,
impl<T> Send for BatchedStreamingTokenizer<T>
impl<T> Sync for BatchedStreamingTokenizer<T>
impl<T> Unpin for BatchedStreamingTokenizer<T>where
T: Unpin,
impl<T> UnsafeUnpin for BatchedStreamingTokenizer<T>where
T: UnsafeUnpin,
impl<T> UnwindSafe for BatchedStreamingTokenizer<T>where
T: UnwindSafe,
Blanket Implementations§
Source§impl<T> BorrowMut<T> for Twhere
T: ?Sized,
impl<T> BorrowMut<T> for Twhere
T: ?Sized,
Source§fn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
Mutably borrows from an owned value. Read more
Source§impl<T> Instrument for T
impl<T> Instrument for T
Source§fn instrument(self, span: Span) -> Instrumented<Self>
fn instrument(self, span: Span) -> Instrumented<Self>
Source§fn in_current_span(self) -> Instrumented<Self>
fn in_current_span(self) -> Instrumented<Self>
Source§impl<T> IntoEither for T
impl<T> IntoEither for T
Source§fn into_either(self, into_left: bool) -> Either<Self, Self>
fn into_either(self, into_left: bool) -> Either<Self, Self>
Converts
self into a Left variant of Either<Self, Self>
if into_left is true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read moreSource§fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
Converts
self into a Left variant of Either<Self, Self>
if into_left(&self) returns true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read more