Struct ConfigurableAsyncTokenizer 

pub struct ConfigurableAsyncTokenizer<T> { /* private fields */ }

Advanced async tokenizer with configurable behavior

Implementations

impl<T> ConfigurableAsyncTokenizer<T>

where T: Tokenizer + Send + Sync + 'static,

pub fn new(tokenizer: T, config: AsyncTokenizerConfig) -> Self

Create a new configurable async tokenizer

pub fn update_config(&mut self, config: AsyncTokenizerConfig)

Update configuration

pub fn config(&self) -> &AsyncTokenizerConfig

Get current configuration

pub async fn encode_large_batch_with_progress<F>( &self, texts: &[&str], progress_callback: F, ) -> Result<Vec<TokenizedInput>>

where F: FnMut(usize, usize) + Send + 'static,

Process a large batch with progress reporting

Trait Implementations

impl<T> AsyncTokenizer for ConfigurableAsyncTokenizer<T>

where T: Tokenizer + Send + Sync + 'static,

fn encode_async<'life0, 'life1, 'async_trait>( &'life0 self, text: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<TokenizedInput>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Asynchronously encode a single text

fn encode_pair_async<'life0, 'life1, 'life2, 'async_trait>( &'life0 self, text: &'life1 str, text2: &'life2 str, ) -> Pin<Box<dyn Future<Output = Result<TokenizedInput>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait,

Asynchronously encode text pairs

fn decode_async<'life0, 'life1, 'async_trait>( &'life0 self, ids: &'life1 [u32], ) -> Pin<Box<dyn Future<Output = Result<String>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Asynchronously decode token IDs to text

fn encode_batch_async<'life0, 'life1, 'life2, 'async_trait>( &'life0 self, texts: &'life1 [&'life2 str], ) -> Pin<Box<dyn Future<Output = Result<Vec<TokenizedInput>>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait,

Asynchronously encode multiple texts in parallel

fn encode_pair_batch_async<'life0, 'life1, 'life2, 'life3, 'async_trait>( &'life0 self, text_pairs: &'life1 [(&'life2 str, &'life3 str)], ) -> Pin<Box<dyn Future<Output = Result<Vec<TokenizedInput>>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait,

Asynchronously encode text pairs in parallel

fn encode_stream<'a>( &'a self, texts: Vec<String>, ) -> BoxFuture<'a, Result<Box<dyn Stream<Item = Result<TokenizedInput>> + Send + Unpin>>>

Stream-based encoding for large datasets