Struct ParallelTokenizer 

pub struct ParallelTokenizer<T: Tokenizer + Sync> { /* private fields */ }

Parallel batch tokenization utilities for improved throughput

Implementations

impl<T: Tokenizer + Sync> ParallelTokenizer<T>

pub fn new(tokenizer: T) -> Self

Create a new parallel tokenizer wrapper

pub fn with_chunk_size(tokenizer: T, chunk_size: usize) -> Self

Create a new parallel tokenizer wrapper with custom chunk size

pub fn encode_batch(&self, texts: &[&str]) -> Result<Vec<TokenizedInput>>

Encode a batch of texts in parallel

pub fn encode_pair_batch( &self, text_pairs: &[(&str, &str)], ) -> Result<Vec<TokenizedInput>>

Encode pairs of texts in parallel

pub fn decode_batch(&self, ids_batch: &[&[u32]]) -> Result<Vec<String>>

Decode a batch of token IDs in parallel

pub fn tokenizer(&self) -> &T

Get the underlying tokenizer

pub fn chunk_size(&self) -> usize

Get the chunk size

pub fn set_chunk_size(&mut self, chunk_size: usize)

Set the chunk size for batching