Struct BatchTokenizer 

pub struct BatchTokenizer<T: Tokenizer + Sync> { /* private fields */ }

Batch tokenization with padding and truncation support

Implementations

impl<T: Tokenizer + Sync> BatchTokenizer<T>

pub fn new(tokenizer: T) -> Self

Create a new batch tokenizer

pub fn with_max_length(self, max_length: usize) -> Self

Set the maximum sequence length

pub fn with_padding(self, pad_token_id: u32) -> Self

Enable padding to max length

pub fn with_truncation(self) -> Self

Enable truncation to max length

pub fn encode_batch_padded( &self, texts: &[&str], ) -> Result<BatchedTokenizedInput>

Encode a batch with padding and truncation

pub fn tokenizer(&self) -> &T

Get the underlying tokenizer

Trait Implementations

impl<T: Clone + Tokenizer + Sync> Clone for BatchTokenizer<T>

fn clone(&self) -> BatchTokenizer<T>

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl<T: Debug + Tokenizer + Sync> Debug for BatchTokenizer<T>

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.