Struct Tokenizer 

pub struct Tokenizer(/* private fields */);

Main tokenizer wrapper that provides a unified interface for different tokenizer implementations

Implementations

impl Tokenizer

pub fn from_file(file_path: &str) -> Result<Tokenizer>

Create a tokenizer from a file path

pub fn from_file_with_chat_template( file_path: &str, chat_template_path: Option<&str>, ) -> Result<Tokenizer>

Create a tokenizer from a file path with an optional chat template

pub fn from_arc(tokenizer: Arc<dyn Tokenizer>) -> Self

Create a tokenizer from an Arc

pub fn decode_stream( &self, prompt_token_ids: &[u32], skip_special_tokens: bool, ) -> DecodeStream

Create a stateful sequence object for decoding token_ids into text

pub fn encode(&self, input: &str, add_special_tokens: bool) -> Result<Encoding>

Direct encode method

Set add_special_tokens to true for embeddings (to add BOS/EOS tokens configured in tokenizer_config.json), or false for chat completion (where the chat template handles special tokens).

pub fn encode_batch( &self, inputs: &[&str], add_special_tokens: bool, ) -> Result<Vec<Encoding>>

Direct batch encode method

Set add_special_tokens to true for embeddings (to add BOS/EOS tokens configured in tokenizer_config.json), or false for chat completion (where the chat template handles special tokens).

pub fn decode( &self, token_ids: &[u32], skip_special_tokens: bool, ) -> Result<String>

Direct decode method

pub fn vocab_size(&self) -> usize

Get vocabulary size

pub fn get_special_tokens(&self) -> &SpecialTokens

Get special tokens

pub fn token_to_id(&self, token: &str) -> Option<u32>

Convert token string to ID

pub fn id_to_token(&self, id: u32) -> Option<String>

Convert ID to token string

Trait Implementations

impl Clone for Tokenizer

fn clone(&self) -> Tokenizer

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl From<Arc<dyn Tokenizer>> for Tokenizer

fn from(tokenizer: Arc<dyn Tokenizer>) -> Self

Converts to this type from the input type.