Struct Tokenizer 

pub struct Tokenizer { /* private fields */ }

Tokenizer implementation that wraps HuggingFace tokenizers

Implementations

impl Tokenizer

pub fn new() -> Self

Create a new tokenizer with basic vocabulary (fallback)

pub fn from_file<P: AsRef<Path>>(path: P) -> ModelResult<Self>

Load tokenizer from a tokenizer.json file (HuggingFace format)

pub fn from_model_dir<P: AsRef<Path>>(model_dir: P) -> ModelResult<Self>

Load tokenizer from a model directory (looks for tokenizer.json)

pub fn from_gguf(gguf_tokenizer: &GGUFTokenizer) -> ModelResult<Self>

Create tokenizer from GGUF tokenizer data

pub fn from_model_weights(weights: &ModelWeights) -> ModelResult<Self>

Create tokenizer from ModelWeights (prefers GGUF tokenizer if available)

pub fn from_pretrained(model_name: &str) -> ModelResult<Self>

Create tokenizer from pretrained model name (downloads from HuggingFace) Note: This requires network access and HuggingFace Hub authentication for gated models

pub fn encode(&self, text: &str) -> Vec<u32>

Encode text to token IDs

pub fn encode_with_special_tokens( &self, text: &str, add_bos: bool, add_eos: bool, ) -> Vec<u32>

Encode text with special tokens

pub fn decode(&self, token_ids: &[u32]) -> String

Decode token IDs back to text

pub fn decode_raw(&self, token_ids: &[u32]) -> String

Decode token IDs without special token filtering

pub fn vocab_size(&self) -> usize

Get vocabulary size

pub fn special_tokens(&self) -> &SpecialTokens

Get special tokens

pub fn bos_token_id(&self) -> u32

Get BOS token ID

pub fn eos_token_id(&self) -> u32

Get EOS token ID

pub fn pad_token_id(&self) -> u32

Get PAD token ID

pub fn unk_token_id(&self) -> u32

Get UNK token ID

pub fn contains_token(&self, token: &str) -> bool

Check if token exists in vocabulary

pub fn token_to_id(&self, token: &str) -> Option<u32>

Get token ID for a string

pub fn id_to_token(&self, id: u32) -> Option<String>

Get string for a token ID

Trait Implementations

impl Default for Tokenizer

fn default() -> Self

Returns the “default value” for a type.