Struct Tokenizer 

pub struct Tokenizer { /* private fields */ }

Wrapper around the HuggingFace tokenizer.

Implementations

impl Tokenizer

pub fn from_file(path: impl AsRef<Path>) -> Result<Self, TokenizerError>

Load a tokenizer from a tokenizer.json file.

pub fn from_json(json: &str) -> Result<Self, TokenizerError>

Load a tokenizer from a JSON string.

pub fn encode(&self, text: &str) -> Result<Vec<u32>, TokenizerError>

Encode text into token IDs.

pub fn encode_with_special( &self, text: &str, ) -> Result<Vec<u32>, TokenizerError>

Encode text with special tokens (e.g., BOS).

pub fn decode(&self, ids: &[u32]) -> Result<String, TokenizerError>

Decode token IDs back to text.

pub fn decode_one(&self, id: u32) -> Result<String, TokenizerError>

Decode a single token ID to text.

pub fn vocab_size(&self) -> usize

Get the vocabulary size.

pub fn token_to_id(&self, token: &str) -> Option<u32>

Get the token ID for a special token by content (e.g., “<|endoftext|>”).

pub fn bos_token_id(&self) -> Option<u32>

Get the BOS (beginning of sequence) token ID, if defined.

pub fn eos_token_id(&self) -> Option<u32>

Get the EOS (end of sequence) token ID, if defined.

pub fn stop_token_ids(&self) -> Vec<u32>

Get all stop token IDs (EOS + chat-specific stop tokens). Used to detect when generation should stop.

pub fn is_stop_token(&self, token_id: u32) -> bool

Check if a token ID is a stop token.