Struct TokenizerBridge 

pub struct TokenizerBridge { /* private fields */ }

Tokenizer bridge wrapping the HuggingFace tokenizers library.

Implementations

impl TokenizerBridge

pub fn from_file(path: &str) -> RuntimeResult<Self>

Load a tokenizer from a JSON file path.

pub fn from_bytes(json: &[u8]) -> RuntimeResult<Self>

Create a tokenizer from JSON bytes (e.g., from GGUF metadata).

pub fn encode(&self, text: &str) -> RuntimeResult<Vec<u32>>

Encode text to token IDs.

pub fn decode(&self, tokens: &[u32]) -> RuntimeResult<String>

Decode token IDs back to text.

pub fn vocab_size(&self) -> usize

Get the vocabulary size.

pub fn bos_token_id(&self) -> Option<u32>

Get the BOS (beginning of sequence) token ID, if any.

pub fn eos_token_id(&self) -> Option<u32>

Get the EOS (end of sequence) token ID, if any.

pub fn id_to_token(&self, id: u32) -> Option<String>

Get the string representation of a single token ID.

Returns None if the id is not in the vocabulary.

pub fn token_to_bytes(&self, id: u32) -> Option<Vec<u8>>

Get the byte representation of a single token ID.

Uses decode (skip_special_tokens=false) to produce the canonical bytes for byte-level BPE tokenizers. Returns None if the id is unknown.

pub fn vocab_bytes(&self) -> Vec<(u32, Vec<u8>)>

Build the full vocab as (token_id, byte_representation) pairs.

This is used to pre-compute the vocabulary for grammar masking. The result can be cached and shared across generation steps.

pub fn vocab_bytes_cached(&self) -> &[(u32, Vec<u8>)]

Get cached vocabulary bytes. Computes on first call, returns cached thereafter.