Struct BundleTokenizer 

pub struct BundleTokenizer { /* private fields */ }

Implementations

impl BundleTokenizer

pub fn try_load(dir: &Path) -> Result<Option<Self>, EngineError>

Load from a bundle directory. Returns Ok(None) if no tokenizer.json.

pub fn from_tokenizer_json(raw: &str) -> Result<Self, EngineError>

pub fn encode(&self, text: &str) -> Result<Vec<u32>, EngineError>

Encode text → token ids (no extra specials; chat template is already in text).

pub fn decode(&self, ids: &[u32]) -> String

Decode ids → UTF-8, skipping special tokens by default.

pub fn decode_opts(&self, ids: &[u32], skip_special: bool) -> String

pub fn is_stop(&self, id: u32) -> bool

pub fn stop_ids(&self) -> &[u32]

pub fn has_token(&self, token: &str) -> bool

pub fn chat_family_hint(&self) -> Option<&'static str>

Prefer tokenizer specials over a possibly-wrong Session family (serve used to hardcode gemma even for Qwen bundles).

Trait Implementations

impl Clone for BundleTokenizer

fn clone(&self) -> BundleTokenizer

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for BundleTokenizer

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.