Struct HuggingFaceTokenizer 

pub struct HuggingFaceTokenizer { /* private fields */ }

HuggingFace tokenizer wrapper

Implementations

impl HuggingFaceTokenizer

pub fn from_file(file_path: &str) -> Result<Self>

Create a tokenizer from a HuggingFace tokenizer JSON file

pub fn from_file_with_chat_template( file_path: &str, chat_template_path: Option<&str>, ) -> Result<Self>

Create a tokenizer from a HuggingFace tokenizer JSON file with an optional chat template

pub fn from_tokenizer(tokenizer: HfTokenizer) -> Self

Create from an existing HuggingFace tokenizer

Trait Implementations

impl Decoder for HuggingFaceTokenizer

fn decode_step( &self, token_id: TokenIdType, ids: &mut Vec<TokenIdType>, prefix: &mut String, prefix_index: &mut usize, skip_special_tokens: bool, ) -> Result<Option<String>>

Native incremental decode using the HF step_decode_stream.

This delegates to the same algorithm the default trait method uses, but the two internal decode() calls go directly through the concrete TokenizerImpl rather than through dyn Decoder vtable dispatch.

fn decode( &self, token_ids: &[TokenIdType], skip_special_tokens: bool, ) -> Result<String>

impl Encoder for HuggingFaceTokenizer

fn encode(&self, input: &str, add_special_tokens: bool) -> Result<Encoding>

fn encode_batch( &self, inputs: &[&str], add_special_tokens: bool, ) -> Result<Vec<Encoding>>

impl Tokenizer for HuggingFaceTokenizer

fn vocab_size(&self) -> usize

fn get_special_tokens(&self) -> &SpecialTokens

fn token_to_id(&self, token: &str) -> Option<TokenIdType>

fn id_to_token(&self, id: TokenIdType) -> Option<String>

fn as_any(&self) -> &dyn Any

Enable downcasting to concrete types

fn apply_chat_template( &self, messages: &[Value], params: ChatTemplateParams<'_>, ) -> Result<String>

Apply chat template to messages. Default returns an error for tokenizers without template support.

fn chat_template_content_format(&self) -> ChatTemplateContentFormat

Get the content format expected by the chat template.

fn thinking_toggle(&self) -> ThinkingToggle

Get the thinking toggle support for this template.

fn thinking_key_name(&self) -> Option<ThinkingKeyName>

The variable name the template uses for the thinking toggle.

fn think_in_prefill(&self) -> bool

Whether the template injects <think> in the generation prompt.

fn set_chat_template(&mut self, template: String) -> Result<()>

Set or override the chat template.