Struct HuggingFaceTokenizer 

pub struct HuggingFaceTokenizer { /* private fields */ }

HuggingFace tokenizer wrapper

Implementations

impl HuggingFaceTokenizer

pub fn from_file(file_path: &str) -> Result<Self>

Create a tokenizer from a HuggingFace tokenizer JSON file

pub fn from_file_with_chat_template( file_path: &str, chat_template_path: Option<&str>, ) -> Result<Self>

Create a tokenizer from a HuggingFace tokenizer JSON file with an optional chat template

pub fn from_tokenizer(tokenizer: HfTokenizer) -> Self

Create from an existing HuggingFace tokenizer

pub fn set_chat_template(&mut self, template: String)

Set or override the chat template

pub fn chat_template_content_format(&self) -> ChatTemplateContentFormat

Get the content format expected by the chat template

pub fn apply_chat_template( &self, messages: &[Value], params: ChatTemplateParams<'_>, ) -> Result<String>

Apply chat template if available

Takes transformed JSON Values (already transformed based on content format)

Trait Implementations

impl Decoder for HuggingFaceTokenizer

fn decode( &self, token_ids: &[TokenIdType], skip_special_tokens: bool, ) -> Result<String>

impl Encoder for HuggingFaceTokenizer

fn encode(&self, input: &str, add_special_tokens: bool) -> Result<Encoding>

fn encode_batch( &self, inputs: &[&str], add_special_tokens: bool, ) -> Result<Vec<Encoding>>

impl Tokenizer for HuggingFaceTokenizer

fn vocab_size(&self) -> usize

fn get_special_tokens(&self) -> &SpecialTokens

fn token_to_id(&self, token: &str) -> Option<TokenIdType>

fn id_to_token(&self, id: TokenIdType) -> Option<String>

fn as_any(&self) -> &dyn Any

Enable downcasting to concrete types