Struct HuggingFaceTokenizer 

pub struct HuggingFaceTokenizer { /* private fields */ }

HuggingFace tokenizer wrapper

Implementations

impl HuggingFaceTokenizer

pub async fn new(tokenizer: HfTokenizer) -> Result<Self>

Create new HuggingFace tokenizer

pub async fn from_file(path: &str) -> Result<Self>

Create from file path. Special tokens are resolved from the sibling generation_config.json / tokenizer_config.json when present; vocab name probing is only the fallback for bare tokenizer.json files.

pub async fn from_source_bytes( tokenizer_json: &[u8], tokenizer_config_json: Option<&[u8]>, generation_config_json: Option<&[u8]>, ) -> Result<Self>

Construct from the immutable bytes retained by product source resolution. Config bytes are parsed from the same source bundle so special-token semantics cannot drift through a second filesystem read.

pub async fn from_pretrained( repo_id: &str, _revision: Option<&str>, ) -> Result<Self>

Create from HuggingFace Hub

Trait Implementations

impl IncrementalTokenizer for HuggingFaceTokenizer

type State = IncrementalState

Tokenizer state for incremental operations

fn create_state(&self) -> Self::State

Create initial state for incremental decoding

fn decode_incremental_with_state( &self, state: &mut Self::State, token: TokenId, ) -> Result<String>

Add token to state and get incremental text

fn reset_state(&self, state: &mut Self::State)

Reset state to initial condition

fn get_decoded_text(&self, state: &Self::State) -> String

Get all decoded text from current state

impl Tokenizer for HuggingFaceTokenizer

fn encode(&self, text: &str, add_special: bool) -> Result<Vec<TokenId>>

Encode text to token IDs

fn decode(&self, tokens: &[TokenId], skip_special: bool) -> Result<String>

Decode token IDs to text

fn decode_incremental(&self, prev: &[TokenId], next: TokenId) -> Result<String>

Incremental decode: given previous tokens and new token, return only the new text This is crucial for streaming applications to avoid re-decoding all tokens

fn vocab_size(&self) -> usize

Get vocabulary size

fn special_tokens(&self) -> &SpecialTokens

Get special tokens configuration

fn token_id(&self, text: &str) -> Option<TokenId>

Get token ID for a specific text (if exists in vocabulary)

fn token_text(&self, token_id: TokenId) -> Option<&str>

Get text for a specific token ID

fn token_bytes(&self, token_id: TokenId) -> Option<Vec<u8>>

Return the context-free byte surface represented by one vocabulary token.

fn apply_chat_template(&self, messages: &[ChatMessage]) -> Result<String>

Apply chat template if supported

fn info(&self) -> TokenizerInfo

Get tokenizer information

fn is_special_token(&self, token_id: TokenId) -> bool

Check if token is a special token