Struct HuggingFaceTokenizer 

pub struct HuggingFaceTokenizer { /* private fields */ }

HuggingFace tokenizer wrapper

Implementations

impl HuggingFaceTokenizer

pub async fn new(tokenizer: HfTokenizer) -> Result<Self>

Create new HuggingFace tokenizer

pub async fn from_file(path: &str) -> Result<Self>

Create from file path

pub async fn from_pretrained( repo_id: &str, _revision: Option<&str>, ) -> Result<Self>

Create from HuggingFace Hub

Trait Implementations

impl IncrementalTokenizer for HuggingFaceTokenizer

type State = IncrementalState

Tokenizer state for incremental operations

fn create_state(&self) -> Self::State

Create initial state for incremental decoding

fn decode_incremental_with_state( &self, state: &mut Self::State, token: TokenId, ) -> Result<String>

Add token to state and get incremental text

fn reset_state(&self, state: &mut Self::State)

Reset state to initial condition

fn get_decoded_text(&self, state: &Self::State) -> String

Get all decoded text from current state

impl Tokenizer for HuggingFaceTokenizer

fn encode(&self, text: &str, add_special: bool) -> Result<Vec<TokenId>>

Encode text to token IDs

fn decode(&self, tokens: &[TokenId], skip_special: bool) -> Result<String>

Decode token IDs to text

fn decode_incremental(&self, prev: &[TokenId], next: TokenId) -> Result<String>

Incremental decode: given previous tokens and new token, return only the new text This is crucial for streaming applications to avoid re-decoding all tokens

fn vocab_size(&self) -> usize

Get vocabulary size

fn special_tokens(&self) -> &SpecialTokens

Get special tokens configuration

fn token_id(&self, text: &str) -> Option<TokenId>

Get token ID for a specific text (if exists in vocabulary)

fn token_text(&self, _token_id: TokenId) -> Option<&str>

Get text for a specific token ID

fn apply_chat_template(&self, messages: &[ChatMessage]) -> Result<String>

Apply chat template if supported

fn info(&self) -> TokenizerInfo

Get tokenizer information

fn is_special_token(&self, token_id: TokenId) -> bool

Check if token is a special token