Trait Tokenizer 

pub trait Tokenizer: Send + Sync {
    // Required methods
    fn encode(
        &self,
        text: &str,
        add_special: bool,
    ) -> Result<Vec<TokenId>, FerrumError>;
    fn decode(
        &self,
        tokens: &[TokenId],
        skip_special: bool,
    ) -> Result<String, FerrumError>;
    fn decode_incremental(
        &self,
        prev: &[TokenId],
        next: TokenId,
    ) -> Result<String, FerrumError>;
    fn vocab_size(&self) -> usize;
    fn special_tokens(&self) -> &SpecialTokens;
    fn token_id(&self, text: &str) -> Option<TokenId>;
    fn token_text(&self, token_id: TokenId) -> Option<&str>;
    fn info(&self) -> TokenizerInfo;

    // Provided methods
    fn token_bytes(&self, token_id: TokenId) -> Option<Vec<u8>> { ... }
    fn is_special_token(&self, token_id: TokenId) -> bool { ... }
    fn apply_chat_template(
        &self,
        messages: &[ChatMessage],
    ) -> Result<String, FerrumError> { ... }
}

Core tokenizer trait for encoding/decoding operations

Required Methods

fn encode( &self, text: &str, add_special: bool, ) -> Result<Vec<TokenId>, FerrumError>

Encode text to token IDs

fn decode( &self, tokens: &[TokenId], skip_special: bool, ) -> Result<String, FerrumError>

Decode token IDs to text

fn decode_incremental( &self, prev: &[TokenId], next: TokenId, ) -> Result<String, FerrumError>

Incremental decode: given previous tokens and new token, return only the new text This is crucial for streaming applications to avoid re-decoding all tokens

fn vocab_size(&self) -> usize

Get vocabulary size

fn special_tokens(&self) -> &SpecialTokens

Get special tokens configuration

fn token_id(&self, text: &str) -> Option<TokenId>

Get token ID for a specific text (if exists in vocabulary)

fn token_text(&self, token_id: TokenId) -> Option<&str>

Get text for a specific token ID

fn info(&self) -> TokenizerInfo

Get tokenizer information

Provided Methods

fn token_bytes(&self, token_id: TokenId) -> Option<Vec<u8>>

Return the context-free byte surface represented by one vocabulary token.

This is distinct from decoding a one-token sequence: byte-level BPE vocabularies may split one UTF-8 scalar across multiple tokens, so a string decoder must replace an incomplete fragment. Grammar/token-trie consumers need the original bytes instead. Tokenizers with a byte-level vocabulary should override this method.

fn is_special_token(&self, token_id: TokenId) -> bool

Check if token is a special token

fn apply_chat_template( &self, messages: &[ChatMessage], ) -> Result<String, FerrumError>

Apply chat template if supported

Dyn Compatibility

This trait is dyn compatible.

In older versions of Rust, dyn compatibility was called "object safety".

Implementations on Foreign Types

impl Tokenizer for HuggingFaceTokenizer

fn encode( &self, text: &str, add_special: bool, ) -> Result<Vec<TokenId>, FerrumError>

fn decode( &self, tokens: &[TokenId], skip_special: bool, ) -> Result<String, FerrumError>

fn decode_incremental( &self, prev: &[TokenId], next: TokenId, ) -> Result<String, FerrumError>

fn vocab_size(&self) -> usize

fn special_tokens(&self) -> &SpecialTokens

fn token_id(&self, text: &str) -> Option<TokenId>

fn token_text(&self, token_id: TokenId) -> Option<&str>

fn token_bytes(&self, token_id: TokenId) -> Option<Vec<u8>>

fn apply_chat_template( &self, messages: &[ChatMessage], ) -> Result<String, FerrumError>

fn info(&self) -> TokenizerInfo

Implementors

impl Tokenizer for StubTokenizer