Struct StubTokenizer 

pub struct StubTokenizer { /* private fields */ }

Stub tokenizer for testing

Implementations

impl StubTokenizer

pub fn new(vocab_size: usize) -> Self

Create a new stub tokenizer

Trait Implementations

impl Debug for StubTokenizer

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Tokenizer for StubTokenizer

fn encode(&self, text: &str, _add_special: bool) -> Result<Vec<TokenId>>

Encode text to token IDs

fn decode(&self, tokens: &[TokenId], _skip_special: bool) -> Result<String>

Decode token IDs to text

fn decode_incremental(&self, _prev: &[TokenId], next: TokenId) -> Result<String>

Incremental decode: given previous tokens and new token, return only the new text This is crucial for streaming applications to avoid re-decoding all tokens

fn vocab_size(&self) -> usize

Get vocabulary size

fn special_tokens(&self) -> &SpecialTokens

Get special tokens configuration

fn token_id(&self, _text: &str) -> Option<TokenId>

Get token ID for a specific text (if exists in vocabulary)

fn token_text(&self, _token_id: TokenId) -> Option<&str>

Get text for a specific token ID

fn info(&self) -> TokenizerInfo

Get tokenizer information

fn token_bytes(&self, token_id: TokenId) -> Option<Vec<u8>>

Return the context-free byte surface represented by one vocabulary token.

fn is_special_token(&self, token_id: TokenId) -> bool

Check if token is a special token

fn apply_chat_template( &self, messages: &[ChatMessage], ) -> Result<String, FerrumError>

Apply chat template if supported