Struct MockTokenizer 

pub struct MockTokenizer { /* private fields */ }

Mock tokenizer: splits on whitespace, assigns sequential token IDs. EOS token is vocab_size - 1.

Implementations

impl MockTokenizer

pub fn new(vocab_size: usize) -> Self

Trait Implementations

impl Tokenizer for MockTokenizer

fn encode(&self, text: &str, add_special: bool) -> Result<Vec<TokenId>>

Encode text to token IDs

fn decode(&self, tokens: &[TokenId], _skip_special: bool) -> Result<String>

Decode token IDs to text

fn decode_incremental(&self, _prev: &[TokenId], next: TokenId) -> Result<String>

Incremental decode: given previous tokens and new token, return only the new text This is crucial for streaming applications to avoid re-decoding all tokens

fn vocab_size(&self) -> usize

Get vocabulary size

fn special_tokens(&self) -> &SpecialTokens

Get special tokens configuration

fn token_id(&self, _text: &str) -> Option<TokenId>

Get token ID for a specific text (if exists in vocabulary)

fn token_text(&self, _token_id: TokenId) -> Option<&str>

Get text for a specific token ID

fn info(&self) -> TokenizerInfo

Get tokenizer information

fn is_special_token(&self, token_id: TokenId) -> bool

Check if token is a special token

fn apply_chat_template( &self, messages: &[ChatMessage], ) -> Result<String, FerrumError>

Apply chat template if supported