Struct MockTokenizer 

pub struct MockTokenizer { /* private fields */ }

Mock tokenizer for testing purposes

Implementations

impl MockTokenizer

pub fn new() -> Self

Trait Implementations

impl Decoder for MockTokenizer

fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result<String>

fn decode_step( &self, token_id: TokenIdType, ids: &mut Vec<TokenIdType>, prefix: &mut String, prefix_index: &mut usize, skip_special_tokens: bool, ) -> Result<Option<String>>

Incremental decode step — called once per generated token.

impl Default for MockTokenizer

fn default() -> Self

Returns the “default value” for a type.

impl Encoder for MockTokenizer

fn encode(&self, input: &str, _add_special_tokens: bool) -> Result<Encoding>

fn encode_batch( &self, inputs: &[&str], add_special_tokens: bool, ) -> Result<Vec<Encoding>>

impl Tokenizer for MockTokenizer

fn vocab_size(&self) -> usize

fn get_special_tokens(&self) -> &SpecialTokens

fn token_to_id(&self, token: &str) -> Option<u32>

fn id_to_token(&self, id: u32) -> Option<String>

fn as_any(&self) -> &dyn Any

Enable downcasting to concrete types

fn apply_chat_template( &self, _messages: &[Value], _params: ChatTemplateParams<'_>, ) -> Result<String>

Apply chat template to messages. Default returns an error for tokenizers without template support.

fn chat_template_content_format(&self) -> ChatTemplateContentFormat

Get the content format expected by the chat template.

fn thinking_toggle(&self) -> ThinkingToggle

Get the thinking toggle support for this template.

fn thinking_key_name(&self) -> Option<ThinkingKeyName>

The variable name the template uses for the thinking toggle.

fn think_in_prefill(&self) -> bool

Whether the template injects <think> in the generation prompt.

fn set_chat_template(&mut self, _template: String) -> Result<()>

Set or override the chat template.