Struct MockTokenizer 

pub struct MockTokenizer { /* private fields */ }

Mock tokenizer for testing purposes

Implementations

impl MockTokenizer

pub fn new() -> Self

Trait Implementations

impl Decoder for MockTokenizer

fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result<String>

impl Default for MockTokenizer

fn default() -> Self

Returns the “default value” for a type.

impl Encoder for MockTokenizer

fn encode(&self, input: &str, _add_special_tokens: bool) -> Result<Encoding>

fn encode_batch( &self, inputs: &[&str], add_special_tokens: bool, ) -> Result<Vec<Encoding>>

impl Tokenizer for MockTokenizer

fn vocab_size(&self) -> usize

fn get_special_tokens(&self) -> &SpecialTokens

fn token_to_id(&self, token: &str) -> Option<u32>

fn id_to_token(&self, id: u32) -> Option<String>

fn as_any(&self) -> &dyn Any

Enable downcasting to concrete types