pub trait Tokenizer: Send + Sync {
// Required methods
fn encode(&self, text: &str) -> Vec<u32>;
fn token_bytes(&self, token: u32) -> Vec<u8> ⓘ;
fn vocab_size(&self) -> u32;
fn special_tokens(&self) -> Vec<(String, u32)>;
fn bos_token(&self) -> Option<u32>;
fn eos_token(&self) -> u32;
fn chat_template_jinja(&self) -> String;
// Provided methods
fn chat_template_named(&self, _name: &str) -> Option<String> { ... }
fn encode_plain(&self, _text: &str) -> Option<Vec<u32>> { ... }
fn encode_pieces(&self, _pieces: &[(&str, bool)]) -> Option<Vec<u32>> { ... }
}Required Methods§
fn encode(&self, text: &str) -> Vec<u32>
fn token_bytes(&self, token: u32) -> Vec<u8> ⓘ
fn vocab_size(&self) -> u32
fn special_tokens(&self) -> Vec<(String, u32)>
fn bos_token(&self) -> Option<u32>
fn eos_token(&self) -> u32
fn chat_template_jinja(&self) -> String
Provided Methods§
fn chat_template_named(&self, _name: &str) -> Option<String>
fn encode_plain(&self, _text: &str) -> Option<Vec<u32>>
fn encode_pieces(&self, _pieces: &[(&str, bool)]) -> Option<Vec<u32>>
Dyn Compatibility§
This trait is dyn compatible.
In older versions of Rust, dyn compatibility was called "object safety".