Struct Tokenizer 

pub struct Tokenizer { /* private fields */ }

Implementations

impl Tokenizer

pub fn from_file<P: AsRef<Path>>(path: P) -> Result<Self>

pub fn from_pretrained(identifier: &str) -> Result<Self>

pub fn encode(&self, text: &str, add_special_tokens: bool) -> Result<Vec<u32>>

pub fn encode_batch( &self, texts: &[&str], add_special_tokens: bool, ) -> Result<Vec<Vec<u32>>>

pub fn decode(&self, ids: &[u32], skip_special_tokens: bool) -> Result<String>

pub fn vocab_size(&self) -> usize

pub fn pad_token_id(&self) -> u32

pub fn bos_token_id(&self) -> u32

pub fn eos_token_id(&self) -> u32

pub fn set_pad_token_id(&mut self, id: u32)

pub fn set_bos_token_id(&mut self, id: u32)

pub fn set_eos_token_id(&mut self, id: u32)