Struct TokenizerHandle 

pub struct TokenizerHandle { /* private fields */ }

Owned tokenizer handle. Use load_tokenizer to construct.

Implementations

impl TokenizerHandle

pub fn from_raw(raw: RawTokenizer) -> Self

pub fn raw(&self) -> &RawTokenizer

pub fn raw_mut(&mut self) -> &mut RawTokenizer

pub fn encode(&self, text: &str, add_special: bool) -> Result<Vec<u32>>

Encode text and return token ids.

pub fn decode(&self, ids: &[u32], skip_special: bool) -> Result<String>

Decode ids back to a string.