Struct TokenizerEngine 

pub struct TokenizerEngine { /* private fields */ }

Implementations

impl TokenizerEngine

pub fn new(tokenizer: Tokenizer) -> Result<Self>

pub fn new_with_options( tokenizer: Tokenizer, options: EngineOptions, ) -> Result<Self>

pub fn tokenizer(&self) -> &Tokenizer

pub fn set_options(&mut self, options: EngineOptions) -> Result<()>

pub fn decode_id(&self, id: u32) -> Result<String>

pub fn encode_text(&self, text: &str) -> Result<Vec<u32>>

pub fn random_token( &self, rng: &mut SimpleRng, mode: RandomMode, ) -> Option<TokenPiece>

pub fn text_stream(&self, text: &str, repeat: TextRepeat) -> Result<TextStream>

Trait Implementations

impl Debug for TokenizerEngine

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.