Struct Tokenizer 

pub struct Tokenizer(/* private fields */);

Main tokenizer wrapper that provides a unified interface for different tokenizer implementations

Implementations

impl Tokenizer

pub fn from_file(file_path: &str) -> Result<Tokenizer>

pub fn decode_stream( &self, prompt_token_ids: &[TokenIdType], skip_special_tokens: bool, ) -> DecodeStream

Create a stateful sequence object for decoding token_ids into text

Trait Implementations

impl Clone for Tokenizer

fn clone(&self) -> Tokenizer

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Deref for Tokenizer

type Target = Arc<dyn Tokenizer>

The resulting type after dereferencing.

fn deref(&self) -> &Self::Target

Dereferences the value.

impl<T> From<Arc<T>> for Tokenizer

where T: Tokenizer + 'static,

fn from(tokenizer: Arc<T>) -> Self

Converts to this type from the input type.

impl From<Arc<dyn Tokenizer>> for Tokenizer

fn from(tokenizer: Arc<dyn Tokenizer>) -> Self

Converts to this type from the input type.