Struct kalosm_sample::FasterHuggingFaceTokenizer

pub struct FasterHuggingFaceTokenizer { /* private fields */ }

A tokenizer that uses the HuggingFace tokenizer with a cache for single tokens.

Implementations

impl FasterHuggingFaceTokenizer

pub fn new(tokenizer: Tokenizer) -> Self

Create a new FasterHuggingFaceTokenizer from a tokenizers::Tokenizer.

pub fn tokenizer(&self) -> &Tokenizer

Get the inner tokenizer.

pub fn tokenizer_mut(&mut self) -> &mut Tokenizer

Get the inner tokenizer mutably.

pub fn into_tokenizer(self) -> Tokenizer

Consume the FasterHuggingFaceTokenizer and return the inner tokenizer.

Trait Implementations

impl Tokenizer for FasterHuggingFaceTokenizer

fn encode(&self, text: &str) -> Result<Vec<u32>>

Encode a string into a list of token ids.

fn decode(&self, ids: &[u32]) -> Result<Cow<'_, str>>

Decode a list of token ids into a string.

fn decode_batch(&self, ids: &[&[u32]]) -> Result<Vec<Cow<'_, str>>>

Decode a list of a list of token ids into a string.

fn get_all_tokens(&self) -> Result<Cow<'_, [u32]>>

Get all possible tokens.

fn encode_batch(&self, text: &[&str]) -> Result<Vec<Vec<u32>>>

Encode a list of strings into a list of token ids.