Trait kalosm_sample::Tokenizer

pub trait Tokenizer {
    // Required methods
    fn encode(&self, text: &str, add_special_tokens: bool) -> Result<Vec<u32>>;
    fn decode(&self, ids: &[u32]) -> Result<Cow<'_, str>>;
    fn get_all_tokens(&self) -> Result<Cow<'_, [u32]>>;

    // Provided methods
    fn encode_batch(
        &self,
        text: &[&str],
        add_special_tokens: bool
    ) -> Result<Vec<Vec<u32>>> { ... }
    fn decode_batch(&self, ids: &[&[u32]]) -> Result<Vec<Cow<'_, str>>> { ... }
}

A tokenizer is a type that can decode a list of token ids into a string.

Required Methods

fn encode(&self, text: &str, add_special_tokens: bool) -> Result<Vec<u32>>

Encode a string into a list of token ids.

fn decode(&self, ids: &[u32]) -> Result<Cow<'_, str>>

Decode a list of token ids into a string.

fn get_all_tokens(&self) -> Result<Cow<'_, [u32]>>

Get all possible tokens.

Provided Methods

fn encode_batch( &self, text: &[&str], add_special_tokens: bool ) -> Result<Vec<Vec<u32>>>

Encode a list of strings into a list of token ids.

fn decode_batch(&self, ids: &[&[u32]]) -> Result<Vec<Cow<'_, str>>>

Decode a list of a list of token ids into a string.

Implementations on Foreign Types

impl Tokenizer for Tokenizer

fn encode(&self, text: &str, special_tokens: bool) -> Result<Vec<u32>>

fn decode(&self, ids: &[u32]) -> Result<Cow<'_, str>>

fn get_all_tokens(&self) -> Result<Cow<'_, [u32]>>

impl<M, N, PT, PP, D> Tokenizer for TokenizerImpl<M, N, PT, PP, D>

where M: Model, N: Normalizer, PT: PreTokenizer, PP: PostProcessor, D: Decoder,

fn encode(&self, text: &str, special_tokens: bool) -> Result<Vec<u32>>

fn decode(&self, ids: &[u32]) -> Result<Cow<'_, str>>

fn get_all_tokens(&self) -> Result<Cow<'_, [u32]>>

Implementors

impl Tokenizer for DynTokenizer

impl Tokenizer for FasterHuggingFaceTokenizer