Struct kalosm_sample::DynTokenizer

pub struct DynTokenizer { /* private fields */ }

A type erased wrapper for a tokenizer.

Implementations

impl DynTokenizer

pub fn new<T: Tokenizer + Send + Sync + 'static>(tokenizer: T) -> Self

Create a new DynTokenizer from a Tokenizer.

Trait Implementations

impl From<Arc<dyn Tokenizer + Sync + Send>> for DynTokenizer

fn from(tokenizer: Arc<dyn Tokenizer + Send + Sync>) -> Self

Converts to this type from the input type.

impl From<Tokenizer> for DynTokenizer

fn from(tokenizer: Tokenizer) -> Self

Converts to this type from the input type.

impl<M, N, PT, PP, D> From<TokenizerImpl<M, N, PT, PP, D>> for DynTokenizer

where M: Model + Send + Sync + 'static, N: Normalizer + Send + Sync + 'static, PT: PreTokenizer + Send + Sync + 'static, PP: PostProcessor + Send + Sync + 'static, D: Decoder + Send + Sync + 'static,

fn from(tokenizer: TokenizerImpl<M, N, PT, PP, D>) -> Self

Converts to this type from the input type.

impl Tokenizer for DynTokenizer

fn encode(&self, text: &str, special_tokens: bool) -> Result<Vec<u32>>

Encode a string into a list of token ids.

fn decode(&self, ids: &[u32]) -> Result<Cow<'_, str>>

Decode a list of token ids into a string.

fn get_all_tokens(&self) -> Result<Cow<'_, [u32]>>

Get all possible tokens.

fn encode_batch( &self, text: &[&str], add_special_tokens: bool ) -> Result<Vec<Vec<u32>>>

Encode a list of strings into a list of token ids.

fn decode_batch(&self, ids: &[&[u32]]) -> Result<Vec<Cow<'_, str>>>

Decode a list of a list of token ids into a string.