Enum TokenizerWrapper 

pub enum TokenizerWrapper {
    WordPiece(WordPieceTokenizer),
    BPE(BPETokenizer),
    Unigram(UnigramTokenizer),
    Char(CharTokenizer),
    HuggingFace(TokenizerImpl),
}

Variants

WordPiece(WordPieceTokenizer)

BPE(BPETokenizer)

Unigram(UnigramTokenizer)

Char(CharTokenizer)

HuggingFace(TokenizerImpl)

Implementations

impl TokenizerWrapper

pub fn from_pretrained<P: AsRef<Path>>(model_name_or_path: P) -> Result<Self>

Load a tokenizer from a pretrained model or path

pub fn save_pretrained<P: AsRef<Path>>(&self, path: P) -> Result<()>

Save the tokenizer to a directory path

Trait Implementations

impl Clone for TokenizerWrapper

fn clone(&self) -> TokenizerWrapper

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for TokenizerWrapper

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Tokenizer for TokenizerWrapper

fn encode(&self, text: &str) -> Result<TokenizedInput>

Encodes a single text string into tokens.

fn encode_pair(&self, text: &str, text2: &str) -> Result<TokenizedInput>

Encodes a pair of texts for sequence-pair tasks.

fn decode(&self, ids: &[u32]) -> Result<String>

Decodes token IDs back into text.

fn vocab_size(&self) -> usize

Returns the size of the tokenizer’s vocabulary.

fn get_vocab(&self) -> HashMap<String, u32>

Returns a copy of the vocabulary as a mapping from tokens to IDs.

fn token_to_id(&self, token: &str) -> Option<u32>

Converts a token string to its corresponding ID.

fn id_to_token(&self, id: u32) -> Option<String>

Converts a token ID to its corresponding token string.