Struct WordPieceTokenizer 

pub struct WordPieceTokenizer { /* private fields */ }

Implementations

impl WordPieceTokenizer

pub fn from_file(path: &Path) -> Result<Self>

Load vocabulary from a standard vocab.txt file (one token per line).

pub fn from_vocab_str(vocab_str: &str) -> Result<Self>

Build tokenizer from vocabulary string (one token per line).

pub fn encode(&self, text: &str, max_len: usize) -> TokenizedInput

Encode text into token IDs with [CLS] prefix and [SEP] suffix.

pub fn pad_id(&self) -> i32

pub fn vocab_size(&self) -> usize