Function bert_tokenizer::load_vocab
source · pub fn load_vocab(vocab_file: &str) -> Vocab
Expand description
Load a vocabulary from a vocabulary file.
Not recommended to use this function directly, use FullTokenizerBuilder::vocab_from_file
instead.
Example
use bert_tokenizer::{load_vocab, Vocab};
let vocab_file = "tests/cased_L-12_H-768_A-12/vocab.txt";
let vocab: Vocab = load_vocab(vocab_file);
assert_eq!(vocab.len(), 28996);
let vocab_file = "tests/uncased_L-12_H-768_A-12/vocab.txt";
let vocab: Vocab = load_vocab(vocab_file);
assert_eq!(vocab.len(), 30522);