Function bert_tokenizer::load_vocab

pub fn load_vocab(vocab_file: &str) -> Vocab

Load a vocabulary from a vocabulary file. Not recommended to use this function directly, use FullTokenizerBuilder::vocab_from_file instead.

Example

use bert_tokenizer::{load_vocab, Vocab};
let vocab_file = "tests/cased_L-12_H-768_A-12/vocab.txt";
let vocab: Vocab = load_vocab(vocab_file);
assert_eq!(vocab.len(), 28996);
let vocab_file = "tests/uncased_L-12_H-768_A-12/vocab.txt";
let vocab: Vocab = load_vocab(vocab_file);
assert_eq!(vocab.len(), 30522);