tokenizers 0.22.2

Provides an implementation of today's most used tokenizers, with a focus on performances and versatility.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57

use tokenizers::decoders::wordpiece::WordPiece as WordPieceDecoder;
use tokenizers::models::bpe::BPE;
use tokenizers::models::wordpiece::WordPiece;
use tokenizers::normalizers::bert::BertNormalizer;
use tokenizers::pre_tokenizers::bert::BertPreTokenizer;
use tokenizers::pre_tokenizers::byte_level::ByteLevel;
use tokenizers::processors::bert::BertProcessing;
use tokenizers::tokenizer::{Model, Tokenizer};

#[allow(dead_code)]
pub fn get_empty() -> Tokenizer {
    Tokenizer::new(BPE::default())
}

#[allow(dead_code)]
pub fn get_byte_level_bpe() -> BPE {
    BPE::from_file("data/gpt2-vocab.json", "data/gpt2-merges.txt")
        .build()
        .expect("Files not found, run `make test` to download these files")
}

#[allow(dead_code)]
pub fn get_byte_level(add_prefix_space: bool, trim_offsets: bool) -> Tokenizer {
    let mut tokenizer = Tokenizer::new(get_byte_level_bpe());
    tokenizer
        .with_pre_tokenizer(Some(
            ByteLevel::default().add_prefix_space(add_prefix_space),
        ))
        .with_decoder(Some(ByteLevel::default()))
        .with_post_processor(Some(ByteLevel::default().trim_offsets(trim_offsets)));

    tokenizer
}

#[allow(dead_code)]
pub fn get_bert_wordpiece() -> WordPiece {
    WordPiece::from_file("data/bert-base-uncased-vocab.txt")
        .build()
        .expect("Files not found, run `make test` to download these files")
}

#[allow(dead_code)]
pub fn get_bert() -> Tokenizer {
    let mut tokenizer = Tokenizer::new(get_bert_wordpiece());
    let sep = tokenizer.get_model().token_to_id("[SEP]").unwrap();
    let cls = tokenizer.get_model().token_to_id("[CLS]").unwrap();
    tokenizer
        .with_normalizer(Some(BertNormalizer::default()))
        .with_pre_tokenizer(Some(BertPreTokenizer))
        .with_decoder(Some(WordPieceDecoder::default()))
        .with_post_processor(Some(BertProcessing::new(
            (String::from("[SEP]"), sep),
            (String::from("[CLS]"), cls),
        )));

    tokenizer
}