bert_tokenizer 0.1.3

This crate is a Rust port of Google's BERT WordPiece tokenizer.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69

use bert_tokenizer::{FullTokenizer, Tokenizer};

#[test]
fn test_full_tokenizer_uncased() {
    let vocab_file = "tests/uncased_L-12_H-768_A-12/vocab.txt";
    let do_lower_case = true;
    let tokenizer = FullTokenizer::new()
        .vocab_from_file(vocab_file)
        .do_lower_case(do_lower_case)
        .build();
    let tokens = tokenizer.tokenize("Hello world!");
    assert_eq!(tokens, vec!["hello", "world", "!"]);
    let ids = tokenizer.convert_tokens_to_ids(&tokens);
    let tokens = tokenizer.convert_ids_to_tokens(&ids);
    assert_eq!(tokens, vec!["hello", "world", "!"]);
    let text = tokenizer.convert_tokens_to_string(&tokens);
    assert_eq!(text, "hello world !");
}

#[test]
fn test_full_tokenizer_cased() {
    let vocab_file = "tests/cased_L-12_H-768_A-12/vocab.txt";
    let do_lower_case = false;
    let tokenizer = FullTokenizer::new()
        .vocab_from_file(vocab_file)
        .do_lower_case(do_lower_case)
        .build();
    let tokens = tokenizer.tokenize("Hello world!");
    assert_eq!(tokens, vec!["Hello", "world", "!"]);
    let ids = tokenizer.convert_tokens_to_ids(&tokens);
    let tokens = tokenizer.convert_ids_to_tokens(&ids);
    assert_eq!(tokens, vec!["Hello", "world", "!"]);
    let text = tokenizer.convert_tokens_to_string(&tokens);
    assert_eq!(text, "Hello world !");
}

#[test]
fn test_full_tokenizer_cased_strip_accents() {
    let vocab_file = "tests/cased_L-12_H-768_A-12/vocab.txt";
    let do_lower_case = false;
    let do_strip_accents = true;
    let tokenizer = FullTokenizer::new()
        .vocab_from_file(vocab_file)
        .do_lower_case(do_lower_case)
        .do_strip_accents(do_strip_accents)
        .build();
    let tokens = tokenizer.tokenize("Hello wörld!");
    assert_eq!(tokens, vec!["Hello", "world", "!"]);
    let ids = tokenizer.convert_tokens_to_ids(&tokens);
    let tokens = tokenizer.convert_ids_to_tokens(&ids);
    assert_eq!(tokens, vec!["Hello", "world", "!"]);
    let text = tokenizer.convert_tokens_to_string(&tokens);
    assert_eq!(text, "Hello world !");
}

#[test]
fn test_full_tokenizer_cased_no_strip_accents() {
    let vocab_file = "tests/cased_L-12_H-768_A-12/vocab.txt";
    let tokenizer = FullTokenizer::new()
        .vocab_from_file(vocab_file)
        .build();
    let tokens = tokenizer.tokenize("Hello wörld!");
    assert_eq!(tokens, vec!["Hello", "w", "##ö", "##rl", "##d", "!"]);
    let ids = tokenizer.convert_tokens_to_ids(&tokens);
    let tokens = tokenizer.convert_ids_to_tokens(&ids);
    assert_eq!(tokens,  vec!["Hello", "w", "##ö", "##rl", "##d", "!"]);
    let text = tokenizer.convert_tokens_to_string(&tokens);
    assert_eq!(text, "Hello wörld !");
}