mako 0.3.0

Mako is the main Sidekick AI data processing library
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64

// TOKENIZATION TESTS
use crate::tokenization::*;

#[test]
fn tokenize_alphabet() {
    let letters: Vec<String> = vec!["h", "e", "l", "l", "o"].iter().map(|t| {(*t).to_string()}).collect();
    let tokenizer = AlphabetTokenizer::load();
    assert_eq!(tokenizer.tokenize("hello"), letters);
}

#[test]
fn tokenize_spaces() {
    let tokens: Vec<String> = vec!["hello", "how", "are", "you"].iter().map(|t| {(*t).to_string()}).collect();
    let tokenizer = WhitespaceTokenizer::load();
    assert_eq!(tokenizer.tokenize("hello how are you"), tokens);
}

#[test]
fn tokenize_bpe() {
    let tokens: Vec<String> = vec!["hello", ",", " ", "how", " ", "are", " ", "you"].iter().map(|str| {str.to_string()}).collect();
    let tokenizer = BPETokenizer::load();
    assert_eq!(tokenizer.batch_tokenize(vec!["hello, how are you".to_string()]), vec![tokens.clone()]);
    assert_eq!(tokenizer.tokenize("hello, how are you"), tokens);
}

#[test]
fn tokenize_wordpiece() {
    let tokens: Vec<String> = vec!["hello", ",", "how", "are", "you"].iter().map(|str| {str.to_string()}).collect();
    let tokenizer = WordpieceTokenizer::load();
    assert_eq!(tokenizer.batch_tokenize(vec!["hello, how are you".to_string()]), vec![tokens.clone()]);
    assert_eq!(tokenizer.tokenize("hello, how are you"), tokens);
}

#[test]
fn untokenize_bpe() {
    let sentence = "hello, how are you?";
    let tokenizer = BPETokenizer::load();
    let tokens = tokenizer.tokenize(sentence);
    assert_eq!(tokenizer.untokenize(tokens), sentence.to_string());
}

#[test]
fn untokenize_wordpiece() {
    let sentence = "hello, how are you?";
    let tokenizer = WordpieceTokenizer::load();
    let tokens = tokenizer.tokenize(sentence);
    assert_eq!(tokenizer.untokenize(tokens), sentence.to_string());
}

#[test]
fn untokenize_alphabet() {
    let sentence = "hello, how are you?";
    let tokenizer = AlphabetTokenizer::load();
    let tokens = tokenizer.tokenize(sentence);
    assert_eq!(tokenizer.untokenize(tokens), sentence.to_string());
}

#[test]
fn untokenize_spaces() {
    let sentence = "hello, how are you?";
    let tokenizer = WhitespaceTokenizer::load();
    let tokens = tokenizer.tokenize(sentence);
    assert_eq!(tokenizer.untokenize(tokens), sentence.to_string());
}