fastokens 0.3.1

Fast Tokenizer
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126

use serde_json::{Value, json};
use tokenizers::pre_tokenizers::byte_level::ByteLevel;

use super::Tokenizer;

const REGULAR_TOKEN: &str = "<|regular|>";
const SPECIAL_TOKEN: &str = "<|special|>";

fn tokenizer_json(fused: bool, with_added_tokens: bool) -> Value {
    let mut alphabet: Vec<char> = ByteLevel::alphabet().into_iter().collect();
    alphabet.sort_unstable();
    let vocab = alphabet
        .into_iter()
        .enumerate()
        .map(|(id, token)| (token.to_string(), json!(id)))
        .collect::<serde_json::Map<_, _>>();

    let pre_tokenizer = if fused {
        json!({
            "type": "Sequence",
            "pretokenizers": [
                {
                    "type": "Split",
                    "pattern": {"Regex": "\\S+|\\s+"},
                    "behavior": "Isolated",
                    "invert": false
                },
                {
                    "type": "ByteLevel",
                    "add_prefix_space": false,
                    "trim_offsets": true,
                    "use_regex": false
                }
            ]
        })
    } else {
        json!({
            "type": "ByteLevel",
            "add_prefix_space": false,
            "trim_offsets": true,
            "use_regex": true
        })
    };

    let added_tokens = with_added_tokens.then(|| {
        json!([
            {
                "id": 256,
                "content": REGULAR_TOKEN,
                "single_word": false,
                "lstrip": false,
                "rstrip": false,
                "normalized": true,
                "special": false
            },
            {
                "id": 257,
                "content": SPECIAL_TOKEN,
                "single_word": false,
                "lstrip": false,
                "rstrip": false,
                "normalized": false,
                "special": true
            }
        ])
    });

    json!({
        "version": "1.0",
        "added_tokens": added_tokens.unwrap_or_else(|| json!([])),
        "normalizer": {"type": "NFC"},
        "pre_tokenizer": pre_tokenizer,
        "post_processor": {
            "type": "ByteLevel",
            "add_prefix_space": false,
            "trim_offsets": true,
            "use_regex": true
        },
        "decoder": {
            "type": "ByteLevel",
            "add_prefix_space": false,
            "trim_offsets": true,
            "use_regex": true
        },
        "model": {
            "type": "BPE",
            "dropout": null,
            "unk_token": null,
            "continuing_subword_prefix": null,
            "end_of_word_suffix": null,
            "fuse_unk": false,
            "byte_fallback": false,
            "ignore_merges": false,
            "vocab": vocab,
            "merges": []
        }
    })
}

#[test]
fn encode_ordinary_matches_added_empty_pipeline() {
    for fused in [false, true] {
        let tokenizer = Tokenizer::from_json(tokenizer_json(fused, true)).expect("build tokenizer");
        let added_empty =
            Tokenizer::from_json(tokenizer_json(fused, false)).expect("build added-empty");

        assert_eq!(tokenizer.split_only.is_some(), fused);
        assert_eq!(tokenizer.encode(REGULAR_TOKEN).unwrap(), vec![256]);
        assert_eq!(tokenizer.encode(SPECIAL_TOKEN).unwrap(), vec![257]);

        for text in [
            "",
            "hello",
            "Cafe\u{301}",
            REGULAR_TOKEN,
            SPECIAL_TOKEN,
            "hello <|regular|> Cafe\u{301} <|special|> tail",
        ] {
            assert_eq!(
                tokenizer.encode_ordinary(text).unwrap(),
                added_empty.encode(text).unwrap(),
                "fused={fused}, text={text:?}",
            );
        }
    }
}