unitoken 0.1.3

Fast BPE tokenizer/trainer with a Rust core and Python bindings
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44

# %%
from uni_tokenizer import BpeTrainer, PreTokenizer
pre = PreTokenizer(["<|endoftext|>"])
words = pre.get_words_from_file("fixtures/tinystories_sample_5M.txt", chunk_size=64 * 1024)

# %%

bpe = BpeTrainer(["<|endoftext|>"], ch="char")
assert bpe.vocab_size == 257
bpe.add_words(words)
bpe.train(500)
assert bpe.vocab_size == 500

# %%
vocabs = dict(bpe.vocabs.items())
assert len(vocabs) == 500

# %%
bpe.save("test")

# %%
from uni_tokenizer import BpeEncoder
import numpy as np
encoder = BpeEncoder.load("test", ch="char")
a = encoder.encode_string("Hello, world!")
print(a)
assert isinstance(a, np.ndarray)
assert a.tolist() == [73, 102, 293, 112, 45, 261, 304, 341, 34]
b = encoder.encode_file("fixtures/tinystories_sample_5M.txt", 100)
print(len(b))
assert len(b) == 2184799
s = encoder.decode(b)
# print(s[:100])
with open("fixtures/tinystories_sample_5M.txt", "rb") as f:
  original = f.read().decode("utf-8")
assert s == original

# %%
encoder = BpeEncoder.load("test", ch="char", special_tokens=[])
a1 = encoder.encode_string("<|endoftext|>")
print(a1)
assert a1.tolist() == [61, 125, 355, 112, 103, 117, 102, 121, 117, 125, 63]

# %%