ffbpe 0.1.8

Unicode-aware, streaming BPE training and tiktoken-compatible encoding
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51

# %%
from ffbpe import BpeTrainer, PreTokenizer, train_bpe
pre = PreTokenizer(["<|endoftext|>"])
words = pre.get_words_from_file("fixtures/tinystories_sample_5M.txt", chunk_size=64 * 1024)

# %%

bpe = BpeTrainer(["<|endoftext|>"], unit="unicode")
assert bpe.vocab_size == 257
bpe.add_words(words)
bpe.train(500)
assert bpe.vocab_size == 500

# %%
vocab = bpe.vocab
assert len(vocab) == 500

# %%
bpe.save("test")

# %%
from ffbpe import BpeEncoder
import numpy as np
encoder = BpeEncoder.load("test", unit="unicode")
a = encoder.encode_to_numpy("Hello, world!")
print(a)
assert isinstance(a, np.ndarray)
assert a.tolist() == [73, 102, 293, 112, 45, 261, 304, 341, 34]
b = encoder.encode_file("fixtures/tinystories_sample_5M.txt", 100)
print(len(b))
assert len(b) == 2184799
s = encoder.decode(b)
# print(s[:100])
with open("fixtures/tinystories_sample_5M.txt", "rb") as f:
  original = f.read().decode("utf-8")
assert s == original

# %%
encoder = BpeEncoder.load("test", unit="unicode", special_tokens=[])
a1 = encoder.encode_to_numpy("<|endoftext|>")
print(a1)
assert a1.tolist() == [61, 125, 355, 112, 103, 117, 102, 121, 117, 125, 63]

# %%
model = train_bpe(
  ["hello world", "hello tokenizer"],
  vocab_size=280,
  special_tokens=["<|endoftext|>"],
)
ids = model.encode("hello world")
assert model.decode(ids) == "hello world"