unitoken 0.1.5

Fast BPE tokenizer/trainer with a Rust core and Python bindings
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90

from collections.abc import Sequence
from os import PathLike
from typing import Any, Literal, final

@final
class BigramCounter:
    def __new__(cls, /, pre_tokenizer: PreTokenizer) -> BigramCounter: ...
    def add_batch(self, /, texts: Sequence[str]) -> None: ...
    def add_source(self, /, source: Any, *, max_records: int = 4096, max_bytes: int = 67108864, prefetch: Literal[0, 1] = 1) -> None: ...
    def add_text(self, /, text: str) -> None: ...
    def items(self, /) -> list[tuple[str, int]]: ...
    def merge(self, /, other: BigramCounter) -> None: ...
    def selected(self, /, top_k: int, min_freq: int) -> list[str]: ...

@final
class BpeEncoderBase:
    def __new__(cls, /, format: str, unit: str, vocab: dict[Sequence[int], int] | None, merges: Sequence[tuple[Sequence[int], Sequence[int]]] | None, vocab_file: str | PathLike | None, merges_file: str | PathLike | None, special_tokens: Sequence[str] | None, pat_str: str | None = None) -> BpeEncoderBase: ...
    def decode(self, /, idxs: Any) -> str: ...
    def encode(self, /, text: str) -> list[int]: ...
    def encode_file(self, /, path: str | PathLike, num_chunks: int) -> Any: ...
    def encode_to_numpy(self, /, text: str) -> Any: ...
    def encode_word(self, /, word: str) -> list[int]: ...
    def encode_words(self, /, words: Sequence[str]) -> list[list[int]]: ...
    def pre_tokenizer(self, /) -> PreTokenizer: ...

@final
class BpeModelBase:
    def get_vocab(self, /) -> Vocabulary: ...
    def save_merges_txt(self, /, path: str | PathLike, format: str) -> None: ...
    def save_vocab(self, /, path: str | PathLike, format: str) -> None: ...
    @property
    def unit(self, /) -> str: ...

class BpeTrainerBase: ...

@final
class BpeTrainer_Character_CharIdx(BpeTrainerBase):
    def __new__(cls, /, special_tokens: Sequence[str], initial_alphabet: str | None = None, tie_break: str | None = None, parallel_merge_min_occurs_in: int | None = None) -> BpeTrainer_Character_CharIdx: ...
    def add_word_counter(self, /, counter: WordCounter) -> None: ...
    def add_words(self, /, words: Sequence[tuple[str, int]]) -> None: ...
    def get_vocab(self, /) -> Vocabulary: ...
    def init_training(self, /) -> None: ...
    def step(self, /) -> int: ...
    def train_until(self, /, vocab_size: int) -> int: ...
    def validate_model(self, /) -> BpeModelBase: ...
    def vocab_size(self, /) -> int: ...

@final
class BpeTrainer_u8_Idx(BpeTrainerBase):
    def __new__(cls, /, special_tokens: Sequence[str], initial_alphabet: str | None = None, tie_break: str | None = None, parallel_merge_min_occurs_in: int | None = None) -> BpeTrainer_u8_Idx: ...
    def add_word_counter(self, /, counter: WordCounter) -> None: ...
    def add_words(self, /, words: Sequence[tuple[str, int]]) -> None: ...
    def get_vocab(self, /) -> Vocabulary: ...
    def init_training(self, /) -> None: ...
    def step(self, /) -> int: ...
    def train_until(self, /, vocab_size: int) -> int: ...
    def validate_model(self, /) -> BpeModelBase: ...
    def vocab_size(self, /) -> int: ...

@final
class PreTokenizer:
    def __new__(cls, /, special_tokens: Sequence[str], eot_token: str | None = None, pat_str: str | None = None, unicode_bigrams: Sequence[str] | None = None, unicode_bigram_mixed_boundary: str = "keep") -> PreTokenizer: ...
    def build_unicode_bigrams_from_file(self, /, path: str | PathLike, *, chunk_size: int = 1048576, boundary: str = "auto", top_k: int = 100000, min_freq: int = 16) -> list[str]: ...
    def bigram_counter(self, /) -> BigramCounter: ...
    def find_chunk_boundaries(self, /, path: str | PathLike, *, chunk_size: int = 1048576, boundary: str = "auto") -> list[tuple[int, int]]: ...
    def get_words(self, /, text: str) -> dict[str, int]: ...
    def get_words_from_file(self, /, path: str | PathLike, *, chunk_size: int = 1048576, boundary: str = "auto") -> dict[str, int]: ...
    def get_words_from_segment(self, /, path: str | PathLike, offset: int, length: int) -> dict[str, int]: ...
    def load_word_counter(self, /, path: str | PathLike) -> WordCounter: ...
    def word_counter(self, /) -> WordCounter: ...

@final
class Vocabulary:
    def get(self, /, word: str) -> int | None: ...
    def items(self, /) -> list[tuple[bytes, int]]: ...
    @property
    def len(self, /) -> int: ...

@final
class WordCounter:
    def __new__(cls, /, pre_tokenizer: PreTokenizer) -> WordCounter: ...
    def add_batch(self, /, texts: Sequence[str]) -> None: ...
    def add_source(self, /, source: Any, *, max_records: int = 4096, max_bytes: int = 67108864, prefetch: Literal[0, 1] = 1) -> None: ...
    def add_text(self, /, text: str) -> None: ...
    def clear(self, /) -> None: ...
    def merge(self, /, other: WordCounter) -> None: ...
    def save(self, /, path: str | PathLike) -> None: ...
    def words(self, /) -> dict[str, int]: ...
    @property
    def len(self, /) -> int: ...