unitoken 0.1.4

Fast BPE tokenizer/trainer with a Rust core and Python bindings
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55

from collections.abc import Sequence
from os import PathLike
from typing import Any, final

@final
class BpeEncoderBase:
    def __new__(cls, /, spec: str, char_level: str, vocabs: dict[Sequence[int], int] | None, merges: Sequence[tuple[Sequence[int], Sequence[int]]] | None, vocab_filename: str | PathLike | None, merges_filename: str | PathLike | None, special_tokens: Sequence[str] | None, pat_str: str | None = None) -> BpeEncoderBase: ...
    def decode(self, /, idxs: Any) -> str: ...
    def encode_file(self, /, path: str | PathLike, num_chunks: int) -> Any: ...
    def encode_string(self, /, s: str) -> Any: ...
    def encode_string_to_list(self, /, s: str) -> list[int]: ...
    def encode_word(self, /, word: str) -> list[int]: ...
    def encode_words(self, /, words: Sequence[str]) -> list[list[int]]: ...
    def pre_tokenizer(self, /) -> PreTokenizer: ...

class BpeTrainerBase: ...

@final
class BpeTrainer_Character_CharIdx(BpeTrainerBase):
    def __new__(cls, /, special_tokens: Sequence[str], initial_alphabet: str | None = None, tie_break: str | None = None, parallel_merge_min_occurs_in: int | None = None) -> BpeTrainer_Character_CharIdx: ...
    def add_words(self, /, words: Sequence[tuple[str, int]]) -> None: ...
    def get_vocabs(self, /) -> Vocabs: ...
    def init_training(self, /) -> None: ...
    def save_merges_txt(self, /, path: str | PathLike, spec: str) -> None: ...
    def save_vocab(self, /, path: str | PathLike, spec: str) -> None: ...
    def step(self, /) -> int: ...
    def train_until(self, /, vocab_size: int) -> int: ...
    def vocab_size(self, /) -> int: ...

@final
class BpeTrainer_u8_Idx(BpeTrainerBase):
    def __new__(cls, /, special_tokens: Sequence[str], initial_alphabet: str | None = None, tie_break: str | None = None, parallel_merge_min_occurs_in: int | None = None) -> BpeTrainer_u8_Idx: ...
    def add_words(self, /, words: Sequence[tuple[str, int]]) -> None: ...
    def get_vocabs(self, /) -> Vocabs: ...
    def init_training(self, /) -> None: ...
    def save_merges_txt(self, /, path: str | PathLike, spec: str) -> None: ...
    def save_vocab(self, /, path: str | PathLike, spec: str) -> None: ...
    def step(self, /) -> int: ...
    def train_until(self, /, vocab_size: int) -> int: ...
    def vocab_size(self, /) -> int: ...

@final
class PreTokenizer:
    def __new__(cls, /, special_tokens: Sequence[str], eot_token: str | None = None, pat: str | None = None, unicode_bigrams: Sequence[str] | None = None, unicode_bigram_mixed_boundary: str = "keep") -> PreTokenizer: ...
    def build_unicode_bigrams_from_file(self, /, path: str | PathLike, *, chunk_size: int = 1048576, boundary: str = "auto", top_k: int = 100000, min_freq: int = 16) -> list[str]: ...
    def find_chunk_boundaries(self, /, path: str | PathLike, *, chunk_size: int = 1048576, boundary: str = "auto") -> list[tuple[int, int]]: ...
    def get_words_from_file(self, /, path: str | PathLike, *, chunk_size: int = 1048576, boundary: str = "auto") -> dict[str, int]: ...
    def get_words_from_segment(self, /, path: str | PathLike, offset: int, length: int) -> dict[str, int]: ...

@final
class Vocabs:
    def get(self, /, word: str) -> int | None: ...
    def items(self, /) -> list[tuple[bytes, int]]: ...
    @property
    def len(self, /) -> int: ...