unitoken 0.1.4

Fast BPE tokenizer/trainer with a Rust core and Python bindings
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

from .trainer import BpeTrainer
from .encoder import BpeEncoder
from ._lib import Vocabs
from .pretokenizer import BoundaryMode, PreTokenizer, UnicodeBigramMixedBoundary
from .tiktoken_compat import (
  Encoding,
  encoding_for_model,
  encoding_name_for_model,
  get_encoding,
  list_encoding_names,
)

try:
  from importlib.metadata import version as _pkg_version
  __version__ = _pkg_version("uni-tokenizer")
except Exception:  # pragma: no cover
  __version__ = "0.0.0"

__all__ = [
  "BpeTrainer",
  "BpeEncoder",
  "BoundaryMode",
  "Encoding",
  "PreTokenizer",
  "UnicodeBigramMixedBoundary",
  "Vocabs",
  "encoding_for_model",
  "encoding_name_for_model",
  "get_encoding",
  "list_encoding_names",
  "__version__",
]