unitoken 0.1.7

Deprecated: use ffbpe for Unicode-aware, streaming BPE training and encoding
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50

import warnings

warnings.warn(
  "uni-tokenizer has moved to ffbpe and is no longer maintained. "
  "Install it with `pip install ffbpe` and import from `ffbpe`. "
  "The unitoken 0.1.7 APIs remain functional for migration.",
  FutureWarning,
  stacklevel=2,
)

from .trainer import BpeTrainer, FileFormat, Unit
from .model import BpeModel
from .training import train_bpe
from .encoder import BpeEncoder
from .pretokenizer import BigramCounter, BoundaryMode, PreTokenizer, Source, UnicodeBigramMixedBoundary, UnicodeBigramSelection, WordCounter
from .tiktoken_compat import (
  Encoding,
  encoding_for_model,
  encoding_name_for_model,
  get_encoding,
  list_encoding_names,
)

try:
  from importlib.metadata import version as _pkg_version
  __version__ = _pkg_version("uni-tokenizer")
except Exception:  # pragma: no cover
  __version__ = "0.0.0"

__all__ = [
  "BpeTrainer",
  "BpeModel",
  "BigramCounter",
  "BpeEncoder",
  "BoundaryMode",
  "Encoding",
  "FileFormat",
  "PreTokenizer",
  "Source",
  "UnicodeBigramMixedBoundary",
  "UnicodeBigramSelection",
  "Unit",
  "WordCounter",
  "encoding_for_model",
  "encoding_name_for_model",
  "get_encoding",
  "list_encoding_names",
  "train_bpe",
  "__version__",
]