unitoken 0.1.1

Fast BPE tokenizer/trainer with a Rust core and Python bindings
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17

from .trainer import BpeTrainer
from .encoder import BpeEncoder
from ._lib import PreTokenizer, Vocabs

try:
  from importlib.metadata import version as _pkg_version
  __version__ = _pkg_version("unitoken")
except Exception:  # pragma: no cover
  __version__ = "0.0.0"

__all__ = [
  "BpeTrainer",
  "BpeEncoder",
  "PreTokenizer",
  "Vocabs",
  "__version__",
]