unitoken 0.1.5

Fast BPE tokenizer/trainer with a Rust core and Python bindings
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16

from . import core, load, model, registry
from .core import Encoding
from .model import encoding_for_model, encoding_name_for_model
from .registry import get_encoding, list_encoding_names

__all__ = [
  "Encoding",
  "core",
  "encoding_for_model",
  "encoding_name_for_model",
  "get_encoding",
  "load",
  "list_encoding_names",
  "model",
  "registry",
]