unitoken 0.1.5

Fast BPE tokenizer/trainer with a Rust core and Python bindings
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

from __future__ import annotations

from collections.abc import Callable
import functools
import importlib
import pkgutil
from collections.abc import Sequence
import threading
from typing import Any

import uni_tokenizer.tiktoken as tiktoken
import tiktoken_ext

from uni_tokenizer.tiktoken_compat import Encoding, get_encoding as _get_encoding, list_encoding_names

ENCODINGS: dict[str, Encoding] = {}
ENCODING_CONSTRUCTORS: dict[str, Callable[[], dict[str, Any]]] | None = None


def get_encoding(encoding_name: str) -> Encoding:
  if encoding_name not in ENCODINGS:
    ENCODINGS[encoding_name] = _get_encoding(encoding_name)
  return ENCODINGS[encoding_name]

__all__ = [
  "ENCODINGS",
  "ENCODING_CONSTRUCTORS",
  "Encoding",
  "get_encoding",
  "list_encoding_names",
]