splintr 0.8.0

Fast Rust BPE tokenizer with Python bindings
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39

[build-system]
requires = ["maturin>=1.0,<2.0"]
build-backend = "maturin"

[project]
name = "splintr-rs"
version = "0.8.0"
description = "Fast Rust BPE tokenizer with Python bindings"
readme = "README.md"
license = { text = "MIT" }
requires-python = ">=3.8"
keywords = ["tokenizer", "bpe", "tiktoken", "gpt", "llm"]
authors = [{ name = "Farhan" }]
classifiers = [
    "Development Status :: 4 - Beta",
    "Intended Audience :: Developers",
    "License :: OSI Approved :: MIT License",
    "Programming Language :: Rust",
    "Programming Language :: Python :: 3",
    "Programming Language :: Python :: 3.8",
    "Programming Language :: Python :: 3.9",
    "Programming Language :: Python :: 3.10",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
    "Programming Language :: Python :: Implementation :: CPython",
    "Topic :: Scientific/Engineering :: Artificial Intelligence",
    "Topic :: Text Processing",
]

[project.urls]
Homepage = "https://github.com/ml-rust/splintr"
Repository = "https://github.com/ml-rust/splintr"
Documentation = "https://github.com/ml-rust/splintr#readme"

[tool.maturin]
python-source = "python"
module-name = "splintr._core"
bindings = "pyo3"
features = ["python"]