unitoken 0.1.5

Fast BPE tokenizer/trainer with a Rust core and Python bindings
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88

[package]
name = "unitoken"
version = "0.1.5"
edition = "2024"
description = "Fast BPE tokenizer/trainer with a Rust core and Python bindings"
repository = "https://github.com/tokn-ai/unitoken"
readme = "README.md"
license = "MIT"
keywords = ["tokenizer", "bpe", "nlp"]
categories = ["text-processing"]
exclude = [
  ".venv/**",
  "dist/**",
  "out/**",
  "target/**",
  "fixtures/**",
  "!fixtures/default_special_tokens.txt",
  "flamegraph*.svg",
]

[lib]
crate-type = ["cdylib", "rlib"]

[[bin]]
name = "unitoken"
path = "src/main.rs"
required-features = ["cli"]

[dependencies]
ahash = { version = "0.8.12", features = ["serde"] }
hashbrown = { version = "0.16.1", features = ["serde"] }
fancy-regex = "0.16.2"
lazy_static = "1.5.0"
ordermap = { version = "1.0.0", features = ["serde"] }
memchr = "2.7.6"
serde_json = "1.0.145"
thiserror = "2.0.17"
rayon = "1.11.0"
moka = { version = "0.12.11", features = ["sync"] }
tracing = "0.1.43"
metrics = "0.24.3"
serde = { version = "1.0.228", features = ["derive"] }
hotpath = "0.8.0"
npyz = { version = "0.8.4" }
pyo3 = { version = "0.27", optional = true, features = ["experimental-inspect", "abi3-py38"] }
ndarray = { version = "0.17.1", optional = true }
numpy = { version = "0.27.1", optional = true }

clap = { version = "4.5.53", features = ["derive"], optional = true }
tracing-subscriber = { version = "0.3.22", optional = true }
indicatif = { version = "0.18.3", optional = true }
chrono = { version = "0.4.42", optional = true }
rgb = { version = "0.8.52", optional = true } # optional for textplots
textplots = { version = "0.8.7", optional = true }

[profile.dev.package."*"]
opt-level = 2

[profile.test]
inherits = "release"
debug = true

[profile.release]
# 'line-tables-only' is a good balance between file size and profiling detail
debug = "line-tables-only"
# RUSTFLAGS="-C force-frame-pointers=yes" cargo build --release

[features]
default = []
hotpath = ["hotpath/hotpath", "hotpath/hotpath-alloc"]

fmt-npz = ["npyz/npz"]

plot = ["dep:textplots", "dep:rgb"]
py = ["dep:pyo3", "dep:ndarray", "dep:numpy"]
cli = ["dep:tracing-subscriber", "dep:indicatif", "dep:chrono", "dep:clap"]

[dev-dependencies]
pyo3-introspection = "0.27"
criterion = "0.5.1"

[[bench]]
name = "bpe"
harness = false

# [patch.crates-io]
# pyo3 = { git = 'https://github.com/PyO3/pyo3' } # for pyo3-introspection
# pyo3-introspection = { git = 'https://github.com/PyO3/pyo3' } # for https://github.com/PyO3/pyo3/issues/5137