rustbpe 0.1.0

A BPE (Byte Pair Encoding) tokenizer written in Rust with Python bindings
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40

[package]
name = "rustbpe"
version = "0.1.0"
edition = "2021"
description = "A BPE (Byte Pair Encoding) tokenizer written in Rust with Python bindings"
license = "MIT"
readme = "README.md"

[lib]
# "cdylib" is required for Python bindings (shared library)
# "rlib" allows it to also be used as a normal Rust crate
crate-type = ["cdylib", "rlib"]

[dependencies]
# pyo3 provides the Python bindings
# "extension-module" is enabled via feature flag (see below)
pyo3 = { version = "0.23" }
dary_heap = "0.3"
indexmap = "2.2"
fancy-regex = "0.16.1"
log = "0.4.28"
pyo3-log = "0.12.4"
ahash = "0.8.12"
rayon = "1.11.0"
compact_str = "0.9.0"

[dev-dependencies]
# For Rust-side testing
criterion = "0.5"  # benchmarking (optional)

[features]
default = []
# extension-module is enabled by maturin during Python builds
# but NOT during cargo test, allowing tests to link properly
extension-module = ["pyo3/extension-module"]

# Profile for release builds - optimize for speed
[profile.release]
lto = true          # Link-time optimization
codegen-units = 1   # Better optimization, slower compile