[package]
edition = "2021"
rust-version = "1.88.0"
name = "text_chipper"
version = "0.2.1"
build = false
include = [
"/src/**",
"/benches/**",
"/tests/fixtures/**",
"/Cargo.toml",
"/README.md",
"/LICENSE",
"!.DS_Store",
]
autolib = false
autobins = false
autoexamples = false
autotests = false
autobenches = false
description = "Token-budgeted, Markdown-aware text chunking for RAG and LLM pipelines, with pluggable tokenizers and Python bindings."
readme = "README.md"
keywords = [
"chunking",
"tokenizer",
"markdown",
"rag",
"nlp",
]
categories = ["text-processing"]
license = "Apache-2.0"
repository = "https://github.com/kameh0use/text-chipper"
[features]
default = []
diagnostics = []
hf = ["dep:tokenizers"]
mimalloc = ["dep:mimalloc"]
python = [
"dep:pyo3",
"mimalloc",
]
stats = []
[lib]
name = "text_chipper"
crate-type = [
"cdylib",
"rlib",
]
path = "src/lib.rs"
[[bench]]
name = "chipper_benchmark"
path = "benches/chipper_benchmark.rs"
harness = false
[dependencies.mimalloc]
version = "0.1.43"
optional = true
[dependencies.pyo3]
version = "0.30.1"
features = ["abi3-py39"]
optional = true
[dependencies.rayon]
version = "1.10"
[dependencies.tiktoken]
version = "=4.1.2"
features = ["vocab-openai"]
default-features = false
[dependencies.tokenizers]
version = "0.23"
optional = true
[dependencies.unicode-normalization]
version = "0.1"
[dependencies.unicode-segmentation]
version = "1.11"
[dev-dependencies.criterion]
version = "0.8"
features = ["html_reports"]
[dev-dependencies.proptest]
version = "1.4"
[dev-dependencies.pulldown-cmark]
version = "0.13"
default-features = false
[dev-dependencies.serde_json]
version = "1"
[dev-dependencies.tiktoken-rs]
version = "=0.6.0"
[profile.release]
opt-level = 3
lto = "fat"
codegen-units = 1