tokenfold-core 0.3.2

Token-aware compression for LLM payloads: shrink JSON tool-call bodies, command output, and diffs with exact tiktoken accounting and a typed safety report.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

[package]
name = "tokenfold-core"
version.workspace = true
edition.workspace = true
rust-version.workspace = true
license.workspace = true
repository.workspace = true
description = "Token-aware compression for LLM payloads: shrink JSON tool-call bodies, command output, and diffs with exact tiktoken accounting and a typed safety report."
keywords = ["llm", "tokenizer", "compression", "tiktoken", "openai"]
categories = ["compression", "text-processing"]

[dependencies]
serde = { workspace = true }
serde_json = { workspace = true }
thiserror = { workspace = true }
regex = { workspace = true }
sha2 = { workspace = true }
toml = { workspace = true }
tiktoken-rs = { workspace = true, optional = true }

[dev-dependencies]
proptest = { workspace = true }

[features]
default = ["tiktoken"]
tiktoken = ["dep:tiktoken-rs"]

[[bench]]
name = "compression_bench"
harness = false