[package]
edition = "2024"
name = "tiktoken"
version = "4.0.0"
authors = [
"GOLIA株式会社",
"Li Hao <lihao@golia.jp>",
]
build = false
exclude = [
"tests/",
"examples/react-app/",
"PERF-*.md",
"SIZE-*.md",
]
autolib = false
autobins = false
autoexamples = false
autotests = false
autobenches = false
description = "A high-performance pure-Rust implementation of OpenAI's tiktoken BPE tokenizer"
homepage = "https://github.com/goliajp/rust-tiktoken"
documentation = "https://docs.rs/tiktoken"
readme = "README.md"
keywords = [
"tiktoken",
"tokenizer",
"bpe",
"openai",
"gpt",
]
categories = [
"text-processing",
"algorithms",
]
license = "MIT OR Apache-2.0"
repository = "https://github.com/goliajp/rust-tiktoken"
[package.metadata.docs.rs]
all-features = true
[features]
default = ["vocabs-all"]
parallel = ["rayon"]
vocab-cl100k_base = []
vocab-deepseek = ["vocab-deepseek_v3"]
vocab-deepseek_v3 = []
vocab-glm4 = []
vocab-glm5 = ["vocab-glm4"]
vocab-kimi_k2 = []
vocab-llama3 = ["vocab-cl100k_base"]
vocab-meta = ["vocab-llama3"]
vocab-minimax = ["vocab-minimax_m2"]
vocab-minimax_m2 = []
vocab-mistral = ["vocab-mistral_v3"]
vocab-mistral_v3 = []
vocab-moonshot = ["vocab-kimi_k2"]
vocab-o200k_base = []
vocab-openai = [
"vocab-cl100k_base",
"vocab-o200k_base",
"vocab-p50k_base",
"vocab-r50k_base",
]
vocab-p50k_base = ["vocab-r50k_base"]
vocab-qwen = ["vocab-qwen2"]
vocab-qwen2 = []
vocab-r50k_base = []
vocab-zhipu = [
"vocab-glm4",
"vocab-glm5",
]
vocabs-all = [
"vocab-openai",
"vocab-meta",
"vocab-deepseek",
"vocab-qwen",
"vocab-mistral",
"vocab-moonshot",
"vocab-zhipu",
"vocab-minimax",
]
[lib]
name = "tiktoken"
path = "src/lib.rs"
[[example]]
name = "basic"
path = "examples/basic.rs"
[[example]]
name = "cost_estimation"
path = "examples/cost_estimation.rs"
[[example]]
name = "count_tokens"
path = "examples/count_tokens.rs"
[[example]]
name = "stage_bench"
path = "examples/stage_bench.rs"
[[bench]]
name = "tokenize"
path = "benches/tokenize.rs"
harness = false
[dependencies.rayon]
version = "^1"
optional = true
[dependencies.regex]
version = "^1"
[dependencies.rustc-hash]
version = "^2"
[dependencies.ruzstd]
version = "^0.9"
[dev-dependencies.base64]
version = "^0.23"
[dev-dependencies.criterion]
version = "^0.8"
features = ["html_reports"]
[dev-dependencies.proptest]
version = "^1"
[dev-dependencies.serde]
version = "1"
features = ["derive"]
[dev-dependencies.serde_json]
version = "1"