tiktoken 4.1.2

A high-performance pure-Rust implementation of OpenAI's tiktoken BPE tokenizer
Documentation
# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
#
# When uploading crates to the registry Cargo will automatically
# "normalize" Cargo.toml files for maximal compatibility
# with all versions of Cargo and also rewrite `path` dependencies
# to registry (e.g., crates.io) dependencies.
#
# If you are reading this file be aware that the original Cargo.toml
# will likely look very different (and much more reasonable).
# See Cargo.toml.orig for the original contents.

[package]
edition = "2024"
name = "tiktoken"
version = "4.1.2"
authors = [
    "GOLIA株式会社",
    "Li Hao <lihao@golia.jp>",
]
build = false
exclude = [
    "tests/",
    "examples/react-app/",
    "PERF-*.md",
    "SIZE-*.md",
]
autolib = false
autobins = false
autoexamples = false
autotests = false
autobenches = false
description = "A high-performance pure-Rust implementation of OpenAI's tiktoken BPE tokenizer"
homepage = "https://github.com/goliajp/rust-tiktoken"
documentation = "https://docs.rs/tiktoken"
readme = "README.md"
keywords = [
    "tiktoken",
    "tokenizer",
    "bpe",
    "openai",
    "gpt",
]
categories = [
    "text-processing",
    "algorithms",
]
license = "MIT OR Apache-2.0"
repository = "https://github.com/goliajp/rust-tiktoken"

[package.metadata.docs.rs]
all-features = true

[features]
default = ["vocabs-all"]
parallel = ["rayon"]
vocab-cl100k_base = []
vocab-deepseek = ["vocab-deepseek_v3"]
vocab-deepseek_v3 = []
vocab-glm4 = []
vocab-glm5 = ["vocab-glm4"]
vocab-kimi_k2 = []
vocab-llama3 = ["vocab-cl100k_base"]
vocab-meta = ["vocab-llama3"]
vocab-minimax = ["vocab-minimax_m2"]
vocab-minimax_m2 = []
vocab-mistral = ["vocab-mistral_v3"]
vocab-mistral_v3 = []
vocab-moonshot = ["vocab-kimi_k2"]
vocab-o200k_base = []
vocab-openai = [
    "vocab-cl100k_base",
    "vocab-o200k_base",
    "vocab-p50k_base",
    "vocab-r50k_base",
]
vocab-p50k_base = ["vocab-r50k_base"]
vocab-qwen = ["vocab-qwen2"]
vocab-qwen2 = []
vocab-r50k_base = []
vocab-zhipu = [
    "vocab-glm4",
    "vocab-glm5",
]
vocabs-all = [
    "vocab-openai",
    "vocab-meta",
    "vocab-deepseek",
    "vocab-qwen",
    "vocab-mistral",
    "vocab-moonshot",
    "vocab-zhipu",
    "vocab-minimax",
]

[lib]
name = "tiktoken"
path = "src/lib.rs"

[[example]]
name = "basic"
path = "examples/basic.rs"

[[example]]
name = "cost_estimation"
path = "examples/cost_estimation.rs"

[[example]]
name = "count_tokens"
path = "examples/count_tokens.rs"

[[example]]
name = "stage_bench"
path = "examples/stage_bench.rs"

[[bench]]
name = "tokenize"
path = "benches/tokenize.rs"
harness = false

[dependencies.rayon]
version = "^1"
optional = true

[dependencies.regex]
version = "^1"

[dependencies.rustc-hash]
version = "^2"

[dependencies.ruzstd]
version = "^0.9"

[dev-dependencies.base64]
version = "^0.23"

[dev-dependencies.criterion]
version = "^0.8"
features = ["html_reports"]

[dev-dependencies.proptest]
version = "^1"

[dev-dependencies.serde]
version = "1"
features = ["derive"]

[dev-dependencies.serde_json]
version = "1"