tiktoken 3.7.0

A high-performance pure-Rust implementation of OpenAI's tiktoken BPE tokenizer
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38

[package]
name = "tiktoken"
description = "A high-performance pure-Rust implementation of OpenAI's tiktoken BPE tokenizer"
license.workspace = true
repository.workspace = true
homepage.workspace = true
documentation = "https://docs.rs/tiktoken"
version = "3.7.0"
edition.workspace = true
authors.workspace = true
keywords = ["tiktoken", "tokenizer", "bpe", "openai", "gpt"]
categories = ["text-processing", "algorithms"]
# Oracle fixtures and differential tests stay in the repository but are
# excluded from the published package: they are several MB of reference data
# only used to validate this crate against upstream tokenizers, and crates.io
# enforces a 10 MB package cap that the embedded vocabularies already approach.
exclude = ["tests/", "examples/react-app/"]

[features]
default = []
parallel = ["rayon"]

[dependencies]
base64 = "^0.22"
rayon = { version = "^1", optional = true }
regex = "^1"
rustc-hash = "^2"
ruzstd = "^0.8"

[dev-dependencies]
criterion = { version = "^0.8", features = ["html_reports"] }
proptest = "^1"
serde = { version = "1", features = ["derive"] }
serde_json = "1"

[[bench]]
name = "tokenize"
harness = false