toktok/lib.rs
1//! **toktok** — a fast, exact BPE tokenizer.
2//!
3//! Token ids are byte-identical to [tiktoken](https://github.com/openai/tiktoken),
4//! and encoding runs ~3.5x faster than [`bpe-openai`](https://crates.io/crates/bpe-openai)
5//! and ~10x faster than [`tiktoken-rs`](https://crates.io/crates/tiktoken-rs)
6//! (see the repo's `bench/rust`). The bundled encodings are embedded in the
7//! binary, so there is nothing to download or ship alongside it.
8//!
9//! ```
10//! let tok = toktok::Tokenizer::builtin("cl100k_base")?;
11//!
12//! let ids = tok.encode("Hello, toktok! 日本語 🚀".as_bytes());
13//! assert_eq!(tok.decode(&ids), "Hello, toktok! 日本語 🚀".as_bytes());
14//!
15//! assert_eq!(tok.count(b"how many tokens is this?"), 6);
16//! # Ok::<(), toktok::VocabError>(())
17//! ```
18//!
19//! One tokenizer is safe to share across threads — load it once:
20//!
21//! ```
22//! # let tok = toktok::Tokenizer::builtin("o200k_base")?;
23//! let docs: Vec<&[u8]> = vec![b"first", b"second", b"third"];
24//! let counts = tok.count_batch(&docs, 0, false); // 0 threads = every core
25//! let ids = tok.encode_batch(&docs, 0, false);
26//! # assert_eq!(counts.len(), 3); assert_eq!(ids.len(), 3);
27//! # Ok::<(), toktok::VocabError>(())
28//! ```
29//!
30//! # How it's fast
31//!
32//! Same algorithm as `bpe-openai` (exact backtracking BPE); the speed comes from
33//! data-structure engineering ported from
34//! [quicktok](https://github.com/dmatth1/quicktok)'s C++: a 2-byte-radix trie
35//! whose walk consumes two input bytes per single 8-byte load, dense
36//! bijectively-mixed merge-validity memos, hand-compiled SIMD pretokenizers
37//! instead of a regex engine, and a single-pass machine that fuses
38//! pretokenization with token emission for ASCII text.
39
40mod builtin;
41pub mod mb;
42mod pcache;
43pub mod pretok;
44pub mod pretok_o200k;
45mod scratch;
46pub mod tokenizer;
47pub mod vocab;
48
49pub use builtin::BUILTIN_ENCODINGS;
50pub use pretok::UClass;
51pub use pretok_o200k::UClassO;
52pub use tokenizer::{Scanner, Tokenizer, Truncation};
53pub use vocab::{Vocab, VocabError, RANK_MAX};