simd_tokenizer/lib.rs
1//! SIMD-style token counting without unsafe code.
2//!
3//! Provides a pure-Rust, SWAR-boosted token estimator — extracted from
4//! [clawdius](https://github.com/WyattAu/clawdius) — for environments where
5//! `tiktoken-rs` is unavailable (WASM builds, minimal builds, or when the
6//! C/C++/Python tokenizer runtimes cannot be linked), or where a fast
7//! *estimate* beats a slow *exact* count.
8//!
9//! # Architecture
10//!
11//! | Type | Description |
12//! | --- | --- |
13//! | [`TokenCounter`] | Trait abstracting token counting backends. |
14//! | [`SimdWhitespaceTokenizer`] | SWAR byte-scan for whitespace boundaries. |
15//! | [`TokenEstimator`] | Enum dispatching to the best available backend. |
16//! | [`TiktokenCounter`] | Exact `cl100k_base` counting (feature `tiktoken`). |
17//!
18//! # The "SIMD" in the name, honestly
19//!
20//! clawdius's original scanner used `std::arch` SSE2/NEON intrinsics behind
21//! `unsafe`. This port replaces it with **SWAR** ("SIMD Within A Register"):
22//! 8 bytes are loaded per iteration into a `u64` and classified with
23//! branch-free bit tricks, then run boundaries are counted with `popcount`.
24//! It is fully portable (no `cfg(target_feature)`, no runtime detection) and
25//! lets the whole crate declare `#![forbid(unsafe_code)]` while staying on
26//! stable Rust.
27//!
28//! Performance characteristics, measured honestly:
29//!
30//! - Several times faster than a naive branchy per-byte loop on large inputs,
31//! and much faster than `str::split_whitespace().count()` (which allocates
32//! UTF-8-boundary-aware iterators).
33//! - Not as fast as hand-written SSE2/AVX2 for huge buffers; SWAR moves 8
34//! bytes per instruction-stream step vs 16-32 for vector ISAs.
35//! - For token *estimation* granularity (the result feeds a heuristic blend,
36//! not an exact invoice) this trade is usually invisible end-to-end.
37//!
38//! # Estimate semantics
39//!
40//! [`SimdWhitespaceTokenizer`] splits on any byte ≤ `0x20` (space, tab,
41//! newline, CR, and other C0 controls), then blends the word count with a
42//! ~4 chars/token heuristic and a punctuation overhead term — see
43//! [`estimate_from_whitespace_splits`]. It approximates BPE behavior for
44//! budgeting and rate-limit decisions; it is **not** a tokenizer table.
45//!
46//! # Feature flags
47//!
48//! - `tiktoken` (default **off**) — adds [`TiktokenCounter`] for exact
49//! `cl100k_base` counts and makes [`TokenEstimator::new`] prefer it.
50//!
51//! # Guarantees
52//!
53//! - `#![forbid(unsafe_code)]` — including the SIMD-style scan.
54//! - `#![deny(missing_docs)]` — the entire public API is documented.
55//! - Counting is deterministic, allocation-free, and never panics on any
56//! input (verified by property tests over arbitrary byte strings,
57//! including non-UTF-8 and truncated UTF-8 sequences).
58
59#![forbid(unsafe_code)]
60#![deny(missing_docs)]
61
62mod swar;
63
64#[cfg(all(not(target_arch = "wasm32"), feature = "tiktoken"))]
65mod tiktoken_backend;
66
67pub use swar::{scalar_count_splits, SimdWhitespaceTokenizer};
68#[cfg(all(not(target_arch = "wasm32"), feature = "tiktoken"))]
69pub use tiktoken_backend::{TiktokenCounter, TiktokenError};
70
71/// Abstraction over token counting backends.
72pub trait TokenCounter: Send + Sync {
73 /// Return an estimated (or exact) token count for `text`.
74 fn count(&self, text: &str) -> usize;
75
76 /// Return a human-readable name for this backend.
77 fn backend_name(&self) -> &'static str;
78}
79
80/// Estimate a BPE-approximate token count from whitespace split statistics.
81///
82/// Deliberately kept simple: split on whitespace, add a small overhead for
83/// punctuation that would normally become separate tokens in BPE, and apply
84/// a ~4 chars/token heuristic for whitespace-dense text. The blend trusts
85/// word count when there are clear word boundaries and falls back to the
86/// character-based estimate for dense text.
87pub fn estimate_from_whitespace_splits(splits: usize, byte_len: usize) -> usize {
88 if splits == 0 {
89 return 0;
90 }
91 let word_tokens = splits;
92 let punct_overhead = (byte_len / 40).max(1);
93 let char_based = byte_len.div_ceil(4);
94 // Weighted blend of both estimates.
95 let blended = (word_tokens + punct_overhead + char_based) / 2;
96 blended.max(1)
97}
98
99/// Enum that dispatches to the best available token counter.
100///
101/// With the `tiktoken` feature enabled (on non-WASM targets) this wraps a
102/// [`TiktokenCounter`]; otherwise it uses [`SimdWhitespaceTokenizer`].
103#[derive(Debug)]
104pub enum TokenEstimator {
105 /// Exact counting via tiktoken-rs (cl100k_base).
106 #[cfg(all(not(target_arch = "wasm32"), feature = "tiktoken"))]
107 Tiktoken(TiktokenCounter),
108 /// Fast SWAR-accelerated approximation.
109 Simd(SimdWhitespaceTokenizer),
110}
111
112impl TokenEstimator {
113 /// Create the best estimator available for the current configuration.
114 ///
115 /// # Panics
116 ///
117 /// With the `tiktoken` feature enabled, panics if the `cl100k_base`
118 /// encoding fails to load. Use [`TokenEstimator::try_new_tiktoken`] for
119 /// fallible construction.
120 pub fn new() -> Self {
121 #[cfg(all(not(target_arch = "wasm32"), feature = "tiktoken"))]
122 {
123 Self::Tiktoken(
124 TiktokenCounter::new().expect("failed to initialise cl100k_base tokenizer"),
125 )
126 }
127 #[cfg(not(all(not(target_arch = "wasm32"), feature = "tiktoken")))]
128 {
129 Self::Simd(SimdWhitespaceTokenizer::new())
130 }
131 }
132
133 /// Fallibly construct the tiktoken-backed estimator (only available with
134 /// the `tiktoken` feature on non-WASM targets).
135 #[cfg(all(not(target_arch = "wasm32"), feature = "tiktoken"))]
136 pub fn try_new_tiktoken() -> Result<Self, TiktokenError> {
137 Ok(Self::Tiktoken(TiktokenCounter::new()?))
138 }
139}
140
141impl Default for TokenEstimator {
142 fn default() -> Self {
143 Self::new()
144 }
145}
146
147impl TokenCounter for TokenEstimator {
148 fn count(&self, text: &str) -> usize {
149 match self {
150 #[cfg(all(not(target_arch = "wasm32"), feature = "tiktoken"))]
151 Self::Tiktoken(t) => t.count(text),
152 Self::Simd(s) => s.count(text),
153 }
154 }
155
156 fn backend_name(&self) -> &'static str {
157 match self {
158 #[cfg(all(not(target_arch = "wasm32"), feature = "tiktoken"))]
159 Self::Tiktoken(_) => "tiktoken(cl100k_base)",
160 Self::Simd(_) => "simd-whitespace",
161 }
162 }
163}