Skip to main content

simd_tokenizer/
lib.rs

1//! SIMD-style token counting without unsafe code.
2//!
3//! Provides a pure-Rust, SWAR-boosted token estimator — extracted from
4//! [clawdius](https://github.com/WyattAu/clawdius) — for environments where
5//! `tiktoken-rs` is unavailable (WASM builds, minimal builds, or when the
6//! C/C++/Python tokenizer runtimes cannot be linked), or where a fast
7//! *estimate* beats a slow *exact* count.
8//!
9//! # Architecture
10//!
11//! | Type | Description |
12//! | --- | --- |
13//! | [`TokenCounter`] | Trait abstracting token counting backends. |
14//! | [`SimdWhitespaceTokenizer`] | SWAR byte-scan for whitespace boundaries. |
15//! | [`TokenEstimator`] | Enum dispatching to the best available backend. |
16//! | [`TiktokenCounter`] | Exact `cl100k_base` counting (feature `tiktoken`). |
17//!
18//! # The "SIMD" in the name, honestly
19//!
20//! clawdius's original scanner used `std::arch` SSE2/NEON intrinsics behind
21//! `unsafe`. This port replaces it with **SWAR** ("SIMD Within A Register"):
22//! 8 bytes are loaded per iteration into a `u64` and classified with
23//! branch-free bit tricks, then run boundaries are counted with `popcount`.
24//! It is fully portable (no `cfg(target_feature)`, no runtime detection) and
25//! lets the whole crate declare `#![forbid(unsafe_code)]` while staying on
26//! stable Rust.
27//!
28//! Performance characteristics, measured honestly:
29//!
30//! - Several times faster than a naive branchy per-byte loop on large inputs,
31//!   and much faster than `str::split_whitespace().count()` (which allocates
32//!   UTF-8-boundary-aware iterators).
33//! - Not as fast as hand-written SSE2/AVX2 for huge buffers; SWAR moves 8
34//!   bytes per instruction-stream step vs 16-32 for vector ISAs.
35//! - For token *estimation* granularity (the result feeds a heuristic blend,
36//!   not an exact invoice) this trade is usually invisible end-to-end.
37//!
38//! # Estimate semantics
39//!
40//! [`SimdWhitespaceTokenizer`] splits on any byte ≤ `0x20` (space, tab,
41//! newline, CR, and other C0 controls), then blends the word count with a
42//! ~4 chars/token heuristic and a punctuation overhead term — see
43//! [`estimate_from_whitespace_splits`]. It approximates BPE behavior for
44//! budgeting and rate-limit decisions; it is **not** a tokenizer table.
45//!
46//! # Feature flags
47//!
48//! - `tiktoken` (default **off**) — adds [`TiktokenCounter`] for exact
49//!   `cl100k_base` counts and makes [`TokenEstimator::new`] prefer it.
50//!
51//! # Guarantees
52//!
53//! - `#![forbid(unsafe_code)]` — including the SIMD-style scan.
54//! - `#![deny(missing_docs)]` — the entire public API is documented.
55//! - Counting is deterministic, allocation-free, and never panics on any
56//!   input (verified by property tests over arbitrary byte strings,
57//!   including non-UTF-8 and truncated UTF-8 sequences).
58
59#![forbid(unsafe_code)]
60#![deny(missing_docs)]
61
62mod swar;
63
64#[cfg(all(not(target_arch = "wasm32"), feature = "tiktoken"))]
65mod tiktoken_backend;
66
67pub use swar::{scalar_count_splits, SimdWhitespaceTokenizer};
68#[cfg(all(not(target_arch = "wasm32"), feature = "tiktoken"))]
69pub use tiktoken_backend::{TiktokenCounter, TiktokenError};
70
71/// Abstraction over token counting backends.
72pub trait TokenCounter: Send + Sync {
73    /// Return an estimated (or exact) token count for `text`.
74    fn count(&self, text: &str) -> usize;
75
76    /// Return a human-readable name for this backend.
77    fn backend_name(&self) -> &'static str;
78}
79
80/// Estimate a BPE-approximate token count from whitespace split statistics.
81///
82/// Deliberately kept simple: split on whitespace, add a small overhead for
83/// punctuation that would normally become separate tokens in BPE, and apply
84/// a ~4 chars/token heuristic for whitespace-dense text. The blend trusts
85/// word count when there are clear word boundaries and falls back to the
86/// character-based estimate for dense text.
87pub fn estimate_from_whitespace_splits(splits: usize, byte_len: usize) -> usize {
88    if splits == 0 {
89        return 0;
90    }
91    let word_tokens = splits;
92    let punct_overhead = (byte_len / 40).max(1);
93    let char_based = byte_len.div_ceil(4);
94    // Weighted blend of both estimates.
95    let blended = (word_tokens + punct_overhead + char_based) / 2;
96    blended.max(1)
97}
98
99/// Enum that dispatches to the best available token counter.
100///
101/// With the `tiktoken` feature enabled (on non-WASM targets) this wraps a
102/// [`TiktokenCounter`]; otherwise it uses [`SimdWhitespaceTokenizer`].
103#[derive(Debug)]
104pub enum TokenEstimator {
105    /// Exact counting via tiktoken-rs (cl100k_base).
106    #[cfg(all(not(target_arch = "wasm32"), feature = "tiktoken"))]
107    Tiktoken(TiktokenCounter),
108    /// Fast SWAR-accelerated approximation.
109    Simd(SimdWhitespaceTokenizer),
110}
111
112impl TokenEstimator {
113    /// Create the best estimator available for the current configuration.
114    ///
115    /// # Panics
116    ///
117    /// With the `tiktoken` feature enabled, panics if the `cl100k_base`
118    /// encoding fails to load. Use [`TokenEstimator::try_new_tiktoken`] for
119    /// fallible construction.
120    pub fn new() -> Self {
121        #[cfg(all(not(target_arch = "wasm32"), feature = "tiktoken"))]
122        {
123            Self::Tiktoken(
124                TiktokenCounter::new().expect("failed to initialise cl100k_base tokenizer"),
125            )
126        }
127        #[cfg(not(all(not(target_arch = "wasm32"), feature = "tiktoken")))]
128        {
129            Self::Simd(SimdWhitespaceTokenizer::new())
130        }
131    }
132
133    /// Fallibly construct the tiktoken-backed estimator (only available with
134    /// the `tiktoken` feature on non-WASM targets).
135    #[cfg(all(not(target_arch = "wasm32"), feature = "tiktoken"))]
136    pub fn try_new_tiktoken() -> Result<Self, TiktokenError> {
137        Ok(Self::Tiktoken(TiktokenCounter::new()?))
138    }
139}
140
141impl Default for TokenEstimator {
142    fn default() -> Self {
143        Self::new()
144    }
145}
146
147impl TokenCounter for TokenEstimator {
148    fn count(&self, text: &str) -> usize {
149        match self {
150            #[cfg(all(not(target_arch = "wasm32"), feature = "tiktoken"))]
151            Self::Tiktoken(t) => t.count(text),
152            Self::Simd(s) => s.count(text),
153        }
154    }
155
156    fn backend_name(&self) -> &'static str {
157        match self {
158            #[cfg(all(not(target_arch = "wasm32"), feature = "tiktoken"))]
159            Self::Tiktoken(_) => "tiktoken(cl100k_base)",
160            Self::Simd(_) => "simd-whitespace",
161        }
162    }
163}