use crate::TokenCounter;
#[derive(Debug, thiserror::Error)]
#[error("failed to initialise tiktoken encoding: {0}")]
pub struct TiktokenError(String);
pub struct TiktokenCounter {
bpe: tiktoken_rs::CoreBPE,
backend_name: &'static str,
}
impl std::fmt::Debug for TiktokenCounter {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("TiktokenCounter")
.field("backend", &self.backend_name)
.finish_non_exhaustive()
}
}
impl TiktokenCounter {
pub fn new() -> Result<Self, TiktokenError> {
Self::from_encoding("tiktoken(cl100k_base)", tiktoken_rs::cl100k_base())
}
pub fn o200k() -> Result<Self, TiktokenError> {
Self::from_encoding("tiktoken(o200k_base)", tiktoken_rs::o200k_base())
}
fn from_encoding<E: std::fmt::Display>(
backend_name: &'static str,
init: Result<tiktoken_rs::CoreBPE, E>,
) -> Result<Self, TiktokenError> {
Ok(Self {
bpe: init.map_err(|e| TiktokenError(e.to_string()))?,
backend_name,
})
}
}
impl TokenCounter for TiktokenCounter {
fn count(&self, text: &str) -> usize {
self.bpe.encode_ordinary(text).len()
}
fn backend_name(&self) -> &'static str {
self.backend_name
}
}
#[cfg(test)]
#[allow(clippy::expect_used)]
mod tests {
use super::*;
#[test]
fn test_tiktoken_empty() {
let t = TiktokenCounter::new().expect("tiktoken init");
assert_eq!(t.count(""), 0);
}
#[test]
fn test_tiktoken_hello() {
let t = TiktokenCounter::new().expect("tiktoken init");
let c = t.count("hello world");
assert!(c >= 2, "expected >= 2, got {c}");
}
#[test]
fn test_tiktoken_code() {
let t = TiktokenCounter::new().expect("tiktoken init");
let code = r#"fn main() { println!("Hello"); }"#;
let c = t.count(code);
assert!(c >= 5, "expected >= 5, got {c}");
}
#[test]
fn test_backend_name() {
let t = TiktokenCounter::new().expect("tiktoken init");
assert_eq!(t.backend_name(), "tiktoken(cl100k_base)");
}
#[test]
fn test_deterministic() {
let t = TiktokenCounter::new().expect("tiktoken init");
let text = "The quick brown fox jumps over the lazy dog.";
assert_eq!(t.count(text), t.count(text));
}
#[test]
fn test_o200k_empty() {
let t = TiktokenCounter::o200k().expect("o200k init");
assert_eq!(t.count(""), 0);
}
#[test]
fn test_o200k_known_counts() {
let t = TiktokenCounter::o200k().expect("o200k init");
assert_eq!(t.count("hello world"), 2);
assert_eq!(t.count("Hello, world!"), 4);
assert_eq!(t.count("The quick brown fox jumps over the lazy dog."), 10);
assert_eq!(t.count(r#"fn main() { println!("Hello"); }"#), 9);
}
#[test]
fn test_o200k_backend_name() {
let t = TiktokenCounter::o200k().expect("o200k init");
assert_eq!(t.backend_name(), "tiktoken(o200k_base)");
}
#[test]
fn test_o200k_monotone_in_length() {
let t = TiktokenCounter::o200k().expect("o200k init");
let mut text = String::from("hello world");
let mut prev = t.count(&text);
for word in ["and", "the", "rest", "of", "the", "corpus", "follows"] {
text.push(' ');
text.push_str(word);
let now = t.count(&text);
assert!(now >= prev, "count decreased after append: {prev} -> {now}");
prev = now;
}
}
#[test]
fn test_o200k_special_token_string_is_ordinary() {
let t = TiktokenCounter::o200k().expect("o200k init");
assert_eq!(t.count("<|endoftext|>"), 7);
}
#[test]
fn test_o200k_differs_from_cl100k_on_emoji_and_multilingual() {
let cl = TiktokenCounter::new().expect("cl100k init");
let o2 = TiktokenCounter::o200k().expect("o200k init");
for text in ["🌍", "hello 🌍 world", "こんにちは世界", "Привет, мир!"] {
assert_ne!(
cl.count(text),
o2.count(text),
"cl100k and o200k unexpectedly agree on {text:?}"
);
}
}
#[test]
fn test_o200k_deterministic() {
let t = TiktokenCounter::o200k().expect("o200k init");
let text = "こんにちは世界 🌍 Привет";
assert_eq!(t.count(text), t.count(text));
}
#[test]
fn test_o200k_debug_does_not_panic() {
let t = TiktokenCounter::o200k().expect("o200k init");
let rendered = format!("{t:?}");
assert!(rendered.contains("o200k"));
}
}