Skip to main content

aft_tokenizer/
lib.rs

1//! Claude tokenizer matching `ai-tokenizer`'s lookup/BPE hybrid encoding.
2
3mod claude;
4
5pub use claude::{count_tokens, encode};