splintr 0.15.0

Fast Rust tokenizer (BPE + SentencePiece + WordPiece) with Python bindings
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

mod agent_tokens;
mod backend;
mod builder;
mod cache;
mod decode;
mod encode;
mod error;
mod patterns;
pub(crate) mod scanner;
#[cfg(test)]
mod tests;
mod types;

pub use agent_tokens::{
    cl100k_agent_tokens, deepseek_v3_agent_tokens, glm4_agent_tokens, gpt_oss_agent_tokens,
    llama3_agent_tokens, mistral_v1_agent_tokens, mistral_v2_agent_tokens, mistral_v3_agent_tokens,
    o200k_agent_tokens, qwen3_agent_tokens,
};
pub use error::TokenizerError;
pub use patterns::{
    CL100K_BASE_PATTERN, DEEPSEEK_V3_PATTERNS, GPT2_PATTERN, LLAMA3_PATTERN, MISTRAL_V3_PATTERN,
    NO_SPLIT_PATTERN, O200K_BASE_PATTERN, QWEN2_PATTERN, SENTENCEPIECE_PATTERN,
};
pub use types::{ByteFallback, Tokenizer};