splintr 0.14.1

Fast Rust tokenizer (BPE + SentencePiece + WordPiece) with Python bindings
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19

mod agent_tokens;
mod backend;
mod builder;
mod cache;
mod decode;
mod encode;
mod error;
mod patterns;
#[cfg(test)]
mod tests;
mod types;

pub use agent_tokens::{cl100k_agent_tokens, o200k_agent_tokens};
pub use error::TokenizerError;
pub use patterns::{
    CL100K_BASE_PATTERN, DEEPSEEK_V3_PATTERNS, GPT2_PATTERN, LLAMA3_PATTERN, MISTRAL_V3_PATTERN,
    NO_SPLIT_PATTERN, O200K_BASE_PATTERN, QWEN2_PATTERN, SENTENCEPIECE_PATTERN,
};
pub use types::{ByteFallback, Tokenizer};