Crate splintr 

Re-exports

pub use core::cl100k_agent_tokens;

pub use core::codellama_agent_tokens;

pub use core::deepseek_v3_agent_tokens;

pub use core::gemma2_agent_tokens;

pub use core::gemma3_agent_tokens;

pub use core::gemma4_agent_tokens;

pub use core::glm4_agent_tokens;

pub use core::gpt_oss_agent_tokens;

pub use core::kimi_k2_agent_tokens;

pub use core::kimi_k3_agent_tokens;

pub use core::llama2_agent_tokens;

pub use core::llama3_agent_tokens;

pub use core::mistral_v1_agent_tokens;

pub use core::mistral_v2_agent_tokens;

pub use core::mistral_v3_agent_tokens;

pub use core::modernbert_agent_tokens;

pub use core::o200k_agent_tokens;

pub use core::olmo2_agent_tokens;

pub use core::phi4_agent_tokens;

pub use core::qwen3_agent_tokens;

pub use core::AddedToken;

pub use core::AddedTokenSet;

pub use core::ByteFallback;

pub use core::SentencePieceError;

pub use core::SentencePieceTokenizer;

pub use core::SpmError;

pub use core::SpmPrefixScheme;

pub use core::SpmTokenizer;

pub use core::StreamingDecoder;

pub use core::Tokenize;

pub use core::TokenizeError;

pub use core::Tokenizer;

pub use core::TokenizerError;

pub use core::WordPieceError;

pub use core::WordPieceTokenizer;

pub use core::CL100K_BASE_PATTERN;

pub use core::DEEPSEEK_V3_PATTERNS;

pub use core::GPT2_PATTERN;

pub use core::KIMI_PATTERN;

pub use core::LLAMA3_PATTERN;

pub use core::MISTRAL_V3_PATTERN;

pub use core::NO_SPLIT_PATTERN;

pub use core::O200K_BASE_PATTERN;

pub use core::QWEN2_PATTERN;

pub use core::SENTENCEPIECE_PATTERN;

pub use core::pretrained;

pub use core::whisper;

pub use core::base_vocab_size;

pub use core::base_vocab_size_by_name;

pub use core::bos_token_id;

pub use core::bos_token_id_by_name;

pub use core::cl100k_base_special_tokens;

pub use core::deepseek_v3_special_tokens;

pub use core::eos_token_id;

pub use core::eos_token_id_by_name;

pub use core::from_pretrained;

pub use core::from_vocab;

pub use core::glm4_special_tokens;

pub use core::gpt_oss_special_tokens;

pub use core::llama3_special_tokens;

pub use core::o200k_base_special_tokens;

pub use core::pad_token_id;

pub use core::patterns;

pub use core::qwen3_special_tokens;

pub use core::special_tokens;

pub use core::uses_byte_level;

pub use core::PretrainedVocab;

pub use core::whisper_special_tokens;

pub use core::WhisperVariant;

pub use core::from_gguf_vocab;

pub use core::from_json_bytes;

pub use core::from_json_path;

pub use core::AnyTokenizer;

pub use core::Backend;

pub use core::GgufVocab;

pub use core::GgufVocabError;

pub use core::HfJsonError;

pub use core::PolicyError;

pub use core::SpecialDecode;

pub use core::SpecialMode;

pub use core::SpecialPolicy;

pub use core::CharsmapDialect;

pub use core::NormOp;

pub use core::Normalizer;

pub use core::Precompiled;

pub use core::PreTokStage;

pub use core::PreTokenizer;

pub use core::SplitBehavior;

pub use core::SplitPattern;

Modules

core

Core tokenization engine for splintr.

Type Aliases

FxHashMap

The hash map splintr’s vocabulary constructors take, re-exported for the same reason as FxHashSet: Tokenizer::new takes FxHashMap<Vec<u8>, u32> for the encoder and FxHashMap<String, u32> for the special tokens, so building a tokenizer from your own vocabulary needs this type by name. Type alias for a hash map that uses the Fx hashing algorithm.

FxHashSet

The hash set SpecialMode::Allow borrows, re-exported so that mode is constructible from this crate’s own exports.