Crate splintr 

Re-exports

pub use core::ByteLevelStreamingDecoder;

pub use core::StreamingDecoder;

pub use core::Tokenizer;

pub use core::TokenizerError;

pub use core::CL100K_BASE_PATTERN;

pub use core::LLAMA3_PATTERN;

pub use core::O200K_BASE_PATTERN;

pub use core::SENTENCEPIECE_PATTERN;

pub use core::pretrained;

pub use core::bos_token_id;

pub use core::cl100k_base_special_tokens;

pub use core::deepseek_v3_special_tokens;

pub use core::eos_token_id;

pub use core::eos_token_id_by_name;

pub use core::from_pretrained;

pub use core::from_vocab;

pub use core::llama3_special_tokens;

pub use core::o200k_base_special_tokens;

pub use core::pad_token_id;

pub use core::pattern;

pub use core::special_tokens;

pub use core::uses_byte_level;

pub use core::PretrainedVocab;

Modules

core

Core tokenization engine for splintr.