wordchipper 0.9.2

HPC Rust LLM Tokenizer Library
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16

//! # Vocab Support Tooling

mod pattern_tools;
mod specials_tools;
mod token_list;

pub mod factories;
pub mod testing;
pub mod validators;

#[doc(inline)]
pub use specials_tools::format_carrot;
#[doc(inline)]
pub use specials_tools::format_reserved_carrot;
#[doc(inline)]
pub use token_list::*;