chonkier 0.0.2

🦛 Chonkie, now in Rust 🦀: No-nonsense, ultra-fast, ultra-light chunking library
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14

// A module for tokenizers
pub mod base;
pub mod character;

#[cfg(feature = "tokenizers")]
pub mod hftokenizers;

// Export the individual tokenizers as well so that they can be used directly
// without having to use the module path.
pub use base::Tokenizer;
pub use character::CharacterTokenizer;

#[cfg(feature = "tokenizers")]
pub use hftokenizers::HFTokenizer;