chonkier 0.0.2

🦛 Chonkie, now in Rust 🦀: No-nonsense, ultra-fast, ultra-light chunking library
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13

// Export all the core modules
pub mod chunker;
pub mod tokenizer;
pub mod types;

// Re-export all modules in tokenizer
pub use chunker::{RecursiveChunker, SentenceChunker, TokenChunker};
pub use tokenizer::{CharacterTokenizer, Tokenizer};
pub use types::{Chunk, RecursiveChunk, RecursiveLevel, RecursiveRules, Sentence, SentenceChunk};

// Include all the feature specific modules
#[cfg(feature = "tokenizers")]
pub use tokenizer::HFTokenizer;