codetether_rlm/chunker/
mod.rs

1//! Semantic chunking for large contexts.
2//!
3//! Splits content intelligently at natural boundaries and prioritizes
4//! chunks for token budget selection.
5
6mod boundaries;
7mod chunk;
8mod compress;
9mod detect;
10mod detect_helpers;
11mod estimate;
12mod hints;
13mod reassemble;
14mod select;
15mod split;
16#[cfg(test)]
17mod tests;
18mod types;
19
20pub use types::{Chunk, ChunkOptions, ChunkType, ContentType};
21
22/// Semantic chunker facade — delegates to focused submodules.
23pub struct RlmChunker;
24
25impl RlmChunker {
26    pub fn detect_content_type(content: &str) -> ContentType {
27        detect::detect_content_type(content)
28    }
29    pub fn get_processing_hints(ct: ContentType) -> &'static str {
30        hints::get_processing_hints(ct)
31    }
32    pub fn estimate_tokens(text: &str) -> usize {
33        estimate::estimate_tokens(text)
34    }
35    pub fn chunk(content: &str, options: Option<ChunkOptions>) -> Vec<Chunk> {
36        chunk::chunk(content, options)
37    }
38    pub fn select_chunks(chunks: &[Chunk], max_tokens: usize) -> Vec<Chunk> {
39        select::select_chunks(chunks, max_tokens)
40    }
41    pub fn reassemble(chunks: &[Chunk]) -> String {
42        reassemble::reassemble(chunks)
43    }
44    pub fn compress(content: &str, max_tokens: usize, options: Option<ChunkOptions>) -> String {
45        compress::compress(content, max_tokens, options)
46    }
47}
codetether_rlm/chunker/mod.rs

codetether_rlm/chunker/
mod.rs
