rlm_rs/chunking/
mod.rs

1//! Chunking strategies for RLM-RS.
2//!
3//! This module provides a trait-based system for chunking text content
4//! into processable segments. Multiple strategies are available:
5//!
6//! - **Fixed**: Simple character-based chunking with configurable size and overlap
7//! - **Semantic**: Unicode-aware chunking respecting sentence/paragraph boundaries
8//! - **Code**: Language-aware chunking at function/class boundaries
9//! - **Parallel**: Orchestrator for parallel chunk processing
10
11pub mod code;
12pub mod fixed;
13pub mod parallel;
14pub mod semantic;
15pub mod traits;
16
17pub use code::CodeChunker;
18pub use fixed::FixedChunker;
19pub use parallel::ParallelChunker;
20pub use semantic::SemanticChunker;
21pub use traits::{ChunkMetadata as ChunkerMetadata, Chunker};
22
23/// Default chunk size in characters (~750 tokens at 4 chars/token).
24/// Sized for granular semantic search with embeddings.
25pub const DEFAULT_CHUNK_SIZE: usize = 3_000;
26
27/// Default overlap size in characters (for context continuity).
28pub const DEFAULT_OVERLAP: usize = 500;
29
30/// Maximum allowed chunk size (24k bytes, ~8k tokens).
31///
32/// Enforced in UTF-8 bytes (chunkers compare against `text.len()` and slice
33/// by byte offsets) — equivalent to characters for ASCII text such as
34/// English prose or most source code. Sized at a conservative ~3 bytes/token
35/// so source-code-heavy chunks stay under BGE-M3's 8192-token embedding
36/// limit and avoid silent truncation on embed (see ADR-010). This is a
37/// heuristic, not a hard guarantee: non-ASCII/CJK-dense text can consume
38/// more tokens per byte than ASCII (see
39/// [`crate::core::chunk::estimate_tokens_for_text`]) and could still exceed
40/// the token limit at this ceiling. A full guarantee would require
41/// token-aware validation at chunk time; see issue #313.
42pub const MAX_CHUNK_SIZE: usize = 24_000;
43
44/// Creates the default chunker (semantic).
45#[must_use]
46pub const fn default_chunker() -> SemanticChunker {
47    SemanticChunker::new()
48}
49
50/// Creates a chunker by name.
51///
52/// # Arguments
53///
54/// * `name` - Chunker strategy name: "fixed", "semantic", "code", or "parallel".
55///
56/// # Returns
57///
58/// A boxed chunker trait object, or an error for unknown strategies.
59///
60/// # Errors
61///
62/// Returns [`crate::error::ChunkingError::UnknownStrategy`] if the strategy name is not recognized.
63pub fn create_chunker(name: &str) -> crate::error::Result<Box<dyn Chunker>> {
64    match name.to_lowercase().as_str() {
65        "fixed" => Ok(Box::new(FixedChunker::new())),
66        "semantic" => Ok(Box::new(SemanticChunker::new())),
67        "code" | "ast" => Ok(Box::new(CodeChunker::new())),
68        "parallel" => Ok(Box::new(ParallelChunker::new(SemanticChunker::new()))),
69        _ => Err(crate::error::ChunkingError::UnknownStrategy {
70            name: name.to_string(),
71        }
72        .into()),
73    }
74}
75
76/// Lists available chunking strategy names.
77#[must_use]
78pub fn available_strategies() -> Vec<&'static str> {
79    vec!["fixed", "semantic", "code", "parallel"]
80}
81
82#[cfg(test)]
83mod tests {
84    use super::*;
85
86    #[test]
87    fn test_default_chunker() {
88        // Test default_chunker function (lines 32-33)
89        let chunker = default_chunker();
90        assert_eq!(chunker.name(), "semantic");
91    }
92
93    #[test]
94    fn test_create_chunker_fixed() {
95        let chunker = create_chunker("fixed").unwrap();
96        assert_eq!(chunker.name(), "fixed");
97    }
98
99    #[test]
100    fn test_create_chunker_semantic() {
101        let chunker = create_chunker("semantic").unwrap();
102        assert_eq!(chunker.name(), "semantic");
103    }
104
105    #[test]
106    fn test_create_chunker_parallel() {
107        let chunker = create_chunker("parallel").unwrap();
108        assert_eq!(chunker.name(), "parallel");
109    }
110
111    #[test]
112    fn test_create_chunker_unknown() {
113        let result = create_chunker("unknown");
114        assert!(result.is_err());
115    }
116
117    #[test]
118    fn test_create_chunker_case_insensitive() {
119        let chunker = create_chunker("FIXED").unwrap();
120        assert_eq!(chunker.name(), "fixed");
121    }
122
123    #[test]
124    fn test_available_strategies() {
125        let strategies = available_strategies();
126        assert_eq!(strategies.len(), 4);
127        assert!(strategies.contains(&"fixed"));
128        assert!(strategies.contains(&"semantic"));
129        assert!(strategies.contains(&"code"));
130        assert!(strategies.contains(&"parallel"));
131    }
132
133    #[test]
134    fn test_create_chunker_code() {
135        let chunker = create_chunker("code").unwrap();
136        assert_eq!(chunker.name(), "code");
137    }
138
139    #[test]
140    fn test_create_chunker_ast_alias() {
141        let chunker = create_chunker("ast").unwrap();
142        assert_eq!(chunker.name(), "code");
143    }
144}
rlm_rs/chunking/mod.rs

rlm_rs/chunking/
mod.rs
