1use super::algorithm::ChunkingAlgorithm;
2use crate::config::ChunkingStrategy;
3use crate::data::{ChunkView, DataRecord, RecordChunk, RecordSection};
4use crate::tokenizer::{Tokenizer, WhitespaceTokenizer};
5
6#[derive(Clone, Copy, Debug, Default)]
8pub struct SlidingWindowChunker;
9
10impl ChunkingAlgorithm for SlidingWindowChunker {
11 fn materialize(
12 &self,
13 strategy: &ChunkingStrategy,
14 record: &DataRecord,
15 section_idx: usize,
16 section: &RecordSection,
17 ) -> Vec<RecordChunk> {
18 let raw_text = section.text.as_str();
19 let preprocessed_owned: String;
20 let text = if strategy.preprocessors().is_empty() {
21 raw_text
22 } else {
23 let result = strategy
24 .preprocessors()
25 .iter()
26 .try_fold(raw_text.to_string(), |t, p| p.process(&t));
27 match result {
28 Some(s) => {
29 preprocessed_owned = s;
30 preprocessed_owned.as_str()
31 }
32 None => return Vec::new(),
33 }
34 };
35 let tokens: Vec<&str> = WhitespaceTokenizer.tokenize(text);
36 if tokens.is_empty() {
37 return Vec::new();
38 }
39
40 let mut chunks = Vec::new();
41 let total_tokens = tokens.len();
42 let span = strategy.max_window_tokens.min(total_tokens);
43 if span == tokens.len() {
44 let text = text.to_string();
45 chunks.push(RecordChunk {
46 record_id: record.id.clone(),
47 section_idx,
48 view: ChunkView::Window {
49 index: 0,
50 overlap: 0,
51 span,
52 },
53 text,
54 tokens_estimate: span,
55 quality: record.quality,
56 kvp_meta: Default::default(),
57 source: record.source.clone(),
58 embedding: None,
59 });
60 return chunks;
61 }
62
63 for overlap in &strategy.overlap_tokens {
64 let stride = span.saturating_sub(*overlap).max(1);
65 let mut start = 0;
66 let mut index = 0;
67 while start < tokens.len() {
68 let end = (start + span).min(tokens.len());
69 let window = tokens[start..end].join(" ");
70 chunks.push(RecordChunk {
71 record_id: record.id.clone(),
72 section_idx,
73 view: ChunkView::Window {
74 index,
75 overlap: *overlap,
76 span,
77 },
78 text: window,
79 tokens_estimate: end - start,
80 quality: record.quality,
81 kvp_meta: Default::default(),
82 source: record.source.clone(),
83 embedding: None,
84 });
85 if end == tokens.len() {
86 break;
87 }
88 start += stride;
89 index += 1;
90 }
91 }
92
93 if tokens.len() > strategy.max_window_tokens && strategy.summary_fallback_tokens > 0 {
94 let fallback_cap = strategy
95 .summary_fallback_tokens
96 .min(strategy.max_window_tokens)
97 .max(1);
98 let fallback_len = tokens.len().min(fallback_cap);
99 let summary_tokens = tokens
100 .iter()
101 .take(fallback_len)
102 .copied()
103 .collect::<Vec<_>>()
104 .join(" ");
105 chunks.push(RecordChunk {
106 record_id: record.id.clone(),
107 section_idx,
108 view: ChunkView::SummaryFallback {
109 strategy: "head".into(),
110 weight: strategy.summary_fallback_weight,
111 },
112 text: summary_tokens,
113 tokens_estimate: fallback_len,
114 quality: record.quality,
115 kvp_meta: Default::default(),
116 source: record.source.clone(),
117 embedding: None,
118 });
119 }
120
121 chunks
122 }
123}
124
125#[cfg(test)]
126mod tests {
127 use super::*;
128 use crate::data::{QualityScore, RecordSection, SectionRole};
129 use chrono::Utc;
130
131 fn strategy() -> ChunkingStrategy {
132 ChunkingStrategy {
133 max_window_tokens: 4,
134 overlap_tokens: vec![1],
135 summary_fallback_weight: 0.3,
136 summary_fallback_tokens: 2,
137 chunk_weight_floor: 0.0,
138 ..ChunkingStrategy::default()
139 }
140 }
141
142 fn record(text: &str) -> DataRecord {
143 DataRecord {
144 id: "r1".into(),
145 source: "unit".into(),
146 created_at: Utc::now(),
147 updated_at: Utc::now(),
148 quality: QualityScore { trust: 1.0 },
149 taxonomy: vec![],
150 sections: vec![RecordSection {
151 role: SectionRole::Context,
152 heading: None,
153 text: text.into(),
154 sentences: vec![text.into()],
155 token_count: 0,
156 embedding: None,
157 }],
158 meta_prefix: None,
159 label: None,
160 }
161 }
162
163 #[test]
164 fn sliding_window_chunker_materializes_windows_and_summary() {
165 let strategy = strategy();
166 let record = record("one two three four five six seven");
167 let section = &record.sections[0];
168 let chunks = SlidingWindowChunker.materialize(&strategy, &record, 0, section);
169
170 let window_count = chunks
171 .iter()
172 .filter(|chunk| matches!(chunk.view, ChunkView::Window { .. }))
173 .count();
174 let summary_count = chunks
175 .iter()
176 .filter(|chunk| matches!(chunk.view, ChunkView::SummaryFallback { .. }))
177 .count();
178
179 assert_eq!(window_count, 2);
180 assert_eq!(summary_count, 1);
181 }
182
183 #[test]
184 fn denoiser_disabled_produces_chunks_unchanged() {
185 let strategy = strategy();
187 let rec = record("42 524 10788 143 1995 190 394");
188 let section = &rec.sections[0];
189 let chunks = SlidingWindowChunker.materialize(&strategy, &rec, 0, section);
190 assert!(!chunks.is_empty());
191 }
192
193 #[test]
194 fn denoiser_enabled_drops_pure_numeric_section() {
195 use crate::config::DenoiserConfig;
196 use crate::preprocessor::backends::denoiser_preprocessor::DenoiserPreprocessor;
197 let mut strategy = strategy();
198 strategy.register_preprocessor(DenoiserPreprocessor::new(DenoiserConfig {
199 enabled: true,
200 max_digit_ratio: 0.35,
201 strip_markdown: true,
202 }));
203 let rec = record("42 524 10788 143 1995 190 394 13611 358 6444 266");
204 let section = &rec.sections[0];
205 let chunks = SlidingWindowChunker.materialize(&strategy, &rec, 0, section);
206 assert!(
207 chunks.is_empty(),
208 "digit-heavy section should produce no chunks"
209 );
210 }
211
212 #[test]
213 fn denoiser_line_level_strips_noisy_lines_retaining_text() {
214 use crate::config::DenoiserConfig;
215 use crate::preprocessor::backends::denoiser_preprocessor::DenoiserPreprocessor;
216 let mut strategy = strategy();
217 strategy.register_preprocessor(DenoiserPreprocessor::new(DenoiserConfig {
218 enabled: true,
219 max_digit_ratio: 0.35,
220 strip_markdown: true,
221 }));
222 let rec = record("NOVEX INDUSTRIES Springfield\n42 524 10788 143 1995 190 394 13611 358");
223 let section = &rec.sections[0];
224 let chunks = SlidingWindowChunker.materialize(&strategy, &rec, 0, section);
225 assert!(
226 !chunks.is_empty(),
227 "clean line should yield at least one chunk"
228 );
229 let all_text: String = chunks
230 .iter()
231 .map(|c| c.text.as_str())
232 .collect::<Vec<_>>()
233 .join(" ");
234 assert!(all_text.contains("NOVEX") || all_text.contains("Springfield"));
235 }
236}