1use crate::parse::{Article, Unit, UnitKind};
19
20#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
22pub struct Chunk {
23 pub index: usize,
25 pub url: Option<String>,
27 pub title: Option<String>,
29 pub heading_path: Vec<String>,
31 pub markdown: String,
33 pub text: String,
35 pub tokens: usize,
37 pub units: Vec<usize>,
39}
40
41#[derive(Debug, Clone, Copy)]
43pub struct ChunkConfig {
44 pub target_tokens: usize,
48 pub overlap_tokens: usize,
50 pub min_tokens: usize,
52}
53
54impl Default for ChunkConfig {
55 fn default() -> Self {
56 ChunkConfig { target_tokens: 512, overlap_tokens: 64, min_tokens: 24 }
60 }
61}
62
63pub fn chunk(article: &Article, cfg: &ChunkConfig) -> Vec<Chunk> {
65 let mut out: Vec<Chunk> = Vec::new();
66 let mut run: Vec<&Unit> = Vec::new();
67 let mut tokens = 0usize;
68
69 for unit in &article.units {
70 let starts_section = unit.kind == UnitKind::Heading && !run.is_empty();
71 let would_overrun = tokens + unit.tokens > cfg.target_tokens && !run.is_empty();
72 if starts_section || would_overrun {
73 emit(&mut out, &run, article, cfg);
74 run = tail(&run, cfg.overlap_tokens);
75 tokens = run.iter().map(|u| u.tokens).sum();
76 }
77 tokens += unit.tokens;
78 run.push(unit);
79 }
80 emit(&mut out, &run, article, cfg);
81
82 if out.is_empty() && !article.units.is_empty() {
87 let all: Vec<&Unit> = article.units.iter().collect();
88 emit(&mut out, &all, article, &ChunkConfig { min_tokens: 0, ..*cfg });
89 }
90 out
91}
92
93fn tail<'a>(run: &[&'a Unit], budget: usize) -> Vec<&'a Unit> {
97 if budget == 0 {
98 return Vec::new();
99 }
100 let mut kept = Vec::new();
101 let mut total = 0;
102 for unit in run.iter().rev() {
103 if total + unit.tokens > budget {
104 break;
105 }
106 total += unit.tokens;
107 kept.push(*unit);
108 }
109 kept.reverse();
110 kept
111}
112
113fn emit(out: &mut Vec<Chunk>, run: &[&Unit], article: &Article, cfg: &ChunkConfig) {
114 if run.is_empty() {
115 return;
116 }
117 let tokens: usize = run.iter().map(|u| u.tokens).sum();
118 if tokens < cfg.min_tokens {
119 return;
120 }
121 let first = run[0];
122 out.push(Chunk {
123 index: first.position,
124 url: article.url.clone(),
125 title: article.title().map(str::to_string),
126 heading_path: first.heading_path.clone(),
127 markdown: run.iter().map(|u| u.markdown.as_str()).collect::<Vec<_>>().join("\n\n"),
128 text: run.iter().map(|u| u.text.as_str()).collect::<Vec<_>>().join("\n"),
129 tokens,
130 units: run.iter().map(|u| u.position).collect(),
131 });
132}
133
134#[cfg(test)]
135mod tests {
136 use super::*;
137 use crate::parse::extract;
138
139 fn article(body: &str) -> Article {
140 extract(&format!("<html><body><article>{body}</article></body></html>"), None).unwrap()
141 }
142
143 fn long(word: &str, times: usize) -> String {
144 format!("<p>{}</p>", format!("{word} ").repeat(times))
145 }
146
147 #[test]
148 fn small_units_are_grouped_up() {
149 let body: String =
150 (0..40).map(|i| format!("<p>Sentence number {i} of the body text.</p>")).collect();
151 let doc = article(&body);
152 let chunks = chunk(&doc, &ChunkConfig::default());
153 assert!(chunks.len() < doc.units.len(), "grouping should reduce the count");
154 assert!(chunks.iter().all(|c| c.tokens >= 24));
155 }
156
157 #[test]
158 fn a_heading_opens_a_chunk() {
159 let doc =
160 article(&format!("{}<h2>Second section</h2>{}", long("alpha", 30), long("omega", 30)));
161 let chunks = chunk(&doc, &ChunkConfig { overlap_tokens: 0, ..Default::default() });
162 let opener = chunks.iter().find(|c| c.text.contains("Second section"));
163 assert!(opener.is_some_and(|c| c.text.starts_with("Second section")));
164 }
165
166 #[test]
167 fn an_oversized_unit_is_emitted_whole() {
168 let doc = article(&long("verylongword", 400));
169 let chunks = chunk(&doc, &ChunkConfig { target_tokens: 50, ..Default::default() });
170 assert_eq!(chunks.len(), 1, "a paragraph is never cut mid-sentence");
171 assert!(chunks[0].tokens > 50);
172 }
173
174 #[test]
175 fn overlap_repeats_whole_units_and_zero_means_none() {
176 let body: String = (0..30)
177 .map(|i| format!("<p>Paragraph {i} carrying enough words to count.</p>"))
178 .collect();
179 let doc = article(&body);
180 let with = chunk(
181 &doc,
182 &ChunkConfig { target_tokens: 60, overlap_tokens: 30, ..Default::default() },
183 );
184 let without = chunk(
185 &doc,
186 &ChunkConfig { target_tokens: 60, overlap_tokens: 0, ..Default::default() },
187 );
188 let sum = |cs: &[Chunk]| cs.iter().map(|c| c.tokens).sum::<usize>();
189 assert!(sum(&with) > sum(&without), "overlap repeats text");
190 let seen: Vec<usize> = without.iter().flat_map(|c| c.units.clone()).collect();
191 let mut sorted = seen.clone();
192 sorted.dedup();
193 assert_eq!(seen, sorted, "without overlap no unit appears twice");
194 }
195
196 #[test]
197 fn a_short_article_still_yields_one_chunk() {
198 let doc = article("<p>Short but real, and the only thing on the page.</p>");
200 let chunks = chunk(&doc, &ChunkConfig { min_tokens: 500, ..Default::default() });
201 assert_eq!(chunks.len(), 1);
202 }
203
204 #[test]
205 fn chunks_carry_where_they_came_from() {
206 let doc = extract(
207 "<html><head><title>T</title></head><body><article><h2>H</h2>\
208 <p>Body text long enough to survive the block gate here, and then \
209 some more of it so the chunk clears the minimum.</p></article></body></html>",
210 Some("https://e.com/p"),
211 )
212 .unwrap();
213 let chunks = chunk(&doc, &ChunkConfig::default());
214 assert_eq!(chunks[0].url.as_deref(), Some("https://e.com/p"));
215 assert!(chunks[0].title.is_some());
216 assert!(!chunks[0].units.is_empty());
217 }
218}