use crate::parse::{Article, Unit, UnitKind};
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
pub struct Chunk {
pub index: usize,
pub url: Option<String>,
pub title: Option<String>,
pub heading_path: Vec<String>,
pub markdown: String,
pub text: String,
pub tokens: usize,
pub units: Vec<usize>,
}
#[derive(Debug, Clone, Copy)]
pub struct ChunkConfig {
pub target_tokens: usize,
pub overlap_tokens: usize,
pub min_tokens: usize,
}
impl Default for ChunkConfig {
fn default() -> Self {
ChunkConfig { target_tokens: 512, overlap_tokens: 64, min_tokens: 24 }
}
}
pub fn chunk(article: &Article, cfg: &ChunkConfig) -> Vec<Chunk> {
let mut out: Vec<Chunk> = Vec::new();
let mut run: Vec<&Unit> = Vec::new();
let mut tokens = 0usize;
for unit in &article.units {
let starts_section = unit.kind == UnitKind::Heading && !run.is_empty();
let would_overrun = tokens + unit.tokens > cfg.target_tokens && !run.is_empty();
if starts_section || would_overrun {
emit(&mut out, &run, article, cfg);
run = tail(&run, cfg.overlap_tokens);
tokens = run.iter().map(|u| u.tokens).sum();
}
tokens += unit.tokens;
run.push(unit);
}
emit(&mut out, &run, article, cfg);
if out.is_empty() && !article.units.is_empty() {
let all: Vec<&Unit> = article.units.iter().collect();
emit(&mut out, &all, article, &ChunkConfig { min_tokens: 0, ..*cfg });
}
out
}
fn tail<'a>(run: &[&'a Unit], budget: usize) -> Vec<&'a Unit> {
if budget == 0 {
return Vec::new();
}
let mut kept = Vec::new();
let mut total = 0;
for unit in run.iter().rev() {
if total + unit.tokens > budget {
break;
}
total += unit.tokens;
kept.push(*unit);
}
kept.reverse();
kept
}
fn emit(out: &mut Vec<Chunk>, run: &[&Unit], article: &Article, cfg: &ChunkConfig) {
if run.is_empty() {
return;
}
let tokens: usize = run.iter().map(|u| u.tokens).sum();
if tokens < cfg.min_tokens {
return;
}
let first = run[0];
out.push(Chunk {
index: first.position,
url: article.url.clone(),
title: article.title().map(str::to_string),
heading_path: first.heading_path.clone(),
markdown: run.iter().map(|u| u.markdown.as_str()).collect::<Vec<_>>().join("\n\n"),
text: run.iter().map(|u| u.text.as_str()).collect::<Vec<_>>().join("\n"),
tokens,
units: run.iter().map(|u| u.position).collect(),
});
}
#[cfg(test)]
mod tests {
use super::*;
use crate::parse::extract;
fn article(body: &str) -> Article {
extract(&format!("<html><body><article>{body}</article></body></html>"), None).unwrap()
}
fn long(word: &str, times: usize) -> String {
format!("<p>{}</p>", format!("{word} ").repeat(times))
}
#[test]
fn small_units_are_grouped_up() {
let body: String =
(0..40).map(|i| format!("<p>Sentence number {i} of the body text.</p>")).collect();
let doc = article(&body);
let chunks = chunk(&doc, &ChunkConfig::default());
assert!(chunks.len() < doc.units.len(), "grouping should reduce the count");
assert!(chunks.iter().all(|c| c.tokens >= 24));
}
#[test]
fn a_heading_opens_a_chunk() {
let doc =
article(&format!("{}<h2>Second section</h2>{}", long("alpha", 30), long("omega", 30)));
let chunks = chunk(&doc, &ChunkConfig { overlap_tokens: 0, ..Default::default() });
let opener = chunks.iter().find(|c| c.text.contains("Second section"));
assert!(opener.is_some_and(|c| c.text.starts_with("Second section")));
}
#[test]
fn an_oversized_unit_is_emitted_whole() {
let doc = article(&long("verylongword", 400));
let chunks = chunk(&doc, &ChunkConfig { target_tokens: 50, ..Default::default() });
assert_eq!(chunks.len(), 1, "a paragraph is never cut mid-sentence");
assert!(chunks[0].tokens > 50);
}
#[test]
fn overlap_repeats_whole_units_and_zero_means_none() {
let body: String = (0..30)
.map(|i| format!("<p>Paragraph {i} carrying enough words to count.</p>"))
.collect();
let doc = article(&body);
let with = chunk(
&doc,
&ChunkConfig { target_tokens: 60, overlap_tokens: 30, ..Default::default() },
);
let without = chunk(
&doc,
&ChunkConfig { target_tokens: 60, overlap_tokens: 0, ..Default::default() },
);
let sum = |cs: &[Chunk]| cs.iter().map(|c| c.tokens).sum::<usize>();
assert!(sum(&with) > sum(&without), "overlap repeats text");
let seen: Vec<usize> = without.iter().flat_map(|c| c.units.clone()).collect();
let mut sorted = seen.clone();
sorted.dedup();
assert_eq!(seen, sorted, "without overlap no unit appears twice");
}
#[test]
fn a_short_article_still_yields_one_chunk() {
let doc = article("<p>Short but real, and the only thing on the page.</p>");
let chunks = chunk(&doc, &ChunkConfig { min_tokens: 500, ..Default::default() });
assert_eq!(chunks.len(), 1);
}
#[test]
fn chunks_carry_where_they_came_from() {
let doc = extract(
"<html><head><title>T</title></head><body><article><h2>H</h2>\
<p>Body text long enough to survive the block gate here, and then \
some more of it so the chunk clears the minimum.</p></article></body></html>",
Some("https://e.com/p"),
)
.unwrap();
let chunks = chunk(&doc, &ChunkConfig::default());
assert_eq!(chunks[0].url.as_deref(), Some("https://e.com/p"));
assert!(chunks[0].title.is_some());
assert!(!chunks[0].units.is_empty());
}
}