pub const TARGET_CHUNK_CHARS: usize = 1800;
#[derive(Debug, Clone)]
pub struct Chunk {
pub index: usize,
pub file: String,
pub line_start: usize,
pub line_end: usize,
pub text: String,
}
pub fn chunk_file(index_start: usize, file: &str, text: &str) -> Vec<Chunk> {
let lines: Vec<&str> = text.lines().collect();
if lines.is_empty() {
return Vec::new();
}
let mut paragraphs: Vec<(usize, usize, String)> = Vec::new();
let mut start: Option<usize> = None;
let mut buf: Vec<&str> = Vec::new();
for (i, line) in lines.iter().enumerate() {
if line.trim().is_empty() {
if let Some(s) = start.take() {
paragraphs.push((s + 1, i, buf.join("\n")));
buf.clear();
}
} else {
if start.is_none() {
start = Some(i);
}
buf.push(line);
}
}
if let Some(s) = start {
paragraphs.push((s + 1, lines.len(), buf.join("\n")));
}
let mut out = Vec::new();
let mut buf_text = String::new();
let mut chunk_start = 0usize;
let mut chunk_end = 0usize;
for (ls, le, text) in paragraphs {
if text.len() > TARGET_CHUNK_CHARS {
if !buf_text.is_empty() {
out.push(make_chunk(
index_start + out.len(),
file,
chunk_start,
chunk_end,
&buf_text,
));
buf_text.clear();
}
for (p_start, p_end, piece) in split_paragraph(&text, ls) {
out.push(make_chunk(
index_start + out.len(),
file,
p_start,
p_end,
&piece,
));
}
continue;
}
if !buf_text.is_empty() && buf_text.len() + text.len() + 2 > TARGET_CHUNK_CHARS {
out.push(make_chunk(
index_start + out.len(),
file,
chunk_start,
chunk_end,
&buf_text,
));
buf_text.clear();
}
if buf_text.is_empty() {
chunk_start = ls;
}
if !buf_text.is_empty() {
buf_text.push_str("\n\n");
}
buf_text.push_str(&text);
chunk_end = le;
}
if !buf_text.is_empty() {
out.push(make_chunk(
index_start + out.len(),
file,
chunk_start,
chunk_end,
&buf_text,
));
}
out
}
fn split_paragraph(text: &str, line_start: usize) -> Vec<(usize, usize, String)> {
let lines: Vec<&str> = text.lines().collect();
let mut pieces = Vec::new();
let mut buf = String::new();
let mut start = line_start;
let mut end = line_start;
for (i, line) in lines.iter().enumerate() {
if !buf.is_empty() && buf.len() + line.len() + 1 > TARGET_CHUNK_CHARS {
pieces.push((start, end, buf.clone()));
buf.clear();
start = line_start + i;
}
if !buf.is_empty() {
buf.push('\n');
}
buf.push_str(line);
end = line_start + i;
}
if !buf.is_empty() {
pieces.push((start, end, buf));
}
pieces
}
fn make_chunk(index: usize, file: &str, line_start: usize, line_end: usize, text: &str) -> Chunk {
Chunk {
index,
file: file.to_string(),
line_start,
line_end,
text: text.to_string(),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn short_file_is_one_chunk() {
let chunks = chunk_file(
0,
"src/lib.rs",
"pub fn greet() -> &'static str {\n \"hi\"\n}\n",
);
assert_eq!(chunks.len(), 1);
assert_eq!(chunks[0].index, 0);
assert_eq!(chunks[0].file, "src/lib.rs");
assert_eq!(chunks[0].line_start, 1);
assert_eq!(chunks[0].line_end, 3);
assert!(chunks[0].text.contains("greet"));
}
#[test]
fn long_file_splits_on_paragraph_boundaries_with_line_ranges() {
let mut body = String::new();
for i in 0..60 {
body.push_str(&format!(
"fn function_{i}() -> u64 {{\n // paragraph number {i} with enough words to matter\n 40 + 2\n}}\n\n"
));
}
let chunks = chunk_file(10, "src/lib.rs", &body);
assert!(
chunks.len() > 3,
"expected multiple chunks, got {}",
chunks.len()
);
assert_eq!(chunks[0].index, 10);
assert_eq!(chunks.last().unwrap().index, 10 + chunks.len() - 1);
for c in &chunks {
assert!(c.line_start >= 1);
assert!(c.line_end >= c.line_start);
assert!(!c.text.trim().is_empty());
}
for w in chunks.windows(2) {
assert!(w[0].line_end <= w[1].line_start);
}
}
#[test]
fn oversized_paragraph_splits_on_lines_never_mid_line() {
let line = "x".repeat(30);
let text = (0..300)
.map(|_| line.clone())
.collect::<Vec<_>>()
.join("\n");
let chunks = chunk_file(0, "gen.rs", &text);
assert!(
chunks.len() > 5,
"expected multiple pieces, got {}",
chunks.len()
);
for c in &chunks {
assert!(
c.text.len() <= TARGET_CHUNK_CHARS + 30,
"piece too large: {}",
c.text.len()
);
assert!(c.text.is_empty() || !c.text.starts_with(' ') || c.text.starts_with("xx"));
}
assert_eq!(chunks[0].line_start, 1);
assert_eq!(chunks.last().unwrap().line_end, 300);
let mut prev_end = 0usize;
for c in &chunks {
assert_eq!(c.line_start, prev_end + 1);
prev_end = c.line_end;
}
}
#[test]
fn empty_file_produces_no_chunks() {
assert!(chunk_file(0, "empty.md", "").is_empty());
assert!(chunk_file(0, "blank.md", "\n\n\n").is_empty());
}
#[test]
fn crlf_files_are_handled() {
let text = "fn a() {}\r\n\r\nfn b() {}\r\n";
let chunks = chunk_file(0, "win.rs", text);
assert_eq!(chunks.len(), 1);
assert_eq!(chunks[0].line_start, 1);
assert_eq!(chunks[0].line_end, 3);
}
}