1pub const TARGET_CHUNK_CHARS: usize = 1800;
13
14#[derive(Debug, Clone)]
16pub struct Chunk {
17 pub index: usize,
19 pub file: String,
21 pub line_start: usize,
23 pub line_end: usize,
25 pub text: String,
26}
27
28pub fn chunk_file(index_start: usize, file: &str, text: &str) -> Vec<Chunk> {
31 let lines: Vec<&str> = text.lines().collect();
32 if lines.is_empty() {
33 return Vec::new();
34 }
35
36 let mut paragraphs: Vec<(usize, usize, String)> = Vec::new();
39 let mut start: Option<usize> = None;
40 let mut buf: Vec<&str> = Vec::new();
41 for (i, line) in lines.iter().enumerate() {
42 if line.trim().is_empty() {
43 if let Some(s) = start.take() {
44 paragraphs.push((s + 1, i, buf.join("\n")));
45 buf.clear();
46 }
47 } else {
48 if start.is_none() {
49 start = Some(i);
50 }
51 buf.push(line);
52 }
53 }
54 if let Some(s) = start {
55 paragraphs.push((s + 1, lines.len(), buf.join("\n")));
56 }
57
58 let mut out = Vec::new();
61 let mut buf_text = String::new();
62 let mut chunk_start = 0usize;
63 let mut chunk_end = 0usize;
64 for (ls, le, text) in paragraphs {
65 if text.len() > TARGET_CHUNK_CHARS {
66 if !buf_text.is_empty() {
67 out.push(make_chunk(
68 index_start + out.len(),
69 file,
70 chunk_start,
71 chunk_end,
72 &buf_text,
73 ));
74 buf_text.clear();
75 }
76 for (p_start, p_end, piece) in split_paragraph(&text, ls) {
77 out.push(make_chunk(
78 index_start + out.len(),
79 file,
80 p_start,
81 p_end,
82 &piece,
83 ));
84 }
85 continue;
86 }
87 if !buf_text.is_empty() && buf_text.len() + text.len() + 2 > TARGET_CHUNK_CHARS {
88 out.push(make_chunk(
89 index_start + out.len(),
90 file,
91 chunk_start,
92 chunk_end,
93 &buf_text,
94 ));
95 buf_text.clear();
96 }
97 if buf_text.is_empty() {
98 chunk_start = ls;
99 }
100 if !buf_text.is_empty() {
101 buf_text.push_str("\n\n");
102 }
103 buf_text.push_str(&text);
104 chunk_end = le;
105 }
106 if !buf_text.is_empty() {
107 out.push(make_chunk(
108 index_start + out.len(),
109 file,
110 chunk_start,
111 chunk_end,
112 &buf_text,
113 ));
114 }
115 out
116}
117
118fn split_paragraph(text: &str, line_start: usize) -> Vec<(usize, usize, String)> {
122 let lines: Vec<&str> = text.lines().collect();
123 let mut pieces = Vec::new();
124 let mut buf = String::new();
125 let mut start = line_start;
126 let mut end = line_start;
127 for (i, line) in lines.iter().enumerate() {
128 if !buf.is_empty() && buf.len() + line.len() + 1 > TARGET_CHUNK_CHARS {
129 pieces.push((start, end, buf.clone()));
130 buf.clear();
131 start = line_start + i;
132 }
133 if !buf.is_empty() {
134 buf.push('\n');
135 }
136 buf.push_str(line);
137 end = line_start + i;
138 }
139 if !buf.is_empty() {
140 pieces.push((start, end, buf));
141 }
142 pieces
143}
144
145fn make_chunk(index: usize, file: &str, line_start: usize, line_end: usize, text: &str) -> Chunk {
146 Chunk {
147 index,
148 file: file.to_string(),
149 line_start,
150 line_end,
151 text: text.to_string(),
152 }
153}
154
155#[cfg(test)]
156mod tests {
157 use super::*;
158
159 #[test]
160 fn short_file_is_one_chunk() {
161 let chunks = chunk_file(
162 0,
163 "src/lib.rs",
164 "pub fn greet() -> &'static str {\n \"hi\"\n}\n",
165 );
166 assert_eq!(chunks.len(), 1);
167 assert_eq!(chunks[0].index, 0);
168 assert_eq!(chunks[0].file, "src/lib.rs");
169 assert_eq!(chunks[0].line_start, 1);
170 assert_eq!(chunks[0].line_end, 3);
171 assert!(chunks[0].text.contains("greet"));
172 }
173
174 #[test]
175 fn long_file_splits_on_paragraph_boundaries_with_line_ranges() {
176 let mut body = String::new();
177 for i in 0..60 {
178 body.push_str(&format!(
179 "fn function_{i}() -> u64 {{\n // paragraph number {i} with enough words to matter\n 40 + 2\n}}\n\n"
180 ));
181 }
182 let chunks = chunk_file(10, "src/lib.rs", &body);
183 assert!(
184 chunks.len() > 3,
185 "expected multiple chunks, got {}",
186 chunks.len()
187 );
188 assert_eq!(chunks[0].index, 10);
189 assert_eq!(chunks.last().unwrap().index, 10 + chunks.len() - 1);
190 for c in &chunks {
191 assert!(c.line_start >= 1);
192 assert!(c.line_end >= c.line_start);
193 assert!(!c.text.trim().is_empty());
194 }
195 for w in chunks.windows(2) {
197 assert!(w[0].line_end <= w[1].line_start);
198 }
199 }
200
201 #[test]
202 fn oversized_paragraph_splits_on_lines_never_mid_line() {
203 let line = "x".repeat(30);
205 let text = (0..300)
206 .map(|_| line.clone())
207 .collect::<Vec<_>>()
208 .join("\n");
209 let chunks = chunk_file(0, "gen.rs", &text);
210 assert!(
211 chunks.len() > 5,
212 "expected multiple pieces, got {}",
213 chunks.len()
214 );
215 for c in &chunks {
216 assert!(
217 c.text.len() <= TARGET_CHUNK_CHARS + 30,
218 "piece too large: {}",
219 c.text.len()
220 );
221 assert!(c.text.is_empty() || !c.text.starts_with(' ') || c.text.starts_with("xx"));
223 }
224 assert_eq!(chunks[0].line_start, 1);
226 assert_eq!(chunks.last().unwrap().line_end, 300);
227 let mut prev_end = 0usize;
228 for c in &chunks {
229 assert_eq!(c.line_start, prev_end + 1);
230 prev_end = c.line_end;
231 }
232 }
233
234 #[test]
235 fn empty_file_produces_no_chunks() {
236 assert!(chunk_file(0, "empty.md", "").is_empty());
237 assert!(chunk_file(0, "blank.md", "\n\n\n").is_empty());
238 }
239
240 #[test]
241 fn crlf_files_are_handled() {
242 let text = "fn a() {}\r\n\r\nfn b() {}\r\n";
243 let chunks = chunk_file(0, "win.rs", text);
244 assert_eq!(chunks.len(), 1);
245 assert_eq!(chunks[0].line_start, 1);
246 assert_eq!(chunks[0].line_end, 3);
247 }
248}