Skip to main content

powershell_mcp/
chunk.rs

1//! File chunking — split documentation files into retrieval units with
2//! line provenance.
3//!
4//! Policy: text is cut only on blank-line boundaries, so a chunk never
5//! splits a code block, a function body, or a prose paragraph. Chunks
6//! target roughly TARGET_CHUNK_CHARS characters, about 400-500 tokens. A
7//! single paragraph larger than the target is split on whole-line
8//! boundaries (never mid-line), which keeps minified or generated files
9//! indexable.
10
11/// Target chunk size in characters.
12pub const TARGET_CHUNK_CHARS: usize = 1800;
13
14/// A retrieval unit with provenance: which file, and which lines.
15#[derive(Debug, Clone)]
16pub struct Chunk {
17    /// Global sequential index across the whole project index.
18    pub index: usize,
19    /// Path relative to the project root.
20    pub file: String,
21    /// 1-based inclusive first source line.
22    pub line_start: usize,
23    /// 1-based inclusive last source line.
24    pub line_end: usize,
25    pub text: String,
26}
27
28/// Chunk one file's text. `index_start` is the first global chunk index for
29/// this file, so indices stay sequential across the whole index.
30pub fn chunk_file(index_start: usize, file: &str, text: &str) -> Vec<Chunk> {
31    let lines: Vec<&str> = text.lines().collect();
32    if lines.is_empty() {
33        return Vec::new();
34    }
35
36    // Group consecutive non-blank lines into paragraphs, remembering the
37    // 1-based source line range of each paragraph.
38    let mut paragraphs: Vec<(usize, usize, String)> = Vec::new();
39    let mut start: Option<usize> = None;
40    let mut buf: Vec<&str> = Vec::new();
41    for (i, line) in lines.iter().enumerate() {
42        if line.trim().is_empty() {
43            if let Some(s) = start.take() {
44                paragraphs.push((s + 1, i, buf.join("\n")));
45                buf.clear();
46            }
47        } else {
48            if start.is_none() {
49                start = Some(i);
50            }
51            buf.push(line);
52        }
53    }
54    if let Some(s) = start {
55        paragraphs.push((s + 1, lines.len(), buf.join("\n")));
56    }
57
58    // Greedily pack paragraphs into chunks up to the target size, flushing
59    // on overflow. Oversized paragraphs are hard-split on line boundaries.
60    let mut out = Vec::new();
61    let mut buf_text = String::new();
62    let mut chunk_start = 0usize;
63    let mut chunk_end = 0usize;
64    for (ls, le, text) in paragraphs {
65        if text.len() > TARGET_CHUNK_CHARS {
66            if !buf_text.is_empty() {
67                out.push(make_chunk(
68                    index_start + out.len(),
69                    file,
70                    chunk_start,
71                    chunk_end,
72                    &buf_text,
73                ));
74                buf_text.clear();
75            }
76            for (p_start, p_end, piece) in split_paragraph(&text, ls) {
77                out.push(make_chunk(
78                    index_start + out.len(),
79                    file,
80                    p_start,
81                    p_end,
82                    &piece,
83                ));
84            }
85            continue;
86        }
87        if !buf_text.is_empty() && buf_text.len() + text.len() + 2 > TARGET_CHUNK_CHARS {
88            out.push(make_chunk(
89                index_start + out.len(),
90                file,
91                chunk_start,
92                chunk_end,
93                &buf_text,
94            ));
95            buf_text.clear();
96        }
97        if buf_text.is_empty() {
98            chunk_start = ls;
99        }
100        if !buf_text.is_empty() {
101            buf_text.push_str("\n\n");
102        }
103        buf_text.push_str(&text);
104        chunk_end = le;
105    }
106    if !buf_text.is_empty() {
107        out.push(make_chunk(
108            index_start + out.len(),
109            file,
110            chunk_start,
111            chunk_end,
112            &buf_text,
113        ));
114    }
115    out
116}
117
118/// Split one oversized paragraph into whole-line pieces of at most
119/// TARGET_CHUNK_CHARS. A single line longer than the target stays whole in
120/// its own piece — lines are never cut.
121fn split_paragraph(text: &str, line_start: usize) -> Vec<(usize, usize, String)> {
122    let lines: Vec<&str> = text.lines().collect();
123    let mut pieces = Vec::new();
124    let mut buf = String::new();
125    let mut start = line_start;
126    let mut end = line_start;
127    for (i, line) in lines.iter().enumerate() {
128        if !buf.is_empty() && buf.len() + line.len() + 1 > TARGET_CHUNK_CHARS {
129            pieces.push((start, end, buf.clone()));
130            buf.clear();
131            start = line_start + i;
132        }
133        if !buf.is_empty() {
134            buf.push('\n');
135        }
136        buf.push_str(line);
137        end = line_start + i;
138    }
139    if !buf.is_empty() {
140        pieces.push((start, end, buf));
141    }
142    pieces
143}
144
145fn make_chunk(index: usize, file: &str, line_start: usize, line_end: usize, text: &str) -> Chunk {
146    Chunk {
147        index,
148        file: file.to_string(),
149        line_start,
150        line_end,
151        text: text.to_string(),
152    }
153}
154
155#[cfg(test)]
156mod tests {
157    use super::*;
158
159    #[test]
160    fn short_file_is_one_chunk() {
161        let chunks = chunk_file(
162            0,
163            "src/lib.rs",
164            "pub fn greet() -> &'static str {\n    \"hi\"\n}\n",
165        );
166        assert_eq!(chunks.len(), 1);
167        assert_eq!(chunks[0].index, 0);
168        assert_eq!(chunks[0].file, "src/lib.rs");
169        assert_eq!(chunks[0].line_start, 1);
170        assert_eq!(chunks[0].line_end, 3);
171        assert!(chunks[0].text.contains("greet"));
172    }
173
174    #[test]
175    fn long_file_splits_on_paragraph_boundaries_with_line_ranges() {
176        let mut body = String::new();
177        for i in 0..60 {
178            body.push_str(&format!(
179                "fn function_{i}() -> u64 {{\n    // paragraph number {i} with enough words to matter\n    40 + 2\n}}\n\n"
180            ));
181        }
182        let chunks = chunk_file(10, "src/lib.rs", &body);
183        assert!(
184            chunks.len() > 3,
185            "expected multiple chunks, got {}",
186            chunks.len()
187        );
188        assert_eq!(chunks[0].index, 10);
189        assert_eq!(chunks.last().unwrap().index, 10 + chunks.len() - 1);
190        for c in &chunks {
191            assert!(c.line_start >= 1);
192            assert!(c.line_end >= c.line_start);
193            assert!(!c.text.trim().is_empty());
194        }
195        // Chunks stay in source order.
196        for w in chunks.windows(2) {
197            assert!(w[0].line_end <= w[1].line_start);
198        }
199    }
200
201    #[test]
202    fn oversized_paragraph_splits_on_lines_never_mid_line() {
203        // One huge paragraph with no blank lines: 300 lines of 30 chars.
204        let line = "x".repeat(30);
205        let text = (0..300)
206            .map(|_| line.clone())
207            .collect::<Vec<_>>()
208            .join("\n");
209        let chunks = chunk_file(0, "gen.rs", &text);
210        assert!(
211            chunks.len() > 5,
212            "expected multiple pieces, got {}",
213            chunks.len()
214        );
215        for c in &chunks {
216            assert!(
217                c.text.len() <= TARGET_CHUNK_CHARS + 30,
218                "piece too large: {}",
219                c.text.len()
220            );
221            // No line may be split: every piece boundary is a newline boundary.
222            assert!(c.text.is_empty() || !c.text.starts_with(' ') || c.text.starts_with("xx"));
223        }
224        // Line ranges tile the file exactly.
225        assert_eq!(chunks[0].line_start, 1);
226        assert_eq!(chunks.last().unwrap().line_end, 300);
227        let mut prev_end = 0usize;
228        for c in &chunks {
229            assert_eq!(c.line_start, prev_end + 1);
230            prev_end = c.line_end;
231        }
232    }
233
234    #[test]
235    fn empty_file_produces_no_chunks() {
236        assert!(chunk_file(0, "empty.md", "").is_empty());
237        assert!(chunk_file(0, "blank.md", "\n\n\n").is_empty());
238    }
239
240    #[test]
241    fn crlf_files_are_handled() {
242        let text = "fn a() {}\r\n\r\nfn b() {}\r\n";
243        let chunks = chunk_file(0, "win.rs", text);
244        assert_eq!(chunks.len(), 1);
245        assert_eq!(chunks[0].line_start, 1);
246        assert_eq!(chunks[0].line_end, 3);
247    }
248}