scone_core/
chunker.rs

1//! Structure-aware chunking over immutable episode content.
2//!
3//! Chunks are contiguous byte spans covering the whole text (spec invariant
4//! I1): they reassemble to exactly the original, so provenance can never be
5//! destroyed in the pipeline (memory/bugs.md P-6). Cuts prefer blank lines
6//! and markdown heading starts; a span that would exceed 2x the target is
7//! hard-split at char boundaries.
8
9/// A contiguous byte range of an episode's content.
10#[derive(Debug, Clone, Copy, PartialEq, Eq)]
11pub struct ChunkSpan {
12    pub start: usize,
13    pub end: usize,
14}
15
16pub fn chunk_text(text: &str, target_bytes: usize) -> Vec<ChunkSpan> {
17    let target = target_bytes.max(1);
18    if text.is_empty() {
19        return Vec::new();
20    }
21
22    // Pass 1: cut at preferred boundaries once the target is reached.
23    let mut soft_spans = Vec::new();
24    let mut chunk_start = 0usize;
25    let mut last_soft: Option<usize> = None;
26    let mut pos = 0usize;
27    for line in text.split_inclusive('\n') {
28        let line_start = pos;
29        pos += line.len();
30        // A heading line starts a new logical section: cut before it.
31        if line.starts_with('#') && line_start > chunk_start {
32            last_soft = Some(line_start);
33        }
34        // A blank line ends a paragraph: cut after it.
35        if line.trim().is_empty() {
36            last_soft = Some(pos);
37        }
38        // Cut only at preferred boundaries; text with none is handled by
39        // the 2x-target hard-split below.
40        if pos - chunk_start >= target
41            && let Some(cut) = last_soft.filter(|s| *s > chunk_start)
42        {
43            soft_spans.push(ChunkSpan {
44                start: chunk_start,
45                end: cut,
46            });
47            chunk_start = cut;
48            last_soft = None;
49        }
50    }
51    if chunk_start < text.len() {
52        soft_spans.push(ChunkSpan {
53            start: chunk_start,
54            end: text.len(),
55        });
56    }
57
58    // Pass 2: hard-split anything still over 2x target at char boundaries.
59    let mut spans = Vec::with_capacity(soft_spans.len());
60    for span in soft_spans {
61        if span.end - span.start <= 2 * target {
62            spans.push(span);
63            continue;
64        }
65        let mut piece_start = span.start;
66        let mut last_boundary = span.start;
67        for (off, ch) in text[span.start..span.end].char_indices() {
68            let abs = span.start + off;
69            if abs - piece_start >= target {
70                spans.push(ChunkSpan {
71                    start: piece_start,
72                    end: abs,
73                });
74                piece_start = abs;
75            }
76            last_boundary = abs + ch.len_utf8();
77        }
78        if piece_start < last_boundary {
79            spans.push(ChunkSpan {
80                start: piece_start,
81                end: span.end,
82            });
83        }
84    }
85    spans
86}
scone_core/chunker.rs

scone_core/
chunker.rs
