Skip to main content

second_brain_core/
chunk.rs

1pub const MIN_CHUNK_CHARS: usize = 40;
2pub const MAX_CHUNKS: usize = 16;
3
4/// Bump whenever `split` changes what it produces, so every stored chunk set
5/// reads as stale and is re-split and re-embedded.
6const SPLITTER_VERSION: u8 = 1;
7
8/// Words whose trailing period is part of the word, not a sentence end.
9const ABBREVIATIONS: &[&str] = &["e.g", "i.e", "vs", "cf", "approx"];
10
11/// Splits a memory's content into the sentence windows indexed as extra
12/// retrieval keys for it. Returns nothing when the content is a single
13/// chunk, because the whole-memory vector already covers that case.
14///
15/// A sentence ends at `.`, `;`, `!` or `?` followed by whitespace, outside
16/// backticks. Requiring the whitespace is what keeps IPs, decimals, versions
17/// and paths whole. Fragments under `MIN_CHUNK_CHARS` merge into the next
18/// sentence (a short tail into the previous one), and more than `MAX_CHUNKS`
19/// sentences are grouped evenly so every sentence stays covered.
20pub fn split(content: &str) -> Vec<String> {
21    let mut chunks = merge_short(sentences(content));
22    if chunks.len() > MAX_CHUNKS {
23        chunks = group_evenly(chunks, MAX_CHUNKS);
24    }
25    if chunks.len() < 2 {
26        return Vec::new();
27    }
28    chunks
29}
30
31/// FNV-1a over the splitter version and the content. It only has to tell a
32/// stored chunk set from the content it was split from, and it must be
33/// stable across builds and peers, which std's `DefaultHasher` is not.
34pub fn content_hash(content: &str) -> String {
35    let mut hash: u64 = 0xcbf2_9ce4_8422_2325;
36    for byte in std::iter::once(SPLITTER_VERSION).chain(content.bytes()) {
37        hash ^= u64::from(byte);
38        hash = hash.wrapping_mul(0x0000_0100_0000_01b3);
39    }
40    format!("{hash:016x}")
41}
42
43fn sentences(content: &str) -> Vec<&str> {
44    let mut out = Vec::new();
45    let mut start = 0;
46    let mut in_code = false;
47    let mut chars = content.char_indices().peekable();
48    while let Some((i, c)) = chars.next() {
49        if c == '`' {
50            in_code = !in_code;
51            continue;
52        }
53        if in_code || !matches!(c, '.' | ';' | '!' | '?') {
54            continue;
55        }
56        let followed_by_space = chars.peek().is_some_and(|(_, next)| next.is_whitespace());
57        if !followed_by_space || (c == '.' && ends_with_abbreviation(&content[start..i])) {
58            continue;
59        }
60        let end = i + c.len_utf8();
61        push_trimmed(&mut out, &content[start..end]);
62        start = end;
63    }
64    push_trimmed(&mut out, &content[start..]);
65    out
66}
67
68fn push_trimmed<'a>(out: &mut Vec<&'a str>, piece: &'a str) {
69    let piece = piece.trim();
70    if !piece.is_empty() {
71        out.push(piece);
72    }
73}
74
75fn ends_with_abbreviation(text: &str) -> bool {
76    let word = text
77        .rsplit(char::is_whitespace)
78        .next()
79        .unwrap_or("")
80        .trim_start_matches(['(', '[', '"']);
81    ABBREVIATIONS
82        .iter()
83        .any(|abbreviation| word.eq_ignore_ascii_case(abbreviation))
84}
85
86fn merge_short(sentences: Vec<&str>) -> Vec<String> {
87    let mut out: Vec<String> = Vec::new();
88    let mut pending = String::new();
89    for sentence in sentences {
90        if !pending.is_empty() {
91            pending.push(' ');
92        }
93        pending.push_str(sentence);
94        if pending.chars().count() >= MIN_CHUNK_CHARS {
95            out.push(std::mem::take(&mut pending));
96        }
97    }
98    if !pending.is_empty() {
99        match out.last_mut() {
100            Some(last) => {
101                last.push(' ');
102                last.push_str(&pending);
103            }
104            None => out.push(pending),
105        }
106    }
107    out
108}
109
110fn group_evenly(chunks: Vec<String>, groups: usize) -> Vec<String> {
111    let n = chunks.len();
112    (0..groups)
113        .map(|g| chunks[g * n / groups..(g + 1) * n / groups].join(" "))
114        .collect()
115}
116
117#[cfg(test)]
118mod tests {
119    use super::*;
120
121    const A: &str = "The drain thread applies outbox rows before it reloads the reader.";
122    const B: &str = "The Pi keeps its daemon under a MemoryMax of 1800M at all times.";
123    const C: &str = "Recall falls back to memory vectors for rows that have no chunks.";
124
125    #[test]
126    fn sentences_split_on_period_and_semicolon() {
127        let content = format!("{A} {B} {}; {C}", B.trim_end_matches('.'));
128        let chunks = split(&content);
129        assert_eq!(
130            chunks,
131            vec![
132                A.to_string(),
133                B.to_string(),
134                format!("{};", B.trim_end_matches('.')),
135                C.to_string()
136            ]
137        );
138    }
139
140    #[test]
141    fn one_sentence_yields_no_chunks() {
142        assert!(split(A).is_empty());
143        assert!(split("").is_empty());
144        assert!(split(&format!("{A} Short tail.")).is_empty());
145    }
146
147    #[test]
148    fn ips_decimals_and_paths_are_never_split() {
149        let content = "The Pi answers at 10.0.0.135 over the LAN and the ratio is 1.5 today. \
150                       Its config lives in ~/.second-brain/config.toml beside brain.db now.";
151        let chunks = split(content);
152        assert_eq!(chunks.len(), 2, "{chunks:?}");
153        assert!(chunks[0].contains("10.0.0.135") && chunks[0].contains("1.5 today."));
154        assert!(chunks[1].contains("~/.second-brain/config.toml beside brain.db now."));
155    }
156
157    #[test]
158    fn backticked_terms_are_never_split() {
159        let content = "Run `cargo test -j2; cargo clippy -j2. --workspace` one build at a time. \
160                       A second sentence stands here so there are two chunks.";
161        let chunks = split(content);
162        assert_eq!(chunks.len(), 2, "{chunks:?}");
163        assert!(chunks[0].ends_with("one build at a time."));
164    }
165
166    #[test]
167    fn short_fragments_merge_into_the_next_sentence() {
168        let content = format!("Fixed. See below. {A} {B}");
169        let chunks = split(&content);
170        assert_eq!(
171            chunks,
172            vec![format!("Fixed. See below. {A}"), B.to_string()]
173        );
174    }
175
176    #[test]
177    fn a_short_trailing_fragment_merges_into_the_previous_one() {
178        let content = format!("{A} {B} Done.");
179        let chunks = split(&content);
180        assert_eq!(chunks, vec![A.to_string(), format!("{B} Done.")]);
181    }
182
183    #[test]
184    fn common_abbreviations_do_not_end_a_sentence() {
185        let content = format!("Pick a gate, e.g. the rerank gate at 0.30 in the config file. {B}");
186        let chunks = split(&content);
187        assert_eq!(chunks.len(), 2, "{chunks:?}");
188        assert!(chunks[0].starts_with("Pick a gate, e.g. the rerank gate"));
189    }
190
191    #[test]
192    fn chunks_are_capped_and_still_cover_every_sentence() {
193        let sentences: Vec<String> = (0..40)
194            .map(|i| format!("Sentence number {i:02} carries one separate fact about the system."))
195            .collect();
196        let content = sentences.join(" ");
197        let chunks = split(&content);
198        assert_eq!(chunks.len(), MAX_CHUNKS);
199        let joined = chunks.join(" ");
200        for s in &sentences {
201            assert!(joined.contains(s.as_str()), "lost {s}");
202        }
203    }
204}