second_brain_core/
chunk.rs1pub const MIN_CHUNK_CHARS: usize = 40;
2pub const MAX_CHUNKS: usize = 16;
3
4const SPLITTER_VERSION: u8 = 1;
7
8const ABBREVIATIONS: &[&str] = &["e.g", "i.e", "vs", "cf", "approx"];
10
11pub fn split(content: &str) -> Vec<String> {
21 let mut chunks = merge_short(sentences(content));
22 if chunks.len() > MAX_CHUNKS {
23 chunks = group_evenly(chunks, MAX_CHUNKS);
24 }
25 if chunks.len() < 2 {
26 return Vec::new();
27 }
28 chunks
29}
30
31pub fn content_hash(content: &str) -> String {
35 let mut hash: u64 = 0xcbf2_9ce4_8422_2325;
36 for byte in std::iter::once(SPLITTER_VERSION).chain(content.bytes()) {
37 hash ^= u64::from(byte);
38 hash = hash.wrapping_mul(0x0000_0100_0000_01b3);
39 }
40 format!("{hash:016x}")
41}
42
43fn sentences(content: &str) -> Vec<&str> {
44 let mut out = Vec::new();
45 let mut start = 0;
46 let mut in_code = false;
47 let mut chars = content.char_indices().peekable();
48 while let Some((i, c)) = chars.next() {
49 if c == '`' {
50 in_code = !in_code;
51 continue;
52 }
53 if in_code || !matches!(c, '.' | ';' | '!' | '?') {
54 continue;
55 }
56 let followed_by_space = chars.peek().is_some_and(|(_, next)| next.is_whitespace());
57 if !followed_by_space || (c == '.' && ends_with_abbreviation(&content[start..i])) {
58 continue;
59 }
60 let end = i + c.len_utf8();
61 push_trimmed(&mut out, &content[start..end]);
62 start = end;
63 }
64 push_trimmed(&mut out, &content[start..]);
65 out
66}
67
68fn push_trimmed<'a>(out: &mut Vec<&'a str>, piece: &'a str) {
69 let piece = piece.trim();
70 if !piece.is_empty() {
71 out.push(piece);
72 }
73}
74
75fn ends_with_abbreviation(text: &str) -> bool {
76 let word = text
77 .rsplit(char::is_whitespace)
78 .next()
79 .unwrap_or("")
80 .trim_start_matches(['(', '[', '"']);
81 ABBREVIATIONS
82 .iter()
83 .any(|abbreviation| word.eq_ignore_ascii_case(abbreviation))
84}
85
86fn merge_short(sentences: Vec<&str>) -> Vec<String> {
87 let mut out: Vec<String> = Vec::new();
88 let mut pending = String::new();
89 for sentence in sentences {
90 if !pending.is_empty() {
91 pending.push(' ');
92 }
93 pending.push_str(sentence);
94 if pending.chars().count() >= MIN_CHUNK_CHARS {
95 out.push(std::mem::take(&mut pending));
96 }
97 }
98 if !pending.is_empty() {
99 match out.last_mut() {
100 Some(last) => {
101 last.push(' ');
102 last.push_str(&pending);
103 }
104 None => out.push(pending),
105 }
106 }
107 out
108}
109
110fn group_evenly(chunks: Vec<String>, groups: usize) -> Vec<String> {
111 let n = chunks.len();
112 (0..groups)
113 .map(|g| chunks[g * n / groups..(g + 1) * n / groups].join(" "))
114 .collect()
115}
116
117#[cfg(test)]
118mod tests {
119 use super::*;
120
121 const A: &str = "The drain thread applies outbox rows before it reloads the reader.";
122 const B: &str = "The Pi keeps its daemon under a MemoryMax of 1800M at all times.";
123 const C: &str = "Recall falls back to memory vectors for rows that have no chunks.";
124
125 #[test]
126 fn sentences_split_on_period_and_semicolon() {
127 let content = format!("{A} {B} {}; {C}", B.trim_end_matches('.'));
128 let chunks = split(&content);
129 assert_eq!(
130 chunks,
131 vec![
132 A.to_string(),
133 B.to_string(),
134 format!("{};", B.trim_end_matches('.')),
135 C.to_string()
136 ]
137 );
138 }
139
140 #[test]
141 fn one_sentence_yields_no_chunks() {
142 assert!(split(A).is_empty());
143 assert!(split("").is_empty());
144 assert!(split(&format!("{A} Short tail.")).is_empty());
145 }
146
147 #[test]
148 fn ips_decimals_and_paths_are_never_split() {
149 let content = "The Pi answers at 10.0.0.135 over the LAN and the ratio is 1.5 today. \
150 Its config lives in ~/.second-brain/config.toml beside brain.db now.";
151 let chunks = split(content);
152 assert_eq!(chunks.len(), 2, "{chunks:?}");
153 assert!(chunks[0].contains("10.0.0.135") && chunks[0].contains("1.5 today."));
154 assert!(chunks[1].contains("~/.second-brain/config.toml beside brain.db now."));
155 }
156
157 #[test]
158 fn backticked_terms_are_never_split() {
159 let content = "Run `cargo test -j2; cargo clippy -j2. --workspace` one build at a time. \
160 A second sentence stands here so there are two chunks.";
161 let chunks = split(content);
162 assert_eq!(chunks.len(), 2, "{chunks:?}");
163 assert!(chunks[0].ends_with("one build at a time."));
164 }
165
166 #[test]
167 fn short_fragments_merge_into_the_next_sentence() {
168 let content = format!("Fixed. See below. {A} {B}");
169 let chunks = split(&content);
170 assert_eq!(
171 chunks,
172 vec![format!("Fixed. See below. {A}"), B.to_string()]
173 );
174 }
175
176 #[test]
177 fn a_short_trailing_fragment_merges_into_the_previous_one() {
178 let content = format!("{A} {B} Done.");
179 let chunks = split(&content);
180 assert_eq!(chunks, vec![A.to_string(), format!("{B} Done.")]);
181 }
182
183 #[test]
184 fn common_abbreviations_do_not_end_a_sentence() {
185 let content = format!("Pick a gate, e.g. the rerank gate at 0.30 in the config file. {B}");
186 let chunks = split(&content);
187 assert_eq!(chunks.len(), 2, "{chunks:?}");
188 assert!(chunks[0].starts_with("Pick a gate, e.g. the rerank gate"));
189 }
190
191 #[test]
192 fn chunks_are_capped_and_still_cover_every_sentence() {
193 let sentences: Vec<String> = (0..40)
194 .map(|i| format!("Sentence number {i:02} carries one separate fact about the system."))
195 .collect();
196 let content = sentences.join(" ");
197 let chunks = split(&content);
198 assert_eq!(chunks.len(), MAX_CHUNKS);
199 let joined = chunks.join(" ");
200 for s in &sentences {
201 assert!(joined.contains(s.as_str()), "lost {s}");
202 }
203 }
204}