pub const MIN_CHUNK_CHARS: usize = 40;
pub const MAX_CHUNKS: usize = 16;
const SPLITTER_VERSION: u8 = 1;
const ABBREVIATIONS: &[&str] = &["e.g", "i.e", "vs", "cf", "approx"];
pub fn split(content: &str) -> Vec<String> {
let mut chunks = merge_short(sentences(content));
if chunks.len() > MAX_CHUNKS {
chunks = group_evenly(chunks, MAX_CHUNKS);
}
if chunks.len() < 2 {
return Vec::new();
}
chunks
}
pub fn content_hash(content: &str) -> String {
let mut hash: u64 = 0xcbf2_9ce4_8422_2325;
for byte in std::iter::once(SPLITTER_VERSION).chain(content.bytes()) {
hash ^= u64::from(byte);
hash = hash.wrapping_mul(0x0000_0100_0000_01b3);
}
format!("{hash:016x}")
}
fn sentences(content: &str) -> Vec<&str> {
let mut out = Vec::new();
let mut start = 0;
let mut in_code = false;
let mut chars = content.char_indices().peekable();
while let Some((i, c)) = chars.next() {
if c == '`' {
in_code = !in_code;
continue;
}
if in_code || !matches!(c, '.' | ';' | '!' | '?') {
continue;
}
let followed_by_space = chars.peek().is_some_and(|(_, next)| next.is_whitespace());
if !followed_by_space || (c == '.' && ends_with_abbreviation(&content[start..i])) {
continue;
}
let end = i + c.len_utf8();
push_trimmed(&mut out, &content[start..end]);
start = end;
}
push_trimmed(&mut out, &content[start..]);
out
}
fn push_trimmed<'a>(out: &mut Vec<&'a str>, piece: &'a str) {
let piece = piece.trim();
if !piece.is_empty() {
out.push(piece);
}
}
fn ends_with_abbreviation(text: &str) -> bool {
let word = text
.rsplit(char::is_whitespace)
.next()
.unwrap_or("")
.trim_start_matches(['(', '[', '"']);
ABBREVIATIONS
.iter()
.any(|abbreviation| word.eq_ignore_ascii_case(abbreviation))
}
fn merge_short(sentences: Vec<&str>) -> Vec<String> {
let mut out: Vec<String> = Vec::new();
let mut pending = String::new();
for sentence in sentences {
if !pending.is_empty() {
pending.push(' ');
}
pending.push_str(sentence);
if pending.chars().count() >= MIN_CHUNK_CHARS {
out.push(std::mem::take(&mut pending));
}
}
if !pending.is_empty() {
match out.last_mut() {
Some(last) => {
last.push(' ');
last.push_str(&pending);
}
None => out.push(pending),
}
}
out
}
fn group_evenly(chunks: Vec<String>, groups: usize) -> Vec<String> {
let n = chunks.len();
(0..groups)
.map(|g| chunks[g * n / groups..(g + 1) * n / groups].join(" "))
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
const A: &str = "The drain thread applies outbox rows before it reloads the reader.";
const B: &str = "The Pi keeps its daemon under a MemoryMax of 1800M at all times.";
const C: &str = "Recall falls back to memory vectors for rows that have no chunks.";
#[test]
fn sentences_split_on_period_and_semicolon() {
let content = format!("{A} {B} {}; {C}", B.trim_end_matches('.'));
let chunks = split(&content);
assert_eq!(
chunks,
vec![
A.to_string(),
B.to_string(),
format!("{};", B.trim_end_matches('.')),
C.to_string()
]
);
}
#[test]
fn one_sentence_yields_no_chunks() {
assert!(split(A).is_empty());
assert!(split("").is_empty());
assert!(split(&format!("{A} Short tail.")).is_empty());
}
#[test]
fn ips_decimals_and_paths_are_never_split() {
let content = "The Pi answers at 10.0.0.135 over the LAN and the ratio is 1.5 today. \
Its config lives in ~/.second-brain/config.toml beside brain.db now.";
let chunks = split(content);
assert_eq!(chunks.len(), 2, "{chunks:?}");
assert!(chunks[0].contains("10.0.0.135") && chunks[0].contains("1.5 today."));
assert!(chunks[1].contains("~/.second-brain/config.toml beside brain.db now."));
}
#[test]
fn backticked_terms_are_never_split() {
let content = "Run `cargo test -j2; cargo clippy -j2. --workspace` one build at a time. \
A second sentence stands here so there are two chunks.";
let chunks = split(content);
assert_eq!(chunks.len(), 2, "{chunks:?}");
assert!(chunks[0].ends_with("one build at a time."));
}
#[test]
fn short_fragments_merge_into_the_next_sentence() {
let content = format!("Fixed. See below. {A} {B}");
let chunks = split(&content);
assert_eq!(
chunks,
vec![format!("Fixed. See below. {A}"), B.to_string()]
);
}
#[test]
fn a_short_trailing_fragment_merges_into_the_previous_one() {
let content = format!("{A} {B} Done.");
let chunks = split(&content);
assert_eq!(chunks, vec![A.to_string(), format!("{B} Done.")]);
}
#[test]
fn common_abbreviations_do_not_end_a_sentence() {
let content = format!("Pick a gate, e.g. the rerank gate at 0.30 in the config file. {B}");
let chunks = split(&content);
assert_eq!(chunks.len(), 2, "{chunks:?}");
assert!(chunks[0].starts_with("Pick a gate, e.g. the rerank gate"));
}
#[test]
fn chunks_are_capped_and_still_cover_every_sentence() {
let sentences: Vec<String> = (0..40)
.map(|i| format!("Sentence number {i:02} carries one separate fact about the system."))
.collect();
let content = sentences.join(" ");
let chunks = split(&content);
assert_eq!(chunks.len(), MAX_CHUNKS);
let joined = chunks.join(" ");
for s in &sentences {
assert!(joined.contains(s.as_str()), "lost {s}");
}
}
}