second-brain-core 0.13.5

Core library for second-brain: SQLite storage with vector and lexical indexes, BGE embeddings, and hybrid retrieval
Documentation
pub const MIN_CHUNK_CHARS: usize = 40;
pub const MAX_CHUNKS: usize = 16;

/// Bump whenever `split` changes what it produces, so every stored chunk set
/// reads as stale and is re-split and re-embedded.
const SPLITTER_VERSION: u8 = 1;

/// Words whose trailing period is part of the word, not a sentence end.
const ABBREVIATIONS: &[&str] = &["e.g", "i.e", "vs", "cf", "approx"];

/// Splits a memory's content into the sentence windows indexed as extra
/// retrieval keys for it. Returns nothing when the content is a single
/// chunk, because the whole-memory vector already covers that case.
///
/// A sentence ends at `.`, `;`, `!` or `?` followed by whitespace, outside
/// backticks. Requiring the whitespace is what keeps IPs, decimals, versions
/// and paths whole. Fragments under `MIN_CHUNK_CHARS` merge into the next
/// sentence (a short tail into the previous one), and more than `MAX_CHUNKS`
/// sentences are grouped evenly so every sentence stays covered.
pub fn split(content: &str) -> Vec<String> {
    let mut chunks = merge_short(sentences(content));
    if chunks.len() > MAX_CHUNKS {
        chunks = group_evenly(chunks, MAX_CHUNKS);
    }
    if chunks.len() < 2 {
        return Vec::new();
    }
    chunks
}

/// FNV-1a over the splitter version and the content. It only has to tell a
/// stored chunk set from the content it was split from, and it must be
/// stable across builds and peers, which std's `DefaultHasher` is not.
pub fn content_hash(content: &str) -> String {
    let mut hash: u64 = 0xcbf2_9ce4_8422_2325;
    for byte in std::iter::once(SPLITTER_VERSION).chain(content.bytes()) {
        hash ^= u64::from(byte);
        hash = hash.wrapping_mul(0x0000_0100_0000_01b3);
    }
    format!("{hash:016x}")
}

fn sentences(content: &str) -> Vec<&str> {
    let mut out = Vec::new();
    let mut start = 0;
    let mut in_code = false;
    let mut chars = content.char_indices().peekable();
    while let Some((i, c)) = chars.next() {
        if c == '`' {
            in_code = !in_code;
            continue;
        }
        if in_code || !matches!(c, '.' | ';' | '!' | '?') {
            continue;
        }
        let followed_by_space = chars.peek().is_some_and(|(_, next)| next.is_whitespace());
        if !followed_by_space || (c == '.' && ends_with_abbreviation(&content[start..i])) {
            continue;
        }
        let end = i + c.len_utf8();
        push_trimmed(&mut out, &content[start..end]);
        start = end;
    }
    push_trimmed(&mut out, &content[start..]);
    out
}

fn push_trimmed<'a>(out: &mut Vec<&'a str>, piece: &'a str) {
    let piece = piece.trim();
    if !piece.is_empty() {
        out.push(piece);
    }
}

fn ends_with_abbreviation(text: &str) -> bool {
    let word = text
        .rsplit(char::is_whitespace)
        .next()
        .unwrap_or("")
        .trim_start_matches(['(', '[', '"']);
    ABBREVIATIONS
        .iter()
        .any(|abbreviation| word.eq_ignore_ascii_case(abbreviation))
}

fn merge_short(sentences: Vec<&str>) -> Vec<String> {
    let mut out: Vec<String> = Vec::new();
    let mut pending = String::new();
    for sentence in sentences {
        if !pending.is_empty() {
            pending.push(' ');
        }
        pending.push_str(sentence);
        if pending.chars().count() >= MIN_CHUNK_CHARS {
            out.push(std::mem::take(&mut pending));
        }
    }
    if !pending.is_empty() {
        match out.last_mut() {
            Some(last) => {
                last.push(' ');
                last.push_str(&pending);
            }
            None => out.push(pending),
        }
    }
    out
}

fn group_evenly(chunks: Vec<String>, groups: usize) -> Vec<String> {
    let n = chunks.len();
    (0..groups)
        .map(|g| chunks[g * n / groups..(g + 1) * n / groups].join(" "))
        .collect()
}

#[cfg(test)]
mod tests {
    use super::*;

    const A: &str = "The drain thread applies outbox rows before it reloads the reader.";
    const B: &str = "The Pi keeps its daemon under a MemoryMax of 1800M at all times.";
    const C: &str = "Recall falls back to memory vectors for rows that have no chunks.";

    #[test]
    fn sentences_split_on_period_and_semicolon() {
        let content = format!("{A} {B} {}; {C}", B.trim_end_matches('.'));
        let chunks = split(&content);
        assert_eq!(
            chunks,
            vec![
                A.to_string(),
                B.to_string(),
                format!("{};", B.trim_end_matches('.')),
                C.to_string()
            ]
        );
    }

    #[test]
    fn one_sentence_yields_no_chunks() {
        assert!(split(A).is_empty());
        assert!(split("").is_empty());
        assert!(split(&format!("{A} Short tail.")).is_empty());
    }

    #[test]
    fn ips_decimals_and_paths_are_never_split() {
        let content = "The Pi answers at 10.0.0.135 over the LAN and the ratio is 1.5 today. \
                       Its config lives in ~/.second-brain/config.toml beside brain.db now.";
        let chunks = split(content);
        assert_eq!(chunks.len(), 2, "{chunks:?}");
        assert!(chunks[0].contains("10.0.0.135") && chunks[0].contains("1.5 today."));
        assert!(chunks[1].contains("~/.second-brain/config.toml beside brain.db now."));
    }

    #[test]
    fn backticked_terms_are_never_split() {
        let content = "Run `cargo test -j2; cargo clippy -j2. --workspace` one build at a time. \
                       A second sentence stands here so there are two chunks.";
        let chunks = split(content);
        assert_eq!(chunks.len(), 2, "{chunks:?}");
        assert!(chunks[0].ends_with("one build at a time."));
    }

    #[test]
    fn short_fragments_merge_into_the_next_sentence() {
        let content = format!("Fixed. See below. {A} {B}");
        let chunks = split(&content);
        assert_eq!(
            chunks,
            vec![format!("Fixed. See below. {A}"), B.to_string()]
        );
    }

    #[test]
    fn a_short_trailing_fragment_merges_into_the_previous_one() {
        let content = format!("{A} {B} Done.");
        let chunks = split(&content);
        assert_eq!(chunks, vec![A.to_string(), format!("{B} Done.")]);
    }

    #[test]
    fn common_abbreviations_do_not_end_a_sentence() {
        let content = format!("Pick a gate, e.g. the rerank gate at 0.30 in the config file. {B}");
        let chunks = split(&content);
        assert_eq!(chunks.len(), 2, "{chunks:?}");
        assert!(chunks[0].starts_with("Pick a gate, e.g. the rerank gate"));
    }

    #[test]
    fn chunks_are_capped_and_still_cover_every_sentence() {
        let sentences: Vec<String> = (0..40)
            .map(|i| format!("Sentence number {i:02} carries one separate fact about the system."))
            .collect();
        let content = sentences.join(" ");
        let chunks = split(&content);
        assert_eq!(chunks.len(), MAX_CHUNKS);
        let joined = chunks.join(" ");
        for s in &sentences {
            assert!(joined.contains(s.as_str()), "lost {s}");
        }
    }
}