second-brain-cli 0.10.0

Universal memory layer for AI coding agents — ingest, recall, and sync across Claude Code, Cursor, and more
//! The concrete identifiers a distillate must not lose. A port of hearth's
//! `scripts/second_brain_tokens.py` (calcifer-lab), which the merge job and
//! hearth's consolidation share; the two must agree token for token so a
//! memory is judged by the same rule whichever consolidator wrote it.

use std::collections::BTreeSet;
use std::sync::LazyLock;

use regex::Regex;

const FILE_EXTENSIONS: &str = "py|rs|ts|tsx|js|jsx|json|jsonl|yaml|yml|toml|md|sql|sh|service|timer|\
     conf|cfg|ini|txt|html|css|db|lock|gguf|env";

static BACKTICK_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"`([^`\n]+)`").unwrap());
static TICKET_RE: LazyLock<Regex> =
    LazyLock::new(|| Regex::new(r"\b[A-Z][A-Z0-9]+-\d+\b").unwrap());
static PATH_RE: LazyLock<Regex> =
    LazyLock::new(|| Regex::new(r"(?:[\w.\-~]*/)+[\w.\-~]+").unwrap());
static FILENAME_RE: LazyLock<Regex> =
    LazyLock::new(|| Regex::new(&format!(r"(?i)\b[\w\-]+\.(?:{FILE_EXTENSIONS})\b")).unwrap());
// The Python rule is `(?<![\w.\-/])\d+(?:\.\d+)?`; the regex crate has no
// lookbehind, so the preceding character is checked by hand against the same
// class. Digits inside a ticket id, path or identifier are covered by that token.
static NUMBER_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\d+(?:\.\d+)?").unwrap());
static NUMBER_BLOCKER_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^[\w.\-/]$").unwrap());
static DIGIT_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^\d$").unwrap());

fn char_matches(re: &Regex, c: char) -> bool {
    re.is_match(c.encode_utf8(&mut [0u8; 4]))
}

fn is_path(token: &str) -> bool {
    if ["/", "~", "./", "../"].iter().any(|p| token.starts_with(p)) {
        return true;
    }
    let last = token.rsplit('/').next().unwrap_or("");
    // "and/or" or "input/output" is prose, not a path.
    last.contains('.') || token.matches('/').count() >= 2
}

fn is_number(token: &str) -> bool {
    let mut digits = token.chars().filter(|c| *c != '.').peekable();
    digits.peek().is_some() && digits.all(|c| char_matches(&DIGIT_RE, c))
}

pub(crate) fn concrete_tokens(text: &str) -> BTreeSet<String> {
    let mut tokens = BTreeSet::new();
    for caps in BACKTICK_RE.captures_iter(text) {
        tokens.insert(caps[1].trim().to_lowercase());
    }
    for m in TICKET_RE.find_iter(text) {
        tokens.insert(m.as_str().to_lowercase());
    }
    for m in PATH_RE.find_iter(text) {
        let path = m.as_str().trim_end_matches(['.', ',', ';', ':']);
        if is_path(path) {
            tokens.insert(path.to_lowercase());
        }
    }
    for m in FILENAME_RE.find_iter(text) {
        tokens.insert(m.as_str().to_lowercase());
    }
    for m in NUMBER_RE.find_iter(text) {
        let blocked = text[..m.start()]
            .chars()
            .next_back()
            .is_some_and(|c| char_matches(&NUMBER_BLOCKER_RE, c));
        // A single digit is usually a list marker or step number that a merge
        // may reword; demanding it would reject good merges.
        if !blocked && m.as_str().chars().count() >= 2 {
            tokens.insert(m.as_str().to_string());
        }
    }
    tokens.remove("");
    tokens
        .iter()
        .filter(|t| is_number(t) || !tokens.iter().any(|o| o != *t && o.contains(t.as_str())))
        .cloned()
        .collect()
}

/// Byte index of the token's first occurrence in lowercased text. A number
/// only counts where it stands alone, so "80" is not found in "8080".
fn find(token: &str, text_norm: &str) -> Option<usize> {
    if !is_number(token) {
        return text_norm.find(token);
    }
    // Overlapping candidates can be skipped: one starting inside a rejected
    // match is preceded by a digit or '.', which the boundary rejects anyway.
    text_norm.match_indices(token).map(|(i, _)| i).find(|&i| {
        let before = text_norm[..i].chars().next_back();
        let after = text_norm[i + token.len()..].chars().next();
        !before.is_some_and(|c| c == '.' || char_matches(&DIGIT_RE, c))
            && !after.is_some_and(|c| char_matches(&DIGIT_RE, c))
    })
}

pub(crate) fn missing_tokens(members: &[&str], merged: &str) -> Vec<String> {
    let merged_norm = merged.to_lowercase().replace('`', "");
    let wanted: BTreeSet<String> = members.iter().flat_map(|m| concrete_tokens(m)).collect();
    wanted
        .into_iter()
        .filter(|t| find(t, &merged_norm).is_none())
        .collect()
}

/// The given tokens (lowercased, as `concrete_tokens` returns them) in order
/// of first appearance in text, each with the casing it has there, so a
/// prompt can list them the way the excerpt wrote them.
pub(crate) fn ordered_identifiers<'a>(
    text: &str,
    tokens: impl IntoIterator<Item = &'a String>,
) -> Vec<String> {
    let text_norm = text.to_lowercase();
    let mut found: Vec<(usize, String)> = Vec::new();
    for token in tokens {
        let Some(byte_index) = find(token, &text_norm) else {
            continue;
        };
        // Positions are counted in chars on both strings, as Python slices
        // the original with an index from the lowercased copy. Lowercasing
        // can change a few non-ASCII letters' length, which shifts the slice;
        // then the lowercased token stands in.
        let index = text_norm[..byte_index].chars().count();
        let original: String = text
            .chars()
            .skip(index)
            .take(token.chars().count())
            .collect();
        let shown = if original.to_lowercase() == *token {
            original
        } else {
            token.clone()
        };
        found.push((index, shown));
    }
    found.sort();
    found.into_iter().map(|(_, shown)| shown).collect()
}

#[cfg(test)]
mod tests {
    use super::*;

    fn set(items: &[&str]) -> BTreeSet<String> {
        items.iter().map(|s| s.to_string()).collect()
    }

    #[test]
    fn concrete_tokens_cover_backticks_tickets_numbers_and_paths() {
        let tokens = concrete_tokens(
            "AUS-7903 inserts rows via `flask migrate` in src/app/config.py, \
             cap 3600s with 60 padding at step 1",
        );
        assert!(
            set(&[
                "aus-7903",
                "flask migrate",
                "src/app/config.py",
                "3600",
                "60"
            ])
            .is_subset(&tokens)
        );
        assert!(
            !tokens.contains("1"),
            "a single digit is usually a list marker, not a fact"
        );
    }

    #[test]
    fn missing_tokens_ignore_backticks_and_case() {
        let members = [
            "Set `MemoryMax=2400M` for AUS-12",
            "the unit file lives in /etc/x.service",
        ];
        let merged = "memorymax=2400M is set for aus-12; see /etc/x.service";
        assert!(missing_tokens(&members, merged).is_empty());
        assert_eq!(
            missing_tokens(&members, "MemoryMax=2400M only"),
            vec!["/etc/x.service", "aus-12"]
        );
    }

    #[test]
    fn ordered_identifiers_follow_first_appearance_with_original_casing() {
        let text = "Ran `Cargo Test` on src/Main.rs for AUS-12, timeout 3600";
        assert_eq!(
            ordered_identifiers(text, &concrete_tokens(text)),
            vec!["Cargo Test", "src/Main.rs", "AUS-12", "3600"]
        );
    }

    #[test]
    fn ordered_identifiers_place_a_number_at_its_own_occurrence() {
        // "80" is inside "8080", but a bare number only counts where it stands
        // alone, the same boundary missing_tokens uses to decide it is present.
        let text = "moved 8080 to 80";
        assert_eq!(
            ordered_identifiers(text, &set(&["80", "8080"])),
            vec!["8080", "80"]
        );
    }

    #[test]
    fn ordered_identifiers_keep_only_the_given_tokens() {
        let text = "AUS-1 then AUS-2 then AUS-3";
        assert_eq!(
            ordered_identifiers(text, &set(&["aus-3", "aus-1"])),
            vec!["AUS-1", "AUS-3"]
        );
    }

    #[test]
    fn a_number_inside_a_word_is_not_a_token() {
        let tokens = concrete_tokens("v12 at x.34 and a-56 but 90");
        assert_eq!(tokens, set(&["90"]));
    }

    #[test]
    fn a_token_inside_a_longer_one_is_dropped_unless_it_is_a_number() {
        let tokens = concrete_tokens("`src/app` lives at src/app/config.py, port 8080 and 80");
        assert_eq!(tokens, set(&["src/app/config.py", "8080", "80"]));
    }

    #[test]
    fn prose_slashes_are_not_paths() {
        assert!(concrete_tokens("read and/or write the input/output").is_empty());
    }

    /// Agreement with hearth's Python on real windows: feed it a file written
    /// by the Python module (a list of {text, concrete, ordered, missing:
    /// [{merged, expected}]}) through SB_TOKENS_PARITY. Ignored by default
    /// because the windows are private transcripts that never enter the repo.
    #[test]
    #[ignore]
    fn tokens_parity() {
        let path = std::env::var("SB_TOKENS_PARITY").expect("SB_TOKENS_PARITY names the file");
        let cases: Vec<serde_json::Value> =
            serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap();
        let strings = |v: &serde_json::Value| -> Vec<String> {
            v.as_array()
                .unwrap()
                .iter()
                .map(|s| s.as_str().unwrap().to_string())
                .collect()
        };
        let mut mismatches = 0;
        for (i, case) in cases.iter().enumerate() {
            let text = case["text"].as_str().unwrap();
            let concrete = concrete_tokens(text);
            let expected: BTreeSet<String> = strings(&case["concrete"]).into_iter().collect();
            if concrete != expected {
                mismatches += 1;
                eprintln!(
                    "case {i} concrete: rust-only {:?} python-only {:?}",
                    concrete.difference(&expected).collect::<Vec<_>>(),
                    expected.difference(&concrete).collect::<Vec<_>>()
                );
            }
            if ordered_identifiers(text, &expected) != strings(&case["ordered"]) {
                mismatches += 1;
                eprintln!("case {i} ordered differs");
            }
            for m in case["missing"].as_array().unwrap() {
                if missing_tokens(&[text], m["merged"].as_str().unwrap()) != strings(&m["expected"])
                {
                    mismatches += 1;
                    eprintln!("case {i} missing differs");
                }
            }
        }
        assert_eq!(
            mismatches,
            0,
            "{mismatches} mismatches over {} cases",
            cases.len()
        );
    }
}