second-brain-adapters 0.13.2

Conversation parsers for second-brain: Claude Code and Cursor JSONL adapters
Documentation
use std::path::{Path, PathBuf};
use std::time::SystemTime;

use second_brain_core::schema::{Memory, MemoryType};
use uuid::Uuid;

/// The `source` every vault note is stored under.
pub const SOURCE: &str = "markdown";

/// A note file found in a vault, with the metadata its change check needs.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Note {
    pub path: PathBuf,
    pub modified: SystemTime,
    pub len: u64,
}

#[derive(Debug, Default)]
pub struct Listing {
    pub notes: Vec<Note>,
    pub unreadable: Vec<PathBuf>,
}

/// Every `.md` file under `vault`, skipping hidden files and directories
/// (`.obsidian`, `.trash`, `.git`), paths that are not UTF-8 (notes are keyed
/// by their path as text), and symlinks, so a link back into the vault
/// cannot loop. Failing to list the vault itself is an error; a subdirectory
/// that fails is returned in `unreadable` so its notes are not taken for
/// deleted.
pub fn discover_notes(vault: &Path) -> std::io::Result<Listing> {
    let mut listing = Listing::default();
    let mut dirs = Vec::new();
    list_dir(vault, &mut dirs, &mut listing.notes)?;
    while let Some(dir) = dirs.pop() {
        if list_dir(&dir, &mut dirs, &mut listing.notes).is_err() {
            listing.unreadable.push(dir);
        }
    }
    Ok(listing)
}

fn list_dir(dir: &Path, dirs: &mut Vec<PathBuf>, notes: &mut Vec<Note>) -> std::io::Result<()> {
    for entry in std::fs::read_dir(dir)? {
        let entry = entry?;
        let path = entry.path();
        if path.to_str().is_none() || entry.file_name().to_string_lossy().starts_with('.') {
            continue;
        }
        let file_type = entry.file_type()?;
        if file_type.is_dir() {
            dirs.push(path);
        } else if file_type.is_file() && is_markdown(&path) {
            let meta = entry.metadata()?;
            notes.push(Note {
                path,
                modified: meta.modified()?,
                len: meta.len(),
            });
        }
    }
    Ok(())
}

fn is_markdown(path: &Path) -> bool {
    path.extension()
        .is_some_and(|e| e.eq_ignore_ascii_case("md"))
}

/// The memory a note's text becomes: its file name as a title line, then
/// its body without YAML frontmatter. None when nothing but frontmatter or
/// whitespace is left.
pub fn note_memory(path: &Path, raw: &str, project: Option<&str>) -> Option<Memory> {
    let title = path.file_stem()?.to_string_lossy();
    let body = strip_frontmatter(raw).trim();
    if body.is_empty() {
        return None;
    }
    let mut memory = Memory::new(
        format!("{title}\n\n{body}"),
        MemoryType::Semantic,
        SOURCE.to_string(),
        path.to_string_lossy().to_string(),
    );
    memory.project_path = project.map(str::to_string);
    Some(memory)
}

fn strip_frontmatter(raw: &str) -> &str {
    let raw = raw.strip_prefix('\u{feff}').unwrap_or(raw);
    let Some(rest) = raw
        .strip_prefix("---\n")
        .or_else(|| raw.strip_prefix("---\r\n"))
    else {
        return raw;
    };
    let mut offset = 0;
    for line in rest.split_inclusive('\n') {
        offset += line.len();
        if line.trim_end_matches(['\r', '\n']) == "---" {
            return &rest[offset..];
        }
    }
    raw
}

/// The stored project whose slug is the vault directory's own, under Claude
/// Code's naming (every character but an ASCII letter or digit becomes `-`).
/// Some sources store that slug without its leading `-`, so either form
/// matches and the one holding more memories wins.
pub fn vault_project(vault: &Path, stored: &[(String, u64)]) -> Option<String> {
    let normalized: PathBuf = vault.components().collect();
    let encoded: String = normalized
        .to_string_lossy()
        .chars()
        .map(|c| if c.is_ascii_alphanumeric() { c } else { '-' })
        .collect();
    let bare = encoded.trim_start_matches('-');
    stored
        .iter()
        .filter(|(slug, _)| *slug == encoded || slug == bare)
        .max_by_key(|(_, count)| *count)
        .map(|(slug, _)| slug.clone())
}

/// One conversation per note per machine, so every version of a note hangs
/// off the same node.
pub fn note_conversation_id(machine_id: &str, path: &Path) -> Uuid {
    let name = format!("second-brain:markdown:{machine_id}:{}", path.display());
    Uuid::new_v5(&Uuid::NAMESPACE_URL, name.as_bytes())
}

#[cfg(test)]
mod tests {
    use super::*;

    fn paths(notes: &[Note], root: &Path) -> Vec<String> {
        let mut out: Vec<String> = notes
            .iter()
            .map(|n| {
                n.path
                    .strip_prefix(root)
                    .unwrap()
                    .to_string_lossy()
                    .to_string()
            })
            .collect();
        out.sort();
        out
    }

    #[test]
    fn discovery_finds_markdown_and_skips_hidden_and_other_files() {
        let dir = tempfile::tempdir().unwrap();
        let root = dir.path();
        std::fs::create_dir_all(root.join("sub/deeper")).unwrap();
        std::fs::create_dir_all(root.join(".obsidian")).unwrap();
        std::fs::create_dir_all(root.join(".trash")).unwrap();
        for (name, body) in [
            ("top.md", "a"),
            ("sub/inner.md", "b"),
            ("sub/deeper/UPPER.MD", "c"),
            ("sub/picture.png", "d"),
            ("notes.txt", "e"),
            (".hidden.md", "f"),
            (".obsidian/workspace.md", "g"),
            (".trash/gone.md", "h"),
        ] {
            std::fs::write(root.join(name), body).unwrap();
        }

        let notes = discover_notes(root).unwrap().notes;

        assert_eq!(
            paths(&notes, root),
            vec!["sub/deeper/UPPER.MD", "sub/inner.md", "top.md"]
        );
        let top = notes.iter().find(|n| n.path.ends_with("top.md")).unwrap();
        assert_eq!(top.len, 1);
    }

    #[cfg(unix)]
    #[test]
    fn discovery_does_not_follow_symlinked_directories() {
        let dir = tempfile::tempdir().unwrap();
        let root = dir.path();
        std::fs::write(root.join("a.md"), "a").unwrap();
        std::os::unix::fs::symlink(root, root.join("loop")).unwrap();

        let notes = discover_notes(root).unwrap().notes;

        assert_eq!(paths(&notes, root), vec!["a.md"]);
    }

    #[cfg(unix)]
    #[test]
    fn an_unreadable_subdirectory_is_reported_and_the_rest_is_listed() {
        use std::os::unix::fs::PermissionsExt;
        let dir = tempfile::tempdir().unwrap();
        let root = dir.path();
        std::fs::create_dir_all(root.join("locked")).unwrap();
        std::fs::write(root.join("locked/hidden.md"), "a").unwrap();
        std::fs::write(root.join("open.md"), "b").unwrap();
        let locked = root.join("locked");
        std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o000)).unwrap();

        let listing = discover_notes(root);
        std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o755)).unwrap();

        let listing = listing.unwrap();
        assert_eq!(paths(&listing.notes, root), vec!["open.md"]);
        assert_eq!(listing.unreadable, vec![locked]);
    }

    #[cfg(target_os = "linux")]
    #[test]
    fn a_note_whose_path_is_not_utf8_is_skipped() {
        use std::os::unix::ffi::OsStrExt;
        let dir = tempfile::tempdir().unwrap();
        let root = dir.path();
        let name = std::ffi::OsStr::from_bytes(b"bad\xff.md");
        std::fs::write(root.join(name), "a").unwrap();
        std::fs::write(root.join("good.md"), "b").unwrap();

        let notes = discover_notes(root).unwrap().notes;

        assert_eq!(paths(&notes, root), vec!["good.md"]);
    }

    #[test]
    fn discovery_of_a_missing_vault_is_an_error() {
        let dir = tempfile::tempdir().unwrap();
        assert!(discover_notes(&dir.path().join("absent")).is_err());
    }

    #[test]
    fn a_note_becomes_its_title_then_its_body_without_frontmatter() {
        let raw = "---\ntags: [a, b]\naliases: x\n---\n\n# Heading\n\nBody line.\n";
        let memory = note_memory(Path::new("/v/Release checklist.md"), raw, None).unwrap();

        assert_eq!(
            memory.content,
            "Release checklist\n\n# Heading\n\nBody line."
        );
        assert_eq!(memory.source, SOURCE);
        assert_eq!(memory.source_id, "/v/Release checklist.md");
        assert_eq!(memory.project_path, None);
    }

    #[test]
    fn a_note_without_frontmatter_keeps_its_whole_body() {
        let raw = "plain text\n---\nafter a rule\n";
        let memory = note_memory(Path::new("/v/n.md"), raw, Some("-v")).unwrap();

        assert_eq!(memory.content, "n\n\nplain text\n---\nafter a rule");
        assert_eq!(memory.project_path.as_deref(), Some("-v"));
    }

    #[test]
    fn an_unclosed_frontmatter_block_is_body_text() {
        let raw = "---\nnot closed\nstill body";
        let memory = note_memory(Path::new("/v/n.md"), raw, None).unwrap();

        assert_eq!(memory.content, "n\n\n---\nnot closed\nstill body");
    }

    #[test]
    fn a_note_with_no_body_yields_no_memory() {
        for raw in [
            "",
            "   \n\n",
            "---\ntitle: x\n---\n",
            "---\ntitle: x\n---\n\n  \n",
        ] {
            assert!(
                note_memory(Path::new("/v/n.md"), raw, None).is_none(),
                "{raw:?}"
            );
        }
    }

    #[test]
    fn a_vault_maps_to_the_stored_project_for_its_path() {
        let stored = vec![
            ("-srv-notes-vault".to_string(), 3),
            ("srv-notes-vault".to_string(), 9),
            ("-srv-other".to_string(), 50),
        ];
        assert_eq!(
            vault_project(Path::new("/srv/notes/vault"), &stored).as_deref(),
            Some("srv-notes-vault")
        );
        assert_eq!(
            vault_project(Path::new("/srv/notes/vault/"), &stored[..1]).as_deref(),
            Some("-srv-notes-vault")
        );
    }

    #[test]
    fn a_vault_maps_to_no_project_when_none_is_stored_for_its_path() {
        let stored = vec![("-srv".to_string(), 50), ("-srv-notes".to_string(), 5)];
        assert_eq!(vault_project(Path::new("/srv/notes/vault"), &stored), None);
    }

    #[test]
    fn the_conversation_id_is_stable_per_machine_and_path() {
        let a = note_conversation_id("m1", Path::new("/v/a.md"));
        assert_eq!(a, note_conversation_id("m1", Path::new("/v/a.md")));
        assert_ne!(a, note_conversation_id("m2", Path::new("/v/a.md")));
        assert_ne!(a, note_conversation_id("m1", Path::new("/v/b.md")));
    }
}