Skip to main content

second_brain_adapters/
markdown.rs

1use std::path::{Path, PathBuf};
2use std::time::SystemTime;
3
4use second_brain_core::schema::{Memory, MemoryType};
5use uuid::Uuid;
6
7/// The `source` every vault note is stored under.
8pub const SOURCE: &str = "markdown";
9
10/// A note file found in a vault, with the metadata its change check needs.
11#[derive(Debug, Clone, PartialEq, Eq)]
12pub struct Note {
13    pub path: PathBuf,
14    pub modified: SystemTime,
15    pub len: u64,
16}
17
18#[derive(Debug, Default)]
19pub struct Listing {
20    pub notes: Vec<Note>,
21    pub unreadable: Vec<PathBuf>,
22}
23
24/// Every `.md` file under `vault`, skipping hidden files and directories
25/// (`.obsidian`, `.trash`, `.git`), paths that are not UTF-8 (notes are keyed
26/// by their path as text), and symlinks, so a link back into the vault
27/// cannot loop. Failing to list the vault itself is an error; a subdirectory
28/// that fails is returned in `unreadable` so its notes are not taken for
29/// deleted.
30pub fn discover_notes(vault: &Path) -> std::io::Result<Listing> {
31    let mut listing = Listing::default();
32    let mut dirs = Vec::new();
33    list_dir(vault, &mut dirs, &mut listing.notes)?;
34    while let Some(dir) = dirs.pop() {
35        if list_dir(&dir, &mut dirs, &mut listing.notes).is_err() {
36            listing.unreadable.push(dir);
37        }
38    }
39    Ok(listing)
40}
41
42fn list_dir(dir: &Path, dirs: &mut Vec<PathBuf>, notes: &mut Vec<Note>) -> std::io::Result<()> {
43    for entry in std::fs::read_dir(dir)? {
44        let entry = entry?;
45        let path = entry.path();
46        if path.to_str().is_none() || entry.file_name().to_string_lossy().starts_with('.') {
47            continue;
48        }
49        let file_type = entry.file_type()?;
50        if file_type.is_dir() {
51            dirs.push(path);
52        } else if file_type.is_file() && is_markdown(&path) {
53            let meta = entry.metadata()?;
54            notes.push(Note {
55                path,
56                modified: meta.modified()?,
57                len: meta.len(),
58            });
59        }
60    }
61    Ok(())
62}
63
64fn is_markdown(path: &Path) -> bool {
65    path.extension()
66        .is_some_and(|e| e.eq_ignore_ascii_case("md"))
67}
68
69/// The memory a note's text becomes: its file name as a title line, then
70/// its body without YAML frontmatter. None when nothing but frontmatter or
71/// whitespace is left.
72pub fn note_memory(path: &Path, raw: &str, project: Option<&str>) -> Option<Memory> {
73    let title = path.file_stem()?.to_string_lossy();
74    let body = strip_frontmatter(raw).trim();
75    if body.is_empty() {
76        return None;
77    }
78    let mut memory = Memory::new(
79        format!("{title}\n\n{body}"),
80        MemoryType::Semantic,
81        SOURCE.to_string(),
82        path.to_string_lossy().to_string(),
83    );
84    memory.project_path = project.map(str::to_string);
85    Some(memory)
86}
87
88fn strip_frontmatter(raw: &str) -> &str {
89    let raw = raw.strip_prefix('\u{feff}').unwrap_or(raw);
90    let Some(rest) = raw
91        .strip_prefix("---\n")
92        .or_else(|| raw.strip_prefix("---\r\n"))
93    else {
94        return raw;
95    };
96    let mut offset = 0;
97    for line in rest.split_inclusive('\n') {
98        offset += line.len();
99        if line.trim_end_matches(['\r', '\n']) == "---" {
100            return &rest[offset..];
101        }
102    }
103    raw
104}
105
106/// The stored project whose slug is the vault directory's own, under Claude
107/// Code's naming (every character but an ASCII letter or digit becomes `-`).
108/// Some sources store that slug without its leading `-`, so either form
109/// matches and the one holding more memories wins.
110pub fn vault_project(vault: &Path, stored: &[(String, u64)]) -> Option<String> {
111    let normalized: PathBuf = vault.components().collect();
112    let encoded: String = normalized
113        .to_string_lossy()
114        .chars()
115        .map(|c| if c.is_ascii_alphanumeric() { c } else { '-' })
116        .collect();
117    let bare = encoded.trim_start_matches('-');
118    stored
119        .iter()
120        .filter(|(slug, _)| *slug == encoded || slug == bare)
121        .max_by_key(|(_, count)| *count)
122        .map(|(slug, _)| slug.clone())
123}
124
125/// One conversation per note per machine, so every version of a note hangs
126/// off the same node.
127pub fn note_conversation_id(machine_id: &str, path: &Path) -> Uuid {
128    let name = format!("second-brain:markdown:{machine_id}:{}", path.display());
129    Uuid::new_v5(&Uuid::NAMESPACE_URL, name.as_bytes())
130}
131
132#[cfg(test)]
133mod tests {
134    use super::*;
135
136    fn paths(notes: &[Note], root: &Path) -> Vec<String> {
137        let mut out: Vec<String> = notes
138            .iter()
139            .map(|n| {
140                n.path
141                    .strip_prefix(root)
142                    .unwrap()
143                    .to_string_lossy()
144                    .to_string()
145            })
146            .collect();
147        out.sort();
148        out
149    }
150
151    #[test]
152    fn discovery_finds_markdown_and_skips_hidden_and_other_files() {
153        let dir = tempfile::tempdir().unwrap();
154        let root = dir.path();
155        std::fs::create_dir_all(root.join("sub/deeper")).unwrap();
156        std::fs::create_dir_all(root.join(".obsidian")).unwrap();
157        std::fs::create_dir_all(root.join(".trash")).unwrap();
158        for (name, body) in [
159            ("top.md", "a"),
160            ("sub/inner.md", "b"),
161            ("sub/deeper/UPPER.MD", "c"),
162            ("sub/picture.png", "d"),
163            ("notes.txt", "e"),
164            (".hidden.md", "f"),
165            (".obsidian/workspace.md", "g"),
166            (".trash/gone.md", "h"),
167        ] {
168            std::fs::write(root.join(name), body).unwrap();
169        }
170
171        let notes = discover_notes(root).unwrap().notes;
172
173        assert_eq!(
174            paths(&notes, root),
175            vec!["sub/deeper/UPPER.MD", "sub/inner.md", "top.md"]
176        );
177        let top = notes.iter().find(|n| n.path.ends_with("top.md")).unwrap();
178        assert_eq!(top.len, 1);
179    }
180
181    #[cfg(unix)]
182    #[test]
183    fn discovery_does_not_follow_symlinked_directories() {
184        let dir = tempfile::tempdir().unwrap();
185        let root = dir.path();
186        std::fs::write(root.join("a.md"), "a").unwrap();
187        std::os::unix::fs::symlink(root, root.join("loop")).unwrap();
188
189        let notes = discover_notes(root).unwrap().notes;
190
191        assert_eq!(paths(&notes, root), vec!["a.md"]);
192    }
193
194    #[cfg(unix)]
195    #[test]
196    fn an_unreadable_subdirectory_is_reported_and_the_rest_is_listed() {
197        use std::os::unix::fs::PermissionsExt;
198        let dir = tempfile::tempdir().unwrap();
199        let root = dir.path();
200        std::fs::create_dir_all(root.join("locked")).unwrap();
201        std::fs::write(root.join("locked/hidden.md"), "a").unwrap();
202        std::fs::write(root.join("open.md"), "b").unwrap();
203        let locked = root.join("locked");
204        std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o000)).unwrap();
205
206        let listing = discover_notes(root);
207        std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o755)).unwrap();
208
209        let listing = listing.unwrap();
210        assert_eq!(paths(&listing.notes, root), vec!["open.md"]);
211        assert_eq!(listing.unreadable, vec![locked]);
212    }
213
214    #[cfg(target_os = "linux")]
215    #[test]
216    fn a_note_whose_path_is_not_utf8_is_skipped() {
217        use std::os::unix::ffi::OsStrExt;
218        let dir = tempfile::tempdir().unwrap();
219        let root = dir.path();
220        let name = std::ffi::OsStr::from_bytes(b"bad\xff.md");
221        std::fs::write(root.join(name), "a").unwrap();
222        std::fs::write(root.join("good.md"), "b").unwrap();
223
224        let notes = discover_notes(root).unwrap().notes;
225
226        assert_eq!(paths(&notes, root), vec!["good.md"]);
227    }
228
229    #[test]
230    fn discovery_of_a_missing_vault_is_an_error() {
231        let dir = tempfile::tempdir().unwrap();
232        assert!(discover_notes(&dir.path().join("absent")).is_err());
233    }
234
235    #[test]
236    fn a_note_becomes_its_title_then_its_body_without_frontmatter() {
237        let raw = "---\ntags: [a, b]\naliases: x\n---\n\n# Heading\n\nBody line.\n";
238        let memory = note_memory(Path::new("/v/Release checklist.md"), raw, None).unwrap();
239
240        assert_eq!(
241            memory.content,
242            "Release checklist\n\n# Heading\n\nBody line."
243        );
244        assert_eq!(memory.source, SOURCE);
245        assert_eq!(memory.source_id, "/v/Release checklist.md");
246        assert_eq!(memory.project_path, None);
247    }
248
249    #[test]
250    fn a_note_without_frontmatter_keeps_its_whole_body() {
251        let raw = "plain text\n---\nafter a rule\n";
252        let memory = note_memory(Path::new("/v/n.md"), raw, Some("-v")).unwrap();
253
254        assert_eq!(memory.content, "n\n\nplain text\n---\nafter a rule");
255        assert_eq!(memory.project_path.as_deref(), Some("-v"));
256    }
257
258    #[test]
259    fn an_unclosed_frontmatter_block_is_body_text() {
260        let raw = "---\nnot closed\nstill body";
261        let memory = note_memory(Path::new("/v/n.md"), raw, None).unwrap();
262
263        assert_eq!(memory.content, "n\n\n---\nnot closed\nstill body");
264    }
265
266    #[test]
267    fn a_note_with_no_body_yields_no_memory() {
268        for raw in [
269            "",
270            "   \n\n",
271            "---\ntitle: x\n---\n",
272            "---\ntitle: x\n---\n\n  \n",
273        ] {
274            assert!(
275                note_memory(Path::new("/v/n.md"), raw, None).is_none(),
276                "{raw:?}"
277            );
278        }
279    }
280
281    #[test]
282    fn a_vault_maps_to_the_stored_project_for_its_path() {
283        let stored = vec![
284            ("-srv-notes-vault".to_string(), 3),
285            ("srv-notes-vault".to_string(), 9),
286            ("-srv-other".to_string(), 50),
287        ];
288        assert_eq!(
289            vault_project(Path::new("/srv/notes/vault"), &stored).as_deref(),
290            Some("srv-notes-vault")
291        );
292        assert_eq!(
293            vault_project(Path::new("/srv/notes/vault/"), &stored[..1]).as_deref(),
294            Some("-srv-notes-vault")
295        );
296    }
297
298    #[test]
299    fn a_vault_maps_to_no_project_when_none_is_stored_for_its_path() {
300        let stored = vec![("-srv".to_string(), 50), ("-srv-notes".to_string(), 5)];
301        assert_eq!(vault_project(Path::new("/srv/notes/vault"), &stored), None);
302    }
303
304    #[test]
305    fn the_conversation_id_is_stable_per_machine_and_path() {
306        let a = note_conversation_id("m1", Path::new("/v/a.md"));
307        assert_eq!(a, note_conversation_id("m1", Path::new("/v/a.md")));
308        assert_ne!(a, note_conversation_id("m2", Path::new("/v/a.md")));
309        assert_ne!(a, note_conversation_id("m1", Path::new("/v/b.md")));
310    }
311}