use std::path::{Path, PathBuf};
use std::time::SystemTime;
use second_brain_core::schema::{Memory, MemoryType};
use uuid::Uuid;
pub const SOURCE: &str = "markdown";
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Note {
pub path: PathBuf,
pub modified: SystemTime,
pub len: u64,
}
#[derive(Debug, Default)]
pub struct Listing {
pub notes: Vec<Note>,
pub unreadable: Vec<PathBuf>,
}
pub fn discover_notes(vault: &Path) -> std::io::Result<Listing> {
let mut listing = Listing::default();
let mut dirs = Vec::new();
list_dir(vault, &mut dirs, &mut listing.notes)?;
while let Some(dir) = dirs.pop() {
if list_dir(&dir, &mut dirs, &mut listing.notes).is_err() {
listing.unreadable.push(dir);
}
}
Ok(listing)
}
fn list_dir(dir: &Path, dirs: &mut Vec<PathBuf>, notes: &mut Vec<Note>) -> std::io::Result<()> {
for entry in std::fs::read_dir(dir)? {
let entry = entry?;
let path = entry.path();
if path.to_str().is_none() || entry.file_name().to_string_lossy().starts_with('.') {
continue;
}
let file_type = entry.file_type()?;
if file_type.is_dir() {
dirs.push(path);
} else if file_type.is_file() && is_markdown(&path) {
let meta = entry.metadata()?;
notes.push(Note {
path,
modified: meta.modified()?,
len: meta.len(),
});
}
}
Ok(())
}
fn is_markdown(path: &Path) -> bool {
path.extension()
.is_some_and(|e| e.eq_ignore_ascii_case("md"))
}
pub fn note_memory(path: &Path, raw: &str, project: Option<&str>) -> Option<Memory> {
let title = path.file_stem()?.to_string_lossy();
let body = strip_frontmatter(raw).trim();
if body.is_empty() {
return None;
}
let mut memory = Memory::new(
format!("{title}\n\n{body}"),
MemoryType::Semantic,
SOURCE.to_string(),
path.to_string_lossy().to_string(),
);
memory.project_path = project.map(str::to_string);
Some(memory)
}
fn strip_frontmatter(raw: &str) -> &str {
let raw = raw.strip_prefix('\u{feff}').unwrap_or(raw);
let Some(rest) = raw
.strip_prefix("---\n")
.or_else(|| raw.strip_prefix("---\r\n"))
else {
return raw;
};
let mut offset = 0;
for line in rest.split_inclusive('\n') {
offset += line.len();
if line.trim_end_matches(['\r', '\n']) == "---" {
return &rest[offset..];
}
}
raw
}
pub fn vault_project(vault: &Path, stored: &[(String, u64)]) -> Option<String> {
let normalized: PathBuf = vault.components().collect();
let encoded: String = normalized
.to_string_lossy()
.chars()
.map(|c| if c.is_ascii_alphanumeric() { c } else { '-' })
.collect();
let bare = encoded.trim_start_matches('-');
stored
.iter()
.filter(|(slug, _)| *slug == encoded || slug == bare)
.max_by_key(|(_, count)| *count)
.map(|(slug, _)| slug.clone())
}
pub fn note_conversation_id(machine_id: &str, path: &Path) -> Uuid {
let name = format!("second-brain:markdown:{machine_id}:{}", path.display());
Uuid::new_v5(&Uuid::NAMESPACE_URL, name.as_bytes())
}
#[cfg(test)]
mod tests {
use super::*;
fn paths(notes: &[Note], root: &Path) -> Vec<String> {
let mut out: Vec<String> = notes
.iter()
.map(|n| {
n.path
.strip_prefix(root)
.unwrap()
.to_string_lossy()
.to_string()
})
.collect();
out.sort();
out
}
#[test]
fn discovery_finds_markdown_and_skips_hidden_and_other_files() {
let dir = tempfile::tempdir().unwrap();
let root = dir.path();
std::fs::create_dir_all(root.join("sub/deeper")).unwrap();
std::fs::create_dir_all(root.join(".obsidian")).unwrap();
std::fs::create_dir_all(root.join(".trash")).unwrap();
for (name, body) in [
("top.md", "a"),
("sub/inner.md", "b"),
("sub/deeper/UPPER.MD", "c"),
("sub/picture.png", "d"),
("notes.txt", "e"),
(".hidden.md", "f"),
(".obsidian/workspace.md", "g"),
(".trash/gone.md", "h"),
] {
std::fs::write(root.join(name), body).unwrap();
}
let notes = discover_notes(root).unwrap().notes;
assert_eq!(
paths(¬es, root),
vec!["sub/deeper/UPPER.MD", "sub/inner.md", "top.md"]
);
let top = notes.iter().find(|n| n.path.ends_with("top.md")).unwrap();
assert_eq!(top.len, 1);
}
#[cfg(unix)]
#[test]
fn discovery_does_not_follow_symlinked_directories() {
let dir = tempfile::tempdir().unwrap();
let root = dir.path();
std::fs::write(root.join("a.md"), "a").unwrap();
std::os::unix::fs::symlink(root, root.join("loop")).unwrap();
let notes = discover_notes(root).unwrap().notes;
assert_eq!(paths(¬es, root), vec!["a.md"]);
}
#[cfg(unix)]
#[test]
fn an_unreadable_subdirectory_is_reported_and_the_rest_is_listed() {
use std::os::unix::fs::PermissionsExt;
let dir = tempfile::tempdir().unwrap();
let root = dir.path();
std::fs::create_dir_all(root.join("locked")).unwrap();
std::fs::write(root.join("locked/hidden.md"), "a").unwrap();
std::fs::write(root.join("open.md"), "b").unwrap();
let locked = root.join("locked");
std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o000)).unwrap();
let listing = discover_notes(root);
std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o755)).unwrap();
let listing = listing.unwrap();
assert_eq!(paths(&listing.notes, root), vec!["open.md"]);
assert_eq!(listing.unreadable, vec![locked]);
}
#[cfg(target_os = "linux")]
#[test]
fn a_note_whose_path_is_not_utf8_is_skipped() {
use std::os::unix::ffi::OsStrExt;
let dir = tempfile::tempdir().unwrap();
let root = dir.path();
let name = std::ffi::OsStr::from_bytes(b"bad\xff.md");
std::fs::write(root.join(name), "a").unwrap();
std::fs::write(root.join("good.md"), "b").unwrap();
let notes = discover_notes(root).unwrap().notes;
assert_eq!(paths(¬es, root), vec!["good.md"]);
}
#[test]
fn discovery_of_a_missing_vault_is_an_error() {
let dir = tempfile::tempdir().unwrap();
assert!(discover_notes(&dir.path().join("absent")).is_err());
}
#[test]
fn a_note_becomes_its_title_then_its_body_without_frontmatter() {
let raw = "---\ntags: [a, b]\naliases: x\n---\n\n# Heading\n\nBody line.\n";
let memory = note_memory(Path::new("/v/Release checklist.md"), raw, None).unwrap();
assert_eq!(
memory.content,
"Release checklist\n\n# Heading\n\nBody line."
);
assert_eq!(memory.source, SOURCE);
assert_eq!(memory.source_id, "/v/Release checklist.md");
assert_eq!(memory.project_path, None);
}
#[test]
fn a_note_without_frontmatter_keeps_its_whole_body() {
let raw = "plain text\n---\nafter a rule\n";
let memory = note_memory(Path::new("/v/n.md"), raw, Some("-v")).unwrap();
assert_eq!(memory.content, "n\n\nplain text\n---\nafter a rule");
assert_eq!(memory.project_path.as_deref(), Some("-v"));
}
#[test]
fn an_unclosed_frontmatter_block_is_body_text() {
let raw = "---\nnot closed\nstill body";
let memory = note_memory(Path::new("/v/n.md"), raw, None).unwrap();
assert_eq!(memory.content, "n\n\n---\nnot closed\nstill body");
}
#[test]
fn a_note_with_no_body_yields_no_memory() {
for raw in [
"",
" \n\n",
"---\ntitle: x\n---\n",
"---\ntitle: x\n---\n\n \n",
] {
assert!(
note_memory(Path::new("/v/n.md"), raw, None).is_none(),
"{raw:?}"
);
}
}
#[test]
fn a_vault_maps_to_the_stored_project_for_its_path() {
let stored = vec![
("-srv-notes-vault".to_string(), 3),
("srv-notes-vault".to_string(), 9),
("-srv-other".to_string(), 50),
];
assert_eq!(
vault_project(Path::new("/srv/notes/vault"), &stored).as_deref(),
Some("srv-notes-vault")
);
assert_eq!(
vault_project(Path::new("/srv/notes/vault/"), &stored[..1]).as_deref(),
Some("-srv-notes-vault")
);
}
#[test]
fn a_vault_maps_to_no_project_when_none_is_stored_for_its_path() {
let stored = vec![("-srv".to_string(), 50), ("-srv-notes".to_string(), 5)];
assert_eq!(vault_project(Path::new("/srv/notes/vault"), &stored), None);
}
#[test]
fn the_conversation_id_is_stable_per_machine_and_path() {
let a = note_conversation_id("m1", Path::new("/v/a.md"));
assert_eq!(a, note_conversation_id("m1", Path::new("/v/a.md")));
assert_ne!(a, note_conversation_id("m2", Path::new("/v/a.md")));
assert_ne!(a, note_conversation_id("m1", Path::new("/v/b.md")));
}
}