use crate::index::{self, Entry};
use crate::notes;
use std::collections::HashSet;
use std::fs;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::mpsc::{Receiver, TryRecvError};
use std::sync::Arc;
use std::time::SystemTime;
#[derive(Clone, Debug, PartialEq)]
pub struct Mention {
pub path: PathBuf,
pub name: String,
pub excerpt: String,
pub link: (usize, usize),
pub count: usize,
}
#[derive(Clone, Debug, PartialEq)]
pub struct Hit {
pub line: usize,
pub target: String,
pub excerpt: String,
pub link: (usize, usize),
}
const MAX_BODY_BYTES: u64 = 256 * 1024;
const MAX_MENTIONS: usize = 50;
const BEFORE_LINK: usize = 60;
const AFTER_LINK: usize = 120;
pub fn target_entry(path: &Path, title: &str, roots: &[PathBuf]) -> Entry {
Entry {
title: title.to_string(),
rel: rel_under(path, roots),
folder: String::new(),
modified: SystemTime::UNIX_EPOCH,
path: path.to_path_buf(),
}
}
fn rel_under(path: &Path, roots: &[PathBuf]) -> String {
for root in roots {
if let Ok(rel) = path.strip_prefix(root) {
return rel.to_string_lossy().into_owned();
}
}
index::short(path)
}
pub fn mentions_in(body: &str, names: &[String]) -> Vec<Hit> {
let mut out = Vec::new();
let mut fenced = false;
for (line_no, line) in notes::body_after_front_matter(body).lines().enumerate() {
let trimmed = line.trim_start();
if trimmed.starts_with("```") || trimmed.starts_with("~~~") {
fenced = !fenced;
continue;
}
if fenced {
continue;
}
for w in crate::md::wikilinks(line) {
if names.contains(&crate::md::link_key(&w.target)) {
let (excerpt, link) = excerpt(line, w.start, w.end);
out.push(Hit {
line: line_no,
target: w.target.clone(),
excerpt,
link,
});
}
}
}
out
}
pub fn excerpt(line: &str, at: usize, end: usize) -> (String, (usize, usize)) {
let mut chars: Vec<char> = line.chars().collect();
let mut at = at.min(chars.len());
let mut end = end.min(chars.len());
if line.trim_start().starts_with('|') {
let prev = chars[..at]
.iter()
.rposition(|c| *c == '|')
.map_or(0, |p| p + 1);
let next = chars[end..]
.iter()
.position(|c| *c == '|')
.map_or(chars.len(), |p| end + p);
chars = chars[prev..next].to_vec();
at -= prev;
end -= prev;
} else {
let text: String = chars.iter().collect();
let stripped = strip_markers(&text);
let removed = chars.len() - stripped.chars().count();
if removed <= at {
chars = stripped.chars().collect();
at -= removed;
end -= removed;
}
}
let mut out: Vec<char> = Vec::with_capacity(chars.len());
let (mut new_at, mut new_end) = (0, out.len());
for (i, c) in chars.iter().enumerate() {
if i == at {
new_at = out.len();
}
if i == end {
new_end = out.len();
}
if c.is_whitespace() {
if !out.is_empty() && out.last() != Some(&' ') {
out.push(' ');
}
} else {
out.push(*c);
}
}
if end >= chars.len() {
new_end = out.len();
}
while out.last() == Some(&' ') {
out.pop();
}
let new_end = new_end.min(out.len());
let start = new_at.saturating_sub(BEFORE_LINK);
let stop = (new_end + AFTER_LINK).min(out.len());
let mut text = String::new();
let mut shift = 0;
if start > 0 {
text.push('…');
shift = 1;
}
text.extend(out[start..stop].iter());
if stop < out.len() {
text.push('…');
}
(text, (new_at - start + shift, new_end - start + shift))
}
fn strip_markers(s: &str) -> String {
let t = s.trim_start();
let t = t.trim_start_matches('#').trim_start();
let t = ["- ", "* ", "+ ", "> "]
.iter()
.find_map(|m| t.strip_prefix(m))
.unwrap_or(t);
let t = ["[ ] ", "[x] ", "[X] "]
.iter()
.find_map(|m| t.strip_prefix(m))
.unwrap_or(t);
let t = match t.strip_prefix("[!") {
Some(rest) => rest
.split_once(']')
.map(|(_, after)| after.trim_start())
.unwrap_or(t),
None => t,
};
t.to_string()
}
pub fn scan(target: &Entry, roots: &[PathBuf], cancel: &AtomicBool) -> Vec<Mention> {
let names = index::link_keys(target);
let mut seen: HashSet<PathBuf> = HashSet::new();
let mut entries: Vec<Entry> = Vec::new();
let mut found: Vec<(PathBuf, SystemTime, Vec<Hit>)> = Vec::new();
let mut files = 0usize;
for root in roots {
let root = fs::canonicalize(root).unwrap_or_else(|_| root.clone());
let mut stack = vec![(root.clone(), 0usize)];
while let Some((dir, depth)) = stack.pop() {
if depth > index::MAX_DEPTH || files >= index::MAX_FILES {
continue;
}
let Ok(read) = fs::read_dir(&dir) else {
continue;
};
for entry in read.flatten() {
if cancel.load(Ordering::Relaxed) {
return Vec::new();
}
let path = entry.path();
let Some(name) = path.file_name().and_then(|n| n.to_str()) else {
continue;
};
if entry.file_type().map(|t| t.is_dir()).unwrap_or(false) {
if !index::skip_dir(name) {
stack.push((path, depth + 1));
}
continue;
}
if name.starts_with('.') || !name.ends_with(".md") {
continue;
}
if files >= index::MAX_FILES || !seen.insert(path.clone()) {
continue;
}
let meta = entry.metadata().ok();
files += 1;
let modified = meta
.as_ref()
.and_then(|m| m.modified().ok())
.unwrap_or(SystemTime::UNIX_EPOCH);
let too_big = meta.as_ref().is_some_and(|m| m.len() > MAX_BODY_BYTES);
let body = if too_big {
None
} else {
fs::read_to_string(&path).ok()
};
let title = match &body {
Some(b) => notes::title_of(b),
None => index::title_at(&path),
};
entries.push(Entry {
path: path.clone(),
title,
rel: rel_under(&path, std::slice::from_ref(&root)),
folder: String::new(),
modified,
});
let Some(body) = body else {
continue;
};
if path == target.path {
continue;
}
let hits = mentions_in(&body, &names);
if !hits.is_empty() {
found.push((path, modified, hits));
}
}
}
}
if !entries.iter().any(|e| e.path == target.path) {
entries.push(target.clone());
}
let mut out: Vec<(SystemTime, Mention)> = Vec::new();
let mut verdict: std::collections::HashMap<String, bool> = std::collections::HashMap::new();
for (path, modified, hits) in found {
let mut kept = hits.iter().filter(|h| {
let key = crate::md::link_key(&h.target);
*verdict.entry(key).or_insert_with(|| {
index::resolve(&entries, &h.target).is_some_and(|e| e.path == target.path)
})
});
let Some(first) = kept.next() else {
continue;
};
let count = 1 + kept.count();
let name = path
.file_stem()
.map(|s| s.to_string_lossy().into_owned())
.unwrap_or_default();
out.push((
modified,
Mention {
path,
name,
excerpt: first.excerpt.clone(),
link: first.link,
count,
},
));
}
out.sort_by(|(ma, a), (mb, b)| mb.cmp(ma).then_with(|| a.path.cmp(&b.path)));
out.truncate(MAX_MENTIONS);
out.into_iter().map(|(_, m)| m).collect()
}
pub struct Pending {
rx: Receiver<Vec<Mention>>,
cancel: Arc<AtomicBool>,
}
impl Drop for Pending {
fn drop(&mut self) {
self.cancel.store(true, Ordering::Relaxed);
}
}
pub fn spawn(target: Entry, roots: Vec<PathBuf>) -> Pending {
let (tx, rx) = std::sync::mpsc::channel();
let cancel = Arc::new(AtomicBool::new(false));
let flag = Arc::clone(&cancel);
std::thread::spawn(move || {
let _ = tx.send(scan(&target, &roots, &flag));
});
Pending { rx, cancel }
}
#[derive(Default)]
pub struct Backlinks {
key: Option<Key>,
rows: Vec<Mention>,
pending: Option<(Key, Pending)>,
generation: u64,
}
type Key = (PathBuf, u64);
impl Backlinks {
pub fn invalidate(&mut self) {
self.generation = self.generation.wrapping_add(1);
}
pub fn rows_for(
&mut self,
path: &Path,
start: impl FnOnce() -> (Entry, Vec<PathBuf>),
) -> &[Mention] {
let key = (path.to_path_buf(), self.generation);
if self.key.as_ref() == Some(&key) {
return &self.rows;
}
if let Some((pending, scan)) = self.pending.take() {
if pending == key {
match scan.rx.try_recv() {
Ok(rows) => {
self.rows = rows;
self.key = Some(key);
}
Err(TryRecvError::Empty) => {
self.pending = Some((pending, scan));
return &[];
}
Err(TryRecvError::Disconnected) => {
self.rows = Vec::new();
self.key = Some(key);
}
}
return &self.rows;
}
}
let (target, roots) = start();
self.pending = Some((key, spawn(target, roots)));
&[]
}
}
#[cfg(test)]
mod tests {
use super::*;
fn scan(target: &Entry, roots: &[PathBuf]) -> Vec<Mention> {
super::scan(target, roots, &AtomicBool::new(false))
}
fn tmpdir(name: &str) -> PathBuf {
let dir = std::env::temp_dir().join(format!("catcher-mentions-{name}"));
let _ = fs::remove_dir_all(&dir);
fs::create_dir_all(&dir).unwrap();
fs::canonicalize(&dir).unwrap()
}
fn write(dir: &Path, rel: &str, body: &str) -> PathBuf {
let path = dir.join(rel);
if let Some(parent) = path.parent() {
fs::create_dir_all(parent).unwrap();
}
fs::write(&path, body).unwrap();
path
}
fn target(dir: &Path, rel: &str, title: &str) -> Entry {
target_entry(
&dir.join(rel),
title,
std::slice::from_ref(&dir.to_path_buf()),
)
}
fn names(of: &[&str]) -> Vec<String> {
of.iter().map(|n| crate::md::link_key(n)).collect()
}
#[test]
fn a_wikilink_that_resolves_to_the_note_is_found_and_one_that_does_not_is_ignored() {
let hits = mentions_in(
"see [[story-matrix]] and [[something-else]]\n",
&names(&["story-matrix"]),
);
assert_eq!(hits.len(), 1);
assert_eq!(hits[0].target, "story-matrix");
assert_eq!(hits[0].line, 0);
}
#[test]
fn an_aliased_link_counts_against_the_note_it_points_at_not_the_alias() {
let body = "pulled from [[stories/story-matrix|the matrix]]\n";
assert_eq!(
mentions_in(body, &names(&["stories/story-matrix"])).len(),
1
);
assert!(mentions_in(body, &names(&["the matrix"])).is_empty());
}
#[test]
fn a_wikilink_inside_a_fenced_code_block_is_source_not_a_link() {
let body = "before [[spec]]\n```\nnot a link: [[spec]]\n```\nafter [[spec]]\n";
let hits = mentions_in(body, &names(&["spec"]));
assert_eq!(hits.len(), 2);
assert_eq!(hits[1].line, 4);
}
#[test]
fn front_matter_is_stepped_over_rather_than_scanned_for_mentions() {
let body = "---\nsee: \"[[spec]]\"\n---\nbody mentions [[spec]]\n";
let hits = mentions_in(body, &names(&["spec"]));
assert_eq!(hits.len(), 1);
assert_eq!(hits[0].line, 0);
}
fn excerpt_of(line: &str) -> (String, (usize, usize)) {
let w = &crate::md::wikilinks(line)[0];
excerpt(line, w.start, w.end)
}
fn spanned(e: &str, (a, b): (usize, usize)) -> String {
e.chars().skip(a).take(b - a).collect()
}
#[test]
fn the_excerpt_is_centred_on_the_link_and_says_where_it_was_cut() {
let far = "x".repeat(150);
let line = format!("{far} before [[spec]] after {far}");
let (e, span) = excerpt_of(&line);
assert!(e.starts_with('…') && e.ends_with('…'));
assert_eq!(spanned(&e, span), "[[spec]]");
assert_eq!(e.chars().count(), 1 + BEFORE_LINK + 8 + AFTER_LINK + 1);
let (e, span) = excerpt_of("see [[spec]] for the rest");
assert_eq!(e, "see [[spec]] for the rest");
assert_eq!(spanned(&e, span), "[[spec]]");
}
#[test]
fn the_markers_a_line_is_drawn_with_are_not_part_of_the_excerpt() {
let (e, span) = excerpt_of("- see [[spec]] for the rest");
assert_eq!(e, "see [[spec]] for the rest");
assert_eq!(spanned(&e, span), "[[spec]]");
let (e, _) = excerpt_of("> [!summary] TL;DR of [[spec]]");
assert_eq!(e, "TL;DR of [[spec]]");
let (e, _) = excerpt_of(" ## about [[spec]]");
assert_eq!(e, "about [[spec]]");
let (e, span) = excerpt_of("**Projects:** [[spec]]; more");
assert_eq!(e, "**Projects:** [[spec]]; more");
assert_eq!(spanned(&e, span), "[[spec]]");
}
#[test]
fn a_table_row_is_reduced_to_the_cell_holding_the_link() {
let (e, span) = excerpt_of("| Projects | see [[spec]] here | tight deadline |");
assert_eq!(e, "see [[spec]] here");
assert_eq!(spanned(&e, span), "[[spec]]");
}
#[test]
fn runs_of_whitespace_collapse_without_losing_the_link() {
let (e, span) = excerpt_of("a lot\tof space [[spec|the spec]] here");
assert_eq!(e, "a lot of space [[spec|the spec]] here");
assert_eq!(spanned(&e, span), "[[spec|the spec]]");
}
#[test]
fn several_mentions_in_one_note_collapse_to_one_row_with_the_first_excerpt_and_a_count() {
let dir = tmpdir("collapse");
write(&dir, "spec.md", "# Spec\n");
write(
&dir,
"meta.md",
"# Meta\nsee [[spec]] for the shape.\nand again [[spec]] later.\n",
);
let rows = scan(&target(&dir, "spec.md", "Spec"), std::slice::from_ref(&dir));
assert_eq!(rows.len(), 1);
assert_eq!(rows[0].name, "meta");
assert_eq!(rows[0].count, 2);
assert_eq!(rows[0].excerpt, "see [[spec]] for the shape.");
let _ = fs::remove_dir_all(&dir);
}
#[test]
fn a_note_that_links_to_itself_is_not_a_mention_of_itself() {
let dir = tmpdir("self");
write(&dir, "spec.md", "# Spec\nsee [[spec]], which is here.\n");
let rows = scan(&target(&dir, "spec.md", "Spec"), std::slice::from_ref(&dir));
assert!(rows.is_empty());
let _ = fs::remove_dir_all(&dir);
}
#[test]
fn a_note_nothing_links_to_produces_no_mentions_at_all() {
let dir = tmpdir("none");
write(&dir, "spec.md", "# Spec\n");
write(&dir, "other.md", "# Other\nnothing to say\n");
assert!(scan(&target(&dir, "spec.md", "Spec"), std::slice::from_ref(&dir)).is_empty());
let _ = fs::remove_dir_all(&dir);
}
#[test]
fn the_scan_reaches_subfolders_and_skips_the_dirs_quick_open_skips() {
let dir = tmpdir("walk");
write(&dir, "spec.md", "# Spec\n");
write(&dir, "deep/inner/note.md", "# Inner\nabout [[spec]].\n");
write(&dir, ".obsidian/cache.md", "# Cache\n[[spec]]\n");
write(&dir, "node_modules/readme.md", "# Dep\n[[spec]]\n");
let rows = scan(&target(&dir, "spec.md", "Spec"), std::slice::from_ref(&dir));
let names: Vec<&str> = rows.iter().map(|m| m.name.as_str()).collect();
assert_eq!(names, vec!["note"]);
let _ = fs::remove_dir_all(&dir);
}
#[test]
fn a_body_over_the_size_cap_is_skipped_rather_than_read() {
let dir = tmpdir("huge");
write(&dir, "spec.md", "# Spec\n");
let filler = "x".repeat(MAX_BODY_BYTES as usize + 1);
write(&dir, "dump.md", &format!("# Dump\n[[spec]]\n{filler}"));
assert!(scan(&target(&dir, "spec.md", "Spec"), std::slice::from_ref(&dir)).is_empty());
let _ = fs::remove_dir_all(&dir);
}
#[test]
fn a_note_too_big_to_read_is_still_a_note_the_resolver_has_to_rank() {
let dir = tmpdir("bigrank");
let filler = "x".repeat(MAX_BODY_BYTES as usize + 1);
write(&dir, "spec.md", &format!("# Spec\n{filler}"));
write(&dir, "deep/spec.md", "# Spec\n");
write(&dir, "other.md", "# Other\nsee [[spec]].\n");
assert!(scan(
&target(&dir, "deep/spec.md", "Spec"),
std::slice::from_ref(&dir)
)
.is_empty());
let rows = scan(&target(&dir, "spec.md", "Spec"), std::slice::from_ref(&dir));
assert_eq!(rows.len(), 1);
assert_eq!(rows[0].name, "other");
let _ = fs::remove_dir_all(&dir);
}
#[test]
fn a_scan_nobody_is_waiting_for_stops_rather_than_reading_the_rest_of_the_vault() {
let dir = tmpdir("cancel");
write(&dir, "spec.md", "# Spec\n");
write(&dir, "meta.md", "# Meta\nsee [[spec]].\n");
let stop = AtomicBool::new(true);
let rows = super::scan(
&target(&dir, "spec.md", "Spec"),
std::slice::from_ref(&dir),
&stop,
);
assert!(rows.is_empty());
assert_eq!(
scan(&target(&dir, "spec.md", "Spec"), std::slice::from_ref(&dir)).len(),
1
);
let _ = fs::remove_dir_all(&dir);
}
#[test]
fn two_notes_of_the_same_name_are_told_apart_by_the_resolver_not_the_name() {
let dir = tmpdir("ambiguous");
write(&dir, "spec.md", "# Spec\n");
write(&dir, "deep/spec.md", "# Spec\n");
write(&dir, "other.md", "# Other\nsee [[spec]].\n");
let near = scan(&target(&dir, "spec.md", "Spec"), std::slice::from_ref(&dir));
assert_eq!(near.len(), 1);
assert_eq!(near[0].name, "other");
let far = scan(
&target(&dir, "deep/spec.md", "Spec"),
std::slice::from_ref(&dir),
);
assert!(far.is_empty());
let _ = fs::remove_dir_all(&dir);
}
#[test]
fn a_scan_of_a_vault_that_is_not_there_answers_with_nothing() {
let dir = std::env::temp_dir().join("catcher-mentions-missing");
let _ = fs::remove_dir_all(&dir);
assert!(scan(&target(&dir, "spec.md", "Spec"), std::slice::from_ref(&dir)).is_empty());
}
}