use std::collections::{BTreeMap, BTreeSet};
use std::fmt;
use std::ops::Range;
use std::path::{Path, PathBuf};
use super::Graph;
use crate::error::Result;
use crate::fs::ReadStorage;
use crate::identity::{self, Id};
use crate::index::IdIndex;
use crate::link::{self, Link};
use crate::title::{self, TitleIndex, TitleMatch};
use super::Target;
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum LinkSite {
Relation(String),
Body(Range<usize>),
}
impl fmt::Display for LinkSite {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
LinkSite::Relation(name) => f.write_str(name),
LinkSite::Body(_) => f.write_str("body"),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Resolution {
Path(PathBuf),
CaseMismatch { got: PathBuf, actual: String },
Broken,
Id { id: Id, to: PathBuf },
DanglingId { id: Id, tombstoned: bool },
MalformedId,
AmbiguousAlias {
name: String,
candidates: Vec<PathBuf>,
},
External,
Foreign { workspace: String, id: Id },
}
impl Resolution {
pub fn resolved_path(&self) -> Option<&PathBuf> {
match self {
Resolution::Path(p)
| Resolution::CaseMismatch { got: p, .. }
| Resolution::Id { to: p, .. } => Some(p),
_ => None,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CensusEntry {
pub source: PathBuf,
pub site: LinkSite,
pub target_text: String,
pub label: Option<String>,
pub resolution: Resolution,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Backlink {
pub source: PathBuf,
pub site: LinkSite,
pub by_id: bool,
}
enum NameMatch {
Exact,
CaseOnly(String),
None,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum StructuralFact {
Unreadable { doc: PathBuf, error: String },
IdMismatch {
doc: PathBuf,
frontmatter: Id,
registry: Option<Id>,
},
UnregisteredId { doc: PathBuf, frontmatter: Id },
UnstampedId { doc: PathBuf, registry: Id },
DuplicateContainment { doc: PathBuf, target: String },
MissingInverse {
doc: PathBuf,
child: PathBuf,
inverse: String,
},
CaseMismatch {
doc: PathBuf,
site: LinkSite,
target: String,
actual: String,
},
BrokenLink {
doc: PathBuf,
site: LinkSite,
target: String,
},
ManifestConflict { doc: PathBuf },
}
pub struct Walk {
pub census: Vec<CensusEntry>,
pub facts: Vec<StructuralFact>,
pub content_bodies: Vec<PathBuf>,
}
pub fn reachable_set(
start: &Path,
census: &[CensusEntry],
content_bodies: &[PathBuf],
) -> BTreeSet<PathBuf> {
let mut reachable: BTreeSet<PathBuf> = BTreeSet::new();
reachable.insert(link::normalize(start));
reachable.extend(content_bodies.iter().cloned());
for entry in census {
match &entry.resolution {
Resolution::Path(p) | Resolution::Id { to: p, .. } => {
reachable.insert(p.clone());
}
Resolution::CaseMismatch { got, actual } => {
reachable.insert(got.with_file_name(actual));
}
_ => {}
}
}
reachable
}
impl<FS: ReadStorage, Ix: IdIndex> Graph<FS, Ix> {
pub async fn reachable_documents(
&self,
start: &Path,
census: &[CensusEntry],
content_bodies: &[PathBuf],
) -> Result<BTreeSet<PathBuf>> {
let reachable = reachable_set(start, census, content_bodies);
let reached_dirs = Self::reached_dirs(&reachable);
let listing: BTreeSet<PathBuf> = self
.direct_child_files(&reached_dirs)
.await?
.into_iter()
.collect();
let mut documents = BTreeSet::new();
for path in reachable {
if !self.is_shadowed_payload(&path, &listing).await {
documents.insert(path);
}
}
Ok(documents)
}
pub async fn reachable_files(&self, start: impl AsRef<Path>) -> Result<BTreeSet<PathBuf>> {
self.reachable_files_within(start, &[]).await
}
pub async fn reachable_files_within(
&self,
start: impl AsRef<Path>,
parked: &[PathBuf],
) -> Result<BTreeSet<PathBuf>> {
let start = link::normalize(start);
let Walk {
census,
content_bodies,
..
} = self.walk(&start, parked).await?;
let mut files = BTreeSet::new();
for path in reachable_set(&start, &census, &content_bodies) {
if self.fs().try_exists(&self.root().join(&path)).await? {
files.insert(path);
}
}
Ok(files)
}
pub async fn census(&self, start: impl AsRef<Path>) -> Result<Vec<CensusEntry>> {
self.census_within(start, &[]).await
}
pub async fn census_within(
&self,
start: impl AsRef<Path>,
parked: &[PathBuf],
) -> Result<Vec<CensusEntry>> {
Ok(self.walk(start.as_ref(), parked).await?.census)
}
pub async fn backlinks(
&self,
start: impl AsRef<Path>,
) -> Result<BTreeMap<PathBuf, Vec<Backlink>>> {
Ok(invert(self.census(start).await?))
}
pub async fn backlinks_to(
&self,
start: impl AsRef<Path>,
target: impl AsRef<Path>,
) -> Result<Vec<Backlink>> {
Ok(inbound(self.census(start).await?, target.as_ref()))
}
pub async fn walk(&self, start: &Path, parked: &[PathBuf]) -> Result<Walk> {
let mut census = Vec::new();
let mut structural = Vec::new();
let mut content_bodies = Vec::new();
let mut visited = BTreeSet::new();
let mut queue = vec![link::normalize(start)];
let mut titles: Option<TitleIndex> = None;
let spanning = self.relations().spanning_relation().map(str::to_owned);
let inverse = spanning.as_deref().and_then(|s| {
self.relations()
.relations()
.iter()
.find(|r| r.name == s)
.and_then(|r| r.inverse.clone())
});
while let Some(path) = queue.pop() {
if !visited.insert(path.clone()) {
continue;
}
let doc = match self.load(&path).await {
Ok((_, doc)) => doc,
Err(e) => {
structural.push(StructuralFact::Unreadable {
doc: path,
error: e.to_string(),
});
continue;
}
};
let meta = fig::Value::from(&doc.meta);
if let Some(fm) = meta.get("id").and_then(fig::Value::as_str)
&& !fm.trim().is_empty()
{
let fm = Id(fm.trim().to_string());
match self.index().id_for_path(&path) {
Some(reg) if reg != fm => structural.push(StructuralFact::IdMismatch {
doc: path.clone(),
frontmatter: fm,
registry: Some(reg),
}),
Some(_) => {} None => match self.index().resolve(&fm) {
Some(other) if other != path => {
structural.push(StructuralFact::IdMismatch {
doc: path.clone(),
frontmatter: fm,
registry: None,
})
}
Some(_) => {}
None => structural.push(StructuralFact::UnregisteredId {
doc: path.clone(),
frontmatter: fm,
}),
},
}
} else if self.id_storage().stamps_frontmatter()
&& let Some(reg) = self.index().id_for_path(&path)
{
structural.push(StructuralFact::UnstampedId {
doc: path.clone(),
registry: reg,
});
}
for edge in self.relations().edges(&meta) {
let link = Link::parse(&edge.target);
if titles.is_none() && title::is_alias_shaped(&link.target) {
titles = Some(self.title_index_scoped(start, parked).await?);
}
let resolution = self.resolve_forward(&path, &link, titles.as_ref()).await;
if Some(edge.relation.as_str()) == spanning.as_deref()
&& let Some(resolved) = resolution.resolved_path().cloned()
{
if visited.contains(&resolved) || queue.contains(&resolved) {
structural.push(StructuralFact::DuplicateContainment {
doc: path.clone(),
target: link.target.clone(),
});
} else {
if let Some(inverse) = inverse.as_deref()
&& let Ok((_, child_doc)) = self.load(&resolved).await
&& child_doc.has_meta()
{
let child_meta = fig::Value::from(&child_doc.meta);
let inverse_targets = child_meta
.get(inverse)
.map(crate::meta::link_strings)
.unwrap_or_default();
if titles.is_none()
&& inverse_targets
.iter()
.any(|t| title::is_alias_shaped(&Link::parse(t).target))
{
titles = Some(self.title_index_scoped(start, parked).await?);
}
let points_back = inverse_targets.iter().any(|t| {
self.resolve_link_with(&resolved, &Link::parse(t), titles.as_ref())
== Target::Path(path.clone())
});
if !points_back {
structural.push(StructuralFact::MissingInverse {
doc: path.clone(),
child: resolved.clone(),
inverse: inverse.to_string(),
});
}
}
queue.push(resolved);
}
}
census.push(CensusEntry {
source: path.clone(),
site: LinkSite::Relation(edge.relation),
label: link.label,
target_text: link.target,
resolution,
});
}
for body_link in link::scan_body_links(&path, &doc.body) {
let wl = body_link.link;
if titles.is_none() && title::is_alias_shaped(&wl.target) {
titles = Some(self.title_index_scoped(start, parked).await?);
}
let resolution = self.resolve_forward(&path, &wl, titles.as_ref()).await;
census.push(CensusEntry {
source: path.clone(),
site: LinkSite::Body(body_link.span),
label: wl.label,
target_text: wl.target,
resolution,
});
}
if let Some(content) = doc.content_attr() {
let target = link::resolve(&path, content);
let site = LinkSite::Relation("content".to_string());
match self.exact_name(&target).await {
NameMatch::Exact => content_bodies.push(target),
NameMatch::CaseOnly(actual) => {
content_bodies.push(target.with_file_name(&actual));
structural.push(StructuralFact::CaseMismatch {
doc: path.clone(),
site,
target: content.to_string(),
actual,
});
}
NameMatch::None => structural.push(StructuralFact::BrokenLink {
doc: path.clone(),
site,
target: content.to_string(),
}),
}
}
if let Some(manifest) = doc.manifest_attr() {
if doc.content_attr().is_some() {
structural.push(StructuralFact::ManifestConflict { doc: path.clone() });
}
let target = link::resolve(&path, manifest);
let site = LinkSite::Relation(crate::manifest::MANIFEST_KEY.to_string());
match self.exact_name(&target).await {
NameMatch::Exact => content_bodies.push(target),
NameMatch::CaseOnly(actual) => {
content_bodies.push(target.with_file_name(&actual));
structural.push(StructuralFact::CaseMismatch {
doc: path.clone(),
site,
target: manifest.to_string(),
actual,
});
}
NameMatch::None => structural.push(StructuralFact::BrokenLink {
doc: path.clone(),
site,
target: manifest.to_string(),
}),
}
}
}
Ok(Walk {
census,
facts: structural,
content_bodies,
})
}
async fn resolve_forward(
&self,
source: &Path,
link: &Link,
titles: Option<&TitleIndex>,
) -> Resolution {
if link.is_external() {
return Resolution::External;
}
let local_id = match link.id_ref() {
Some(crate::link::IdRef::Local(id)) => Some(id),
Some(crate::link::IdRef::Foreign { workspace, id }) => {
if self.workspace_id().is_empty() || workspace != self.workspace_id() {
return Resolution::Foreign { workspace, id };
}
Some(id)
}
Some(crate::link::IdRef::Malformed) => return Resolution::MalformedId,
None => None,
};
if let Some(id) = local_id {
if !identity::verify(id.as_str()) {
return Resolution::MalformedId;
}
return match self.index().resolve(&id) {
Some(path) => Resolution::Id {
id,
to: link::normalize(path),
},
None => Resolution::DanglingId {
tombstoned: self.index().is_known(&id),
id,
},
};
}
if let Some(titles) = titles.filter(|_| title::is_alias_shaped(&link.target)) {
match titles.resolve(&link.target) {
TitleMatch::Unique(path) => {
return match self.exact_name(&path).await {
NameMatch::Exact => Resolution::Path(path),
NameMatch::CaseOnly(actual) => {
Resolution::CaseMismatch { got: path, actual }
}
NameMatch::None => Resolution::Broken,
};
}
TitleMatch::Ambiguous(candidates) => {
return Resolution::AmbiguousAlias {
name: link.target.clone(),
candidates,
};
}
TitleMatch::Unknown => {}
}
}
let resolved = link::resolve(source, &link.target);
match self.exact_name(&resolved).await {
NameMatch::Exact => Resolution::Path(resolved),
NameMatch::CaseOnly(actual) => Resolution::CaseMismatch {
got: resolved,
actual,
},
NameMatch::None => Resolution::Broken,
}
}
async fn exact_name(&self, path: &Path) -> NameMatch {
let full = self.root().join(path);
let (Some(parent), Some(name)) = (full.parent(), full.file_name()) else {
return NameMatch::None;
};
let Ok(entries) = self.fs().read_dir(parent).await else {
return NameMatch::None;
};
let mut case_only = None;
for entry in entries {
let Some(entry_name) = entry.file_name() else {
continue;
};
if entry_name == name {
return NameMatch::Exact;
}
if entry_name.eq_ignore_ascii_case(name) {
case_only = Some(entry_name.to_string_lossy().into_owned());
}
}
match case_only {
Some(actual) => NameMatch::CaseOnly(actual),
None => NameMatch::None,
}
}
}
#[cfg(all(test, feature = "yaml"))]
mod tests {
use super::*;
use crate::exec::block_on;
use crate::fs::StdFs;
use crate::graph::ReadSettings;
use crate::index::NoIndex;
fn write(dir: &Path, rel: &str, text: &str) {
let p = dir.join(rel);
std::fs::create_dir_all(p.parent().unwrap()).unwrap();
std::fs::write(p, text).unwrap();
}
fn tempdir(tag: &str) -> PathBuf {
let dir = std::env::temp_dir().join(format!("prov-census-{tag}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).unwrap();
dir
}
#[test]
fn census_covers_frontmatter_edges_and_body_wikilinks() {
let dir = tempdir("census");
write(
&dir,
"index.md",
"---\ncontents:\n- a.md\n---\nBody links [[a.md]] and [[gone.md]].\n",
);
write(&dir, "a.md", "---\npart_of: index.md\n---\n");
let ws = Graph::new(StdFs, &dir, NoIndex, ReadSettings::default());
let census = block_on(ws.census("index.md")).unwrap();
assert!(
census.iter().any(
|e| matches!(&e.site, LinkSite::Relation(r) if r == "contents")
&& matches!(&e.resolution, Resolution::Path(p) if p == &PathBuf::from("a.md"))
),
"{census:?}"
);
assert!(
census.iter().any(|e| matches!(e.site, LinkSite::Body(_))
&& e.target_text == "a.md"
&& matches!(&e.resolution, Resolution::Path(_))),
"{census:?}"
);
assert!(
census
.iter()
.any(|e| e.target_text == "gone.md" && matches!(e.resolution, Resolution::Broken)),
"{census:?}"
);
}
#[test]
fn backlinks_invert_the_census_across_relations_and_body() {
let dir = tempdir("backlinks");
write(&dir, "index.md", "---\ncontents:\n- a.md\n- b.md\n---\n");
write(&dir, "a.md", "---\npart_of: index.md\n---\n");
write(
&dir,
"b.md",
"---\npart_of: index.md\nlinks:\n- a.md\n---\nSee [[a.md]] again.\n",
);
let ws = Graph::new(StdFs, &dir, NoIndex, ReadSettings::default());
let to_a = block_on(ws.backlinks_to("index.md", "a.md")).unwrap();
assert_eq!(to_a.len(), 3, "{to_a:?}");
assert!(
to_a.iter().any(|bl| bl.source == Path::new("index.md")
&& matches!(&bl.site, LinkSite::Relation(r) if r == "contents")),
"{to_a:?}"
);
assert!(
to_a.iter().any(|bl| bl.source == Path::new("b.md")
&& matches!(&bl.site, LinkSite::Relation(r) if r == "links")),
"{to_a:?}"
);
assert!(
to_a.iter()
.any(|bl| bl.source == Path::new("b.md") && matches!(bl.site, LinkSite::Body(_))),
"{to_a:?}"
);
assert!(to_a.iter().all(|bl| !bl.by_id), "{to_a:?}");
let map = block_on(ws.backlinks("index.md")).unwrap();
assert_eq!(map[&PathBuf::from("a.md")].len(), 3);
}
}
pub fn invert(census: Vec<CensusEntry>) -> BTreeMap<PathBuf, Vec<Backlink>> {
let mut map: BTreeMap<PathBuf, Vec<Backlink>> = BTreeMap::new();
for entry in census {
let by_id = matches!(entry.resolution, Resolution::Id { .. });
let Some(target) = entry.resolution.resolved_path().cloned() else {
continue;
};
map.entry(target).or_default().push(Backlink {
source: entry.source,
site: entry.site,
by_id,
});
}
for links in map.values_mut() {
links.sort_by(|a, b| a.source.cmp(&b.source).then(a.by_id.cmp(&b.by_id)));
}
map
}
pub fn inbound(census: Vec<CensusEntry>, target: &Path) -> Vec<Backlink> {
let target = link::normalize(target);
let mut links: Vec<Backlink> = census
.into_iter()
.filter(|entry| entry.resolution.resolved_path() == Some(&target))
.map(|entry| {
let by_id = matches!(entry.resolution, Resolution::Id { .. });
Backlink {
source: entry.source,
site: entry.site,
by_id,
}
})
.collect();
links.sort_by(|a, b| a.source.cmp(&b.source).then(a.by_id.cmp(&b.by_id)));
links
}