use crate::backlinks::Backlinks;
use crate::doc::{Doc, DocMeta};
use crate::query::{OrderKey, Query, SortDir};
use crate::related::{LINKS, Related, idf};
use rayon::prelude::*;
use std::collections::{BTreeMap, HashMap};
use std::path::{Path, PathBuf};
#[derive(Clone, Default)]
pub struct DocIndex {
docs: BTreeMap<PathBuf, Doc>,
collections: HashMap<String, Vec<PathBuf>>,
taxonomies: HashMap<String, BTreeMap<String, Vec<PathBuf>>>,
backlinks: HashMap<PathBuf, Vec<PathBuf>>,
}
impl DocIndex {
pub fn new() -> DocIndex {
DocIndex::default()
}
pub fn insert(&mut self, doc: Doc) {
self.docs.insert(doc.id_path.clone(), doc);
}
pub fn docs(&self) -> impl Iterator<Item = &Doc> {
self.docs.values()
}
pub fn par_docs_mut(&mut self) -> impl ParallelIterator<Item = &mut Doc> {
self.docs.par_iter_mut().map(|(_, doc)| doc)
}
pub fn par_docs(&self) -> impl ParallelIterator<Item = &Doc> {
self.docs.par_iter().map(|(_, doc)| doc)
}
pub fn to_doc_metas(&self) -> Vec<DocMeta> {
self.docs.values().map(DocMeta::from).collect()
}
pub fn doc(&self, id_path: &Path) -> Option<&Doc> {
self.docs.get(id_path)
}
pub fn doc_mut(&mut self, id_path: &Path) -> Option<&mut Doc> {
self.docs.get_mut(id_path)
}
pub fn output_path(&self, id_path: &Path) -> Option<&Path> {
self.docs.get(id_path).map(|d| d.output_path.as_path())
}
pub fn define_collection(&mut self, name: &str, query: &Query) {
let ids = query
.evaluate(self.docs.values())
.into_iter()
.map(|d| d.id_path.clone())
.collect();
self.collections.insert(name.to_string(), ids);
}
pub fn get_collection(&self, name: &str) -> impl Iterator<Item = &Doc> {
self.collections
.get(name)
.into_iter()
.flat_map(|ids| ids.iter())
.filter_map(|id| self.docs.get(id))
}
pub fn define_taxonomies(&mut self, taxonomies: &[String]) {
for name in taxonomies {
let key = name.clone();
self.define_taxonomy(name, move |doc| {
doc.terms
.get(&key)
.map(|bucket| bucket.keys().cloned().collect())
.unwrap_or_default()
});
}
}
fn define_taxonomy<F>(&mut self, name: &str, term_fn: F)
where
F: Fn(&Doc) -> Vec<String>,
{
let mut taxonomy: BTreeMap<String, Vec<PathBuf>> = BTreeMap::new();
for doc in self.docs.values() {
for term in term_fn(doc) {
taxonomy.entry(term).or_default().push(doc.id_path.clone());
}
}
for ids in taxonomy.values_mut() {
ids.sort_by(|a, b| {
let da = &self.docs[a];
let db = &self.docs[b];
db.date.cmp(&da.date).then_with(|| a.cmp(b))
});
}
self.taxonomies.insert(name.to_string(), taxonomy);
}
pub fn get_taxonomy(&self, name: &str) -> Option<&BTreeMap<String, Vec<PathBuf>>> {
self.taxonomies.get(name)
}
pub fn define_backlinks(&mut self) {
let mut backlinks: HashMap<PathBuf, Vec<PathBuf>> = HashMap::new();
for doc in self.docs.values() {
for target in &doc.links {
backlinks
.entry(target.clone())
.or_default()
.push(doc.id_path.clone());
}
}
self.backlinks = backlinks;
}
pub fn list_backlinks(&self, target: &Path, opts: &Backlinks) -> Vec<&Doc> {
let omit: std::collections::HashSet<&Path> =
opts.omit.iter().map(PathBuf::as_path).collect();
let mut results: Vec<&Doc> = self
.backlinks
.get(target)
.into_iter()
.flat_map(|ids| ids.iter())
.filter(|id| !omit.contains(id.as_path()))
.filter_map(|id| self.docs.get(id))
.collect();
results.sort_by(|a, b| {
let cmp = match opts.order_by {
OrderKey::Title => a.title.cmp(&b.title),
OrderKey::Date => a.date.cmp(&b.date),
OrderKey::Updated => a.updated.cmp(&b.updated),
};
let cmp = match opts.sort {
SortDir::Asc => cmp,
SortDir::Desc => cmp.reverse(),
};
cmp.then_with(|| a.id_path.cmp(&b.id_path))
});
if let Some(n) = opts.limit {
results.truncate(n);
}
results
}
pub fn related(&self, post: &Path, opts: &Related) -> Vec<&Doc> {
let Some(post_doc) = self.docs.get(post) else {
return Vec::new();
};
let n = self.docs.len();
let mut scores: HashMap<PathBuf, f64> = HashMap::new();
for (namespace, weight) in &opts.weights {
if namespace == LINKS {
let w = weight * idf(self.link_df(post), n);
for linker in self.linkers(post) {
*scores.entry(linker.clone()).or_default() += w;
}
for target in &post_doc.links {
let w = weight * idf(self.link_df(target), n);
*scores.entry(target.clone()).or_default() += w;
for linker in self.linkers(target) {
*scores.entry(linker.clone()).or_default() += w;
}
}
} else if let Some(taxonomy) = self.taxonomies.get(namespace) {
let Some(bucket) = post_doc.terms.get(namespace) else {
continue;
};
for term in bucket.keys() {
if let Some(ids) = taxonomy.get(term) {
let w = weight * idf(ids.len(), n);
for id in ids {
*scores.entry(id.clone()).or_default() += w;
}
}
}
}
}
scores.remove(post);
for omit in &opts.omit {
scores.remove(omit);
}
let mut ranked: Vec<(&Doc, f64)> = scores
.into_iter()
.filter_map(|(id, score)| self.docs.get(&id).map(|doc| (doc, score)))
.collect();
ranked.sort_by(|(a, sa), (b, sb)| {
sb.partial_cmp(sa)
.unwrap_or(std::cmp::Ordering::Equal) .then_with(|| b.date.cmp(&a.date)) .then_with(|| a.id_path.cmp(&b.id_path)) });
let mut docs: Vec<&Doc> = ranked.into_iter().map(|(doc, _)| doc).collect();
if let Some(n) = opts.limit {
docs.truncate(n);
}
docs
}
fn linkers(&self, target: &Path) -> &[PathBuf] {
self.backlinks.get(target).map_or(&[], Vec::as_slice)
}
fn link_df(&self, target: &Path) -> usize {
self.linkers(target).len() + usize::from(self.docs.contains_key(target))
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::query::OrderKey;
use chrono::{DateTime, NaiveDate, Utc};
fn at(date: &str) -> DateTime<Utc> {
NaiveDate::parse_from_str(date, "%Y-%m-%d")
.unwrap()
.and_hms_opt(0, 0, 0)
.unwrap()
.and_utc()
}
fn doc(id_path: &str, title: &str, date: &str) -> Doc {
Doc {
id_path: PathBuf::from(id_path),
output_path: PathBuf::from(id_path).with_extension("html"),
title: title.to_string(),
date: at(date),
updated: at(date),
..Default::default()
}
}
fn index(docs: Vec<Doc>) -> DocIndex {
let mut index = DocIndex::new();
for d in docs {
index.insert(d);
}
index
}
#[test]
fn define_collection_filters_and_orders() {
let mut index = index(vec![
doc("posts/a.md", "A", "2025-01-01"),
doc("posts/b.md", "B", "2025-03-01"),
doc("pages/c.md", "C", "2025-02-01"),
]);
let q = Query {
path: Some(
globset::GlobBuilder::new("posts/*.md")
.literal_separator(true)
.build()
.unwrap(),
),
..Query::default()
};
index.define_collection("posts", &q);
let titles: Vec<&str> = index
.get_collection("posts")
.map(|d| d.title.as_str())
.collect();
assert_eq!(titles, vec!["B", "A"]);
}
#[test]
fn get_collection_unknown_name_is_empty() {
let index = index(vec![doc("a.md", "A", "2025-01-01")]);
assert_eq!(index.get_collection("nope").count(), 0);
}
#[test]
fn collection_order_is_stable_across_equal_sort_keys() {
let mut index = index(vec![
doc("a.md", "A", "2025-01-01"),
doc("b.md", "B", "2025-01-01"),
doc("c.md", "C", "2025-01-01"),
]);
let q = Query {
order_by: OrderKey::Date,
..Query::default()
};
index.define_collection("all", &q);
let ids: Vec<&str> = index
.get_collection("all")
.map(|d| d.id_path.to_str().unwrap())
.collect();
assert_eq!(ids, vec!["a.md", "b.md", "c.md"]);
}
fn with_tags(mut d: Doc, tags: &[&str]) -> Doc {
let bucket = d.terms.entry("tags".into()).or_default();
for t in tags {
bucket.insert((*t).to_string(), (*t).to_string());
}
d
}
#[test]
fn define_taxonomies_buckets_by_term_slug() {
let a = with_tags(doc("a.md", "A", "2025-01-01"), &["rust"]);
let b = with_tags(doc("b.md", "B", "2025-03-01"), &["rust", "go"]);
let index = {
let mut idx = index(vec![a, b]);
idx.define_taxonomies(&["tags".to_string()]);
idx
};
let tags = index.get_taxonomy("tags").unwrap();
assert_eq!(
tags.get("rust").unwrap(),
&vec![PathBuf::from("b.md"), PathBuf::from("a.md")]
);
assert_eq!(tags.get("go").unwrap(), &vec![PathBuf::from("b.md")]);
}
#[test]
fn define_taxonomies_handles_multiple_taxonomies() {
let mut a = doc("a.md", "A", "2025-01-01");
a.terms
.entry("categories".into())
.or_default()
.insert("tech".into(), "Tech".into());
let index = {
let mut idx = index(vec![a]);
idx.define_taxonomies(&["tags".to_string(), "categories".to_string()]);
idx
};
assert!(index.get_taxonomy("tags").unwrap().is_empty());
assert_eq!(
index
.get_taxonomy("categories")
.unwrap()
.get("tech")
.unwrap(),
&vec![PathBuf::from("a.md")]
);
}
#[test]
fn get_taxonomy_unknown_name_is_none() {
let index = index(vec![doc("a.md", "A", "2025-01-01")]);
assert!(index.get_taxonomy("tags").is_none());
}
#[test]
fn output_path_round_trips() {
let index = index(vec![doc("posts/a.md", "A", "2025-01-01")]);
assert_eq!(
index.output_path(Path::new("posts/a.md")),
Some(Path::new("posts/a.html"))
);
assert_eq!(index.output_path(Path::new("missing.md")), None);
}
#[test]
fn to_doc_metas_covers_all_docs() {
let index = index(vec![
doc("a.md", "A", "2025-01-01"),
doc("b.md", "B", "2025-01-02"),
]);
assert_eq!(index.to_doc_metas().len(), 2);
}
use crate::backlinks::Backlinks;
fn linking(id_path: &str, title: &str, date: &str, links: &[&str]) -> Doc {
Doc {
links: links.iter().map(PathBuf::from).collect(),
..doc(id_path, title, date)
}
}
fn linked_index(docs: Vec<Doc>) -> DocIndex {
let mut idx = index(docs);
idx.define_backlinks();
idx
}
#[test]
fn list_backlinks_no_backlinks_returns_empty() {
let index = linked_index(vec![linking("a.md", "A", "2025-01-01", &[])]);
let results = index.list_backlinks(Path::new("b.md"), &Backlinks::default());
assert!(results.is_empty());
}
#[test]
fn list_backlinks_finds_single_backlink() {
let index = linked_index(vec![
linking("a.md", "A", "2025-01-01", &["b.md"]),
linking("b.md", "B", "2025-01-02", &[]),
]);
let results = index.list_backlinks(Path::new("b.md"), &Backlinks::default());
assert_eq!(results.len(), 1);
assert_eq!(results[0].title, "A");
}
#[test]
fn list_backlinks_finds_multiple_backlinks() {
let index = linked_index(vec![
linking("a.md", "A", "2025-01-01", &["b.md"]),
linking("c.md", "C", "2025-01-03", &["b.md", "other.md"]),
linking("d.md", "D", "2025-01-02", &["other.md"]),
]);
let results = index.list_backlinks(Path::new("b.md"), &Backlinks::default());
assert_eq!(results.len(), 2);
}
#[test]
fn list_backlinks_default_order_is_date_desc() {
let index = linked_index(vec![
linking("a.md", "A", "2025-01-01", &["target.md"]),
linking("b.md", "B", "2025-02-01", &["target.md"]),
linking("c.md", "C", "2025-03-01", &["target.md"]),
]);
let results = index.list_backlinks(Path::new("target.md"), &Backlinks::default());
let titles: Vec<&str> = results.iter().map(|d| d.title.as_str()).collect();
assert_eq!(titles, vec!["C", "B", "A"]);
}
#[test]
fn list_backlinks_order_by_title_asc() {
let index = linked_index(vec![
linking("a.md", "Charlie", "2025-01-01", &["target.md"]),
linking("b.md", "Alpha", "2025-01-02", &["target.md"]),
linking("c.md", "Bravo", "2025-01-03", &["target.md"]),
]);
let opts = Backlinks {
order_by: OrderKey::Title,
sort: SortDir::Asc,
..Default::default()
};
let results = index.list_backlinks(Path::new("target.md"), &opts);
let titles: Vec<&str> = results.iter().map(|d| d.title.as_str()).collect();
assert_eq!(titles, vec!["Alpha", "Bravo", "Charlie"]);
}
#[test]
fn list_backlinks_excludes_omitted_docs() {
let index = linked_index(vec![
linking("a.md", "A", "2025-01-01", &["target.md"]),
linking("b.md", "B", "2025-02-01", &["target.md"]),
linking("c.md", "C", "2025-03-01", &["target.md"]),
]);
let opts = Backlinks {
omit: vec![PathBuf::from("b.md")],
..Default::default()
};
let results = index.list_backlinks(Path::new("target.md"), &opts);
let titles: Vec<&str> = results.iter().map(|d| d.title.as_str()).collect();
assert_eq!(titles, vec!["C", "A"]);
}
#[test]
fn list_backlinks_omit_self_drops_self_link() {
let index = linked_index(vec![
linking("a.md", "A", "2025-01-01", &["target.md"]),
linking("target.md", "Target", "2025-01-02", &["target.md"]),
]);
let opts = Backlinks {
omit: vec![PathBuf::from("target.md")],
..Default::default()
};
let results = index.list_backlinks(Path::new("target.md"), &opts);
let titles: Vec<&str> = results.iter().map(|d| d.title.as_str()).collect();
assert_eq!(titles, vec!["A"]);
}
#[test]
fn list_backlinks_applies_limit() {
let index = linked_index(vec![
linking("a.md", "A", "2025-01-01", &["target.md"]),
linking("b.md", "B", "2025-02-01", &["target.md"]),
linking("c.md", "C", "2025-03-01", &["target.md"]),
]);
let opts = Backlinks {
limit: Some(2),
..Default::default()
};
let results = index.list_backlinks(Path::new("target.md"), &opts);
let titles: Vec<&str> = results.iter().map(|d| d.title.as_str()).collect();
assert_eq!(titles, vec!["C", "B"]);
}
#[test]
fn list_backlinks_omit_then_limit_compose() {
let index = linked_index(vec![
linking("a.md", "A", "2025-01-01", &["target.md"]),
linking("b.md", "B", "2025-02-01", &["target.md"]),
linking("c.md", "C", "2025-03-01", &["target.md"]),
]);
let opts = Backlinks {
omit: vec![PathBuf::from("c.md")],
limit: Some(1),
..Default::default()
};
let results = index.list_backlinks(Path::new("target.md"), &opts);
let titles: Vec<&str> = results.iter().map(|d| d.title.as_str()).collect();
assert_eq!(titles, vec!["B"]);
}
#[test]
fn list_backlinks_does_not_include_self_unless_self_links() {
let index = linked_index(vec![linking("a.md", "A", "2025-01-01", &["a.md"])]);
let results = index.list_backlinks(Path::new("a.md"), &Backlinks::default());
assert_eq!(results.len(), 1);
}
use crate::related::{LINKS, Related};
fn related_index(docs: Vec<Doc>) -> DocIndex {
let mut idx = index(docs);
idx.define_taxonomies(&["tags".to_string()]);
idx.define_backlinks();
idx
}
fn weights(pairs: &[(&str, f64)]) -> Related {
Related {
weights: pairs.iter().map(|(n, w)| (n.to_string(), *w)).collect(),
..Default::default()
}
}
fn titles(docs: &[&Doc]) -> Vec<String> {
docs.iter().map(|d| d.title.clone()).collect()
}
#[test]
fn related_ranks_by_shared_tag_overlap() {
let index = related_index(vec![
with_tags(doc("p.md", "P", "2025-01-01"), &["rust", "ssg"]),
with_tags(doc("a.md", "A", "2025-01-02"), &["rust", "ssg"]), with_tags(doc("b.md", "B", "2025-01-03"), &["rust"]), ]);
let results = index.related(Path::new("p.md"), &weights(&[("tags", 1.0)]));
assert_eq!(titles(&results), vec!["A", "B"]);
}
#[test]
fn related_weights_reorder_namespaces() {
let docs = || {
vec![
with_tags(linking("p.md", "P", "2025-01-01", &["b.md"]), &["rust"]),
with_tags(doc("a.md", "A", "2025-01-02"), &["rust"]), doc("b.md", "B", "2025-01-03"), ]
};
let tags_heavy = related_index(docs());
let r = tags_heavy.related(Path::new("p.md"), &weights(&[("tags", 3.0), (LINKS, 1.0)]));
assert_eq!(titles(&r), vec!["A", "B"]);
let links_heavy = related_index(docs());
let r = links_heavy.related(Path::new("p.md"), &weights(&[("tags", 1.0), (LINKS, 3.0)]));
assert_eq!(titles(&r), vec!["B", "A"]);
}
#[test]
fn related_never_includes_self() {
let index = related_index(vec![
with_tags(linking("p.md", "P", "2025-01-01", &["p.md"]), &["rust"]),
with_tags(doc("a.md", "A", "2025-01-02"), &["rust"]),
]);
let results = index.related(Path::new("p.md"), &weights(&[("tags", 1.0), (LINKS, 1.0)]));
assert_eq!(titles(&results), vec!["A"]);
}
#[test]
fn related_co_citation_without_shared_tags() {
let index = related_index(vec![
linking("p.md", "P", "2025-01-01", &["c.md"]),
linking("b.md", "B", "2025-01-02", &["c.md"]),
doc("c.md", "C", "2025-01-03"),
]);
let results = index.related(Path::new("p.md"), &weights(&[(LINKS, 1.0)]));
let mut got = titles(&results);
got.sort();
assert_eq!(got, vec!["B", "C"]);
}
#[test]
fn related_forward_and_backlink_are_symmetric() {
let index = related_index(vec![
linking("a.md", "A", "2025-01-01", &["b.md"]),
doc("b.md", "B", "2025-01-02"),
]);
let from_a = index.related(Path::new("a.md"), &weights(&[(LINKS, 1.0)]));
assert_eq!(titles(&from_a), vec!["B"]);
let from_b = index.related(Path::new("b.md"), &weights(&[(LINKS, 1.0)]));
assert_eq!(titles(&from_b), vec!["A"]);
}
#[test]
fn related_tie_break_is_date_then_id_path() {
let index = related_index(vec![
with_tags(doc("p.md", "P", "2025-01-01"), &["rust"]),
with_tags(doc("a.md", "A", "2025-03-01"), &["rust"]), with_tags(doc("b.md", "B", "2025-02-01"), &["rust"]),
with_tags(doc("c.md", "C", "2025-02-01"), &["rust"]), ]);
let results = index.related(Path::new("p.md"), &weights(&[("tags", 1.0)]));
assert_eq!(titles(&results), vec!["A", "B", "C"]);
}
#[test]
fn related_limit_truncates_and_absent_returns_all() {
let index = related_index(vec![
with_tags(doc("p.md", "P", "2025-01-01"), &["rust"]),
with_tags(doc("a.md", "A", "2025-03-01"), &["rust"]),
with_tags(doc("b.md", "B", "2025-02-01"), &["rust"]),
]);
let all = index.related(Path::new("p.md"), &weights(&[("tags", 1.0)]));
assert_eq!(all.len(), 2);
let limited = index.related(
Path::new("p.md"),
&Related {
limit: Some(1),
..weights(&[("tags", 1.0)])
},
);
assert_eq!(titles(&limited), vec!["A"]);
}
#[test]
fn related_omit_excludes_docs() {
let index = related_index(vec![
with_tags(doc("p.md", "P", "2025-01-01"), &["rust"]),
with_tags(doc("a.md", "A", "2025-03-01"), &["rust"]),
with_tags(doc("b.md", "B", "2025-02-01"), &["rust"]),
]);
let results = index.related(
Path::new("p.md"),
&Related {
omit: vec![PathBuf::from("a.md")],
..weights(&[("tags", 1.0)])
},
);
assert_eq!(titles(&results), vec!["B"]);
}
#[test]
fn related_empty_doc_relates_to_nothing() {
let index = related_index(vec![
doc("p.md", "P", "2025-01-01"),
with_tags(doc("a.md", "A", "2025-01-02"), &["rust"]),
]);
let results = index.related(Path::new("p.md"), &weights(&[("tags", 1.0), (LINKS, 1.0)]));
assert!(results.is_empty());
}
#[test]
fn related_unknown_post_is_empty() {
let index = related_index(vec![with_tags(doc("p.md", "P", "2025-01-01"), &["rust"])]);
let results = index.related(Path::new("missing.md"), &weights(&[("tags", 1.0)]));
assert!(results.is_empty());
}
}