use crate::collections::Collection;
use crate::config::Config;
use crate::content::{Backlink, Page, Site};
use crate::date::{extract_date, strip_date_prefix};
use crate::directives::process_directives;
use crate::git::GitRepo;
use crate::lang::detect_language;
use crate::links::{extract_contexts, extract_internal_links, extract_tags};
use crate::nav::load_nav;
use crate::snippets::process_snippet;
use anyhow::Result;
use pulldown_cmark::{Event, HeadingLevel, Parser, Tag, TagEnd};
use std::collections::{HashMap, HashSet};
use std::path::{Path, PathBuf};
use walkdir::WalkDir;
const IGNORED_DIRS: &[&str] = &[
"layouts",
"static",
"images",
"dist",
"node_modules",
"target",
];
pub fn discover_site(config: &Config) -> Result<Site> {
let mut site = Site::new();
let git_repo = GitRepo::open(&config.source_dir);
let (nav_items, nav_html) = load_nav(&config.source_dir);
site.nav_items = nav_items;
site.nav_html = nav_html;
let content_files = discover_content_files(&config.source_dir, config.include_drafts)?;
for path in &content_files.markdown_files {
match process_markdown_file(path, &config.source_dir, git_repo.as_ref()) {
Ok(page) => {
if !page.is_draft || config.include_drafts {
site.pages.push(page);
}
}
Err(e) => {
tracing::warn!("Failed to process {:?}: {}", path, e);
}
}
}
for path in &content_files.snippet_files {
match process_snippet(path, git_repo.as_ref()) {
Ok(snippet) => {
site.snippets.push(snippet);
}
Err(e) => {
tracing::warn!("Failed to process snippet {:?}: {}", path, e);
}
}
}
generate_auto_indexes(&mut site, &config.source_dir)?;
generate_tag_pages(&mut site, &config.source_dir, git_repo.as_ref())?;
generate_context_pages(&mut site, &config.source_dir, git_repo.as_ref())?;
let site_clone = site.clone();
for page in &mut site.pages {
page.html_content = process_directives(&page.html_content, &site_clone);
}
compute_backlinks(&mut site);
Ok(site)
}
struct ContentFiles {
markdown_files: Vec<PathBuf>,
snippet_files: Vec<PathBuf>,
}
fn discover_content_files(source_dir: &Path, include_drafts: bool) -> Result<ContentFiles> {
let mut markdown_files = Vec::new();
let mut snippet_files = Vec::new();
for entry in WalkDir::new(source_dir)
.follow_links(true)
.into_iter()
.filter_entry(|e| !is_ignored(e.path(), source_dir))
{
let entry = entry?;
let path = entry.path();
if !path.is_file() {
continue;
}
if !include_drafts && is_draft(path) {
continue;
}
let relative = path.strip_prefix(source_dir).unwrap_or(path);
let first_component = relative
.components()
.next()
.and_then(|c| c.as_os_str().to_str())
.unwrap_or("");
if first_component == "snippets" {
if let Some(ext) = path.extension().and_then(|e| e.to_str()) {
if ext != "md" {
snippet_files.push(path.to_path_buf());
}
}
} else if path.extension().and_then(|e| e.to_str()) == Some("md") {
if path.file_name().and_then(|f| f.to_str()) != Some("nav.md") {
markdown_files.push(path.to_path_buf());
}
}
}
Ok(ContentFiles {
markdown_files,
snippet_files,
})
}
fn is_ignored(path: &Path, source_dir: &Path) -> bool {
let relative = path.strip_prefix(source_dir).unwrap_or(path);
for component in relative.components() {
if let Some(name) = component.as_os_str().to_str() {
if name.starts_with('.') {
return true;
}
if IGNORED_DIRS.contains(&name) {
return true;
}
}
}
false
}
fn is_draft(path: &Path) -> bool {
if let Some(filename) = path.file_name().and_then(|f| f.to_str()) {
if filename.starts_with('_') {
return true;
}
}
for component in path.components() {
if let Some(name) = component.as_os_str().to_str() {
if name == "_drafts" {
return true;
}
}
}
false
}
fn process_markdown_file(
path: &Path,
source_dir: &Path,
git_repo: Option<&GitRepo>,
) -> Result<Page> {
let content = std::fs::read_to_string(path)?;
let relative = path.strip_prefix(source_dir).unwrap_or(path);
let relative_str = relative.to_string_lossy();
let collection = Collection::from_path(&relative_str);
let mut page = Page::new(path.to_path_buf(), collection);
page.raw_content = content.clone();
page.title = extract_title(&content).unwrap_or_else(|| {
path.file_stem()
.and_then(|s| s.to_str())
.map(|s| strip_date_prefix(s).replace(['-', '_'], " "))
.unwrap_or_else(|| "Untitled".to_string())
});
page.date = extract_date(path, git_repo);
page.is_draft = is_draft(path);
let (lang, dir) = detect_language(&content);
page.lang = lang;
page.dir = dir;
page.links = extract_internal_links(&content);
page.tags = extract_tags(&content);
page.contexts = extract_contexts(&content);
page.url = generate_url(path, source_dir, collection);
page.is_index = is_index_file(path);
page.html_content = markdown_to_html(&content);
Ok(page)
}
fn extract_title(content: &str) -> Option<String> {
let parser = Parser::new(content);
let mut in_h1 = false;
let mut title = String::new();
for event in parser {
match event {
Event::Start(Tag::Heading {
level: HeadingLevel::H1,
..
}) => {
in_h1 = true;
}
Event::End(TagEnd::Heading(HeadingLevel::H1)) => {
if !title.is_empty() {
return Some(title);
}
in_h1 = false;
}
Event::Text(text) if in_h1 => {
title.push_str(&text);
}
Event::Code(code) if in_h1 => {
title.push_str(&code);
}
_ => {}
}
}
None
}
fn generate_url(path: &Path, source_dir: &Path, collection: Collection) -> String {
let relative = path.strip_prefix(source_dir).unwrap_or(path);
if relative.to_string_lossy() == "index.md" {
return "/".to_string();
}
let mut components: Vec<&str> = relative
.components()
.filter_map(|c| c.as_os_str().to_str())
.collect();
if let Some(last) = components.last_mut() {
let filename = *last;
if let Some(stem) = filename.strip_suffix(".md") {
let clean_stem = strip_date_prefix(stem);
if clean_stem == "index" {
components.pop();
} else {
*last = clean_stem;
}
}
}
if !collection.is_hierarchical() && components.len() > 1 {
let collection_name = components[0];
let slug = slug::slugify(components.last().unwrap_or(&""));
return format!("/{}/{}/", collection_name, slug);
}
let path_str = components.join("/");
if path_str.is_empty() {
"/".to_string()
} else {
format!("/{}/", path_str)
}
}
fn is_index_file(path: &Path) -> bool {
path.file_name()
.and_then(|f| f.to_str())
.map(|f| f == "index.md")
.unwrap_or(false)
}
pub fn markdown_to_html(content: &str) -> String {
use pulldown_cmark::{html, Options};
let content = crate::links::transform_tags_to_links(content);
let content = crate::links::transform_contexts_to_links(&content);
let mut options = Options::empty();
options.insert(Options::ENABLE_TABLES);
options.insert(Options::ENABLE_FOOTNOTES);
options.insert(Options::ENABLE_STRIKETHROUGH);
options.insert(Options::ENABLE_HEADING_ATTRIBUTES);
let parser = Parser::new_ext(&content, options);
let mut html_output = String::new();
html::push_html(&mut html_output, parser);
html_output
}
fn generate_auto_indexes(site: &mut Site, source_dir: &Path) -> Result<()> {
let notes_dir = source_dir.join("notes");
if !notes_dir.exists() {
return Ok(());
}
let mut dirs_needing_index: HashSet<PathBuf> = HashSet::new();
for entry in WalkDir::new(¬es_dir)
.follow_links(true)
.into_iter()
.filter_entry(|e| !is_ignored(e.path(), source_dir))
{
let entry = entry?;
if entry.path().is_dir() && entry.path() != notes_dir {
let index_path = entry.path().join("index.md");
if !index_path.exists() {
dirs_needing_index.insert(entry.path().to_path_buf());
}
}
}
let mut all_dirs: HashSet<PathBuf> = HashSet::new();
for entry in WalkDir::new(¬es_dir)
.follow_links(true)
.into_iter()
.filter_entry(|e| !is_ignored(e.path(), source_dir))
{
let entry = entry?;
if entry.path().is_dir() && entry.path() != notes_dir {
all_dirs.insert(entry.path().to_path_buf());
}
}
for dir in dirs_needing_index {
let relative = dir.strip_prefix(source_dir).unwrap_or(&dir);
let url = format!("/{}/", relative.to_string_lossy());
let title = dir
.file_name()
.and_then(|f| f.to_str())
.unwrap_or("Index")
.replace(['-', '_'], " ");
let mut subdir_pages: HashMap<String, Vec<(&str, &str)>> = HashMap::new();
let mut root_pages: Vec<(&str, &str)> = Vec::new();
for page in &site.pages {
if page.url.starts_with(&url) && page.url != url {
let suffix = &page.url[url.len()..];
let trimmed = suffix.trim_matches('/');
if let Some(slash_pos) = trimmed.find('/') {
let subdir_name = &trimmed[..slash_pos];
subdir_pages
.entry(subdir_name.to_string())
.or_default()
.push((&page.title, &page.url));
} else {
root_pages.push((&page.title, &page.url));
}
}
}
let mut child_dir_names: HashSet<String> = HashSet::new();
for child_dir in &all_dirs {
if let Some(parent) = child_dir.parent() {
if parent == dir {
if let Some(name) = child_dir.file_name().and_then(|f| f.to_str()) {
child_dir_names.insert(name.to_string());
}
}
}
}
let mut html = format!("<h1>{}</h1>\n", html_escape::encode_text(&title));
let mut all_subdirs: Vec<_> = child_dir_names.into_iter().collect();
all_subdirs.sort();
for subdir_name in all_subdirs {
let display_name = subdir_name.replace(['-', '_'], " ");
let subdir_url = format!("{}{}/", url, subdir_name);
html.push_str(&format!(
"<h2><a href=\"{}\">{}/</a></h2>\n",
html_escape::encode_double_quoted_attribute(&subdir_url),
html_escape::encode_text(&display_name)
));
if let Some(pages) = subdir_pages.get(&subdir_name) {
let mut sorted: Vec<_> = pages.iter().collect();
sorted.sort_by(|a, b| a.0.to_lowercase().cmp(&b.0.to_lowercase()));
html.push_str("<ul class=\"collection-list\">\n");
for (page_title, page_url) in sorted {
html.push_str(&format!(
" <li><a href=\"{}\">{}</a></li>\n",
html_escape::encode_double_quoted_attribute(page_url),
html_escape::encode_text(page_title)
));
}
html.push_str("</ul>\n");
}
}
if !root_pages.is_empty() {
html.push_str("<h2>Pages</h2>\n");
root_pages.sort_by(|a, b| a.0.to_lowercase().cmp(&b.0.to_lowercase()));
html.push_str("<ul class=\"collection-list\">\n");
for (page_title, page_url) in root_pages {
html.push_str(&format!(
" <li><a href=\"{}\">{}</a></li>\n",
html_escape::encode_double_quoted_attribute(page_url),
html_escape::encode_text(page_title)
));
}
html.push_str("</ul>");
}
let mut page = Page::new(dir.clone(), Collection::Notes);
page.url = url;
page.title = title;
page.html_content = html;
page.is_index = true;
page.is_auto_index = true;
site.pages.push(page);
}
Ok(())
}
fn generate_tag_pages(
site: &mut Site,
source_dir: &Path,
git_repo: Option<&GitRepo>,
) -> Result<()> {
let mut all_tags: HashMap<String, Vec<usize>> = HashMap::new();
for (idx, page) in site.pages.iter().enumerate() {
for tag in &page.tags {
all_tags.entry(tag.slug.clone()).or_default().push(idx);
}
}
for (slug, page_indices) in all_tags {
let url = format!("/tags/{}/", slug);
let tag_md_path = source_dir.join("tags").join(format!("{}.md", slug));
let custom_content = if tag_md_path.exists() {
std::fs::read_to_string(&tag_md_path).ok()
} else {
None
};
let title = slug.replace('-', " ");
let mut html = if let Some(content) = custom_content {
markdown_to_html(&content)
} else {
format!("<h1>#{}</h1>\n", html_escape::encode_text(&title))
};
html.push_str("<ul class=\"tag-list\">\n");
for idx in page_indices {
let page = &site.pages[idx];
html.push_str(&format!(
" <li><a href=\"{}\">{}</a></li>\n",
html_escape::encode_double_quoted_attribute(&page.url),
html_escape::encode_text(&page.title)
));
}
html.push_str("</ul>");
let mut page = Page::new(tag_md_path.clone(), Collection::Tags);
page.url = url;
page.title = format!("#{}", title);
page.html_content = html;
page.date = extract_date(&tag_md_path, git_repo);
site.pages.push(page);
}
Ok(())
}
fn generate_context_pages(
site: &mut Site,
source_dir: &Path,
git_repo: Option<&GitRepo>,
) -> Result<()> {
let mut all_contexts: HashMap<String, Vec<usize>> = HashMap::new();
for (idx, page) in site.pages.iter().enumerate() {
for ctx in &page.contexts {
all_contexts.entry(ctx.slug.clone()).or_default().push(idx);
}
}
for (slug, page_indices) in all_contexts {
let url = format!("/contexts/{}/", slug);
let ctx_md_path = source_dir.join("contexts").join(format!("{}.md", slug));
let custom_content = if ctx_md_path.exists() {
std::fs::read_to_string(&ctx_md_path).ok()
} else {
None
};
let title = slug.replace('-', " ");
let mut html = if let Some(content) = custom_content {
markdown_to_html(&content)
} else {
format!("<h1>@{}</h1>\n", html_escape::encode_text(&title))
};
html.push_str("<ul class=\"context-list\">\n");
for idx in page_indices {
let page = &site.pages[idx];
html.push_str(&format!(
" <li><a href=\"{}\">{}</a></li>\n",
html_escape::encode_double_quoted_attribute(&page.url),
html_escape::encode_text(&page.title)
));
}
html.push_str("</ul>");
let mut page = Page::new(ctx_md_path.clone(), Collection::Contexts);
page.url = url;
page.title = format!("@{}", title);
page.html_content = html;
page.date = extract_date(&ctx_md_path, git_repo);
site.pages.push(page);
}
Ok(())
}
fn compute_backlinks(site: &mut Site) {
let url_to_idx: HashMap<&str, usize> = site
.pages
.iter()
.enumerate()
.map(|(idx, page)| (page.url.as_str(), idx))
.collect();
let snippet_url_to_idx: HashMap<&str, usize> = site
.snippets
.iter()
.enumerate()
.map(|(idx, snippet)| (snippet.url.as_str(), idx))
.collect();
let mut page_backlinks: HashMap<usize, Vec<Backlink>> = HashMap::new();
let mut snippet_backlinks: HashMap<usize, Vec<Backlink>> = HashMap::new();
for (source_idx, page) in site.pages.iter().enumerate() {
if page.collection == Collection::Slides {
continue;
}
for link in &page.links {
let target_url = normalize_url(&link.url);
if let Some(&target_idx) = url_to_idx.get(target_url.as_str()) {
if target_idx != source_idx {
let target_page = &site.pages[target_idx];
if target_page.collection.supports_backlinks() {
page_backlinks
.entry(target_idx)
.or_default()
.push(Backlink {
url: page.url.clone(),
title: page.title.clone(),
});
}
}
}
if let Some(&target_idx) = snippet_url_to_idx.get(target_url.as_str()) {
snippet_backlinks
.entry(target_idx)
.or_default()
.push(Backlink {
url: page.url.clone(),
title: page.title.clone(),
});
}
}
}
for (idx, backlinks) in page_backlinks {
site.pages[idx].backlinks = backlinks;
}
for (idx, backlinks) in snippet_backlinks {
site.snippets[idx].backlinks = backlinks;
}
}
fn normalize_url(url: &str) -> String {
let url = url.trim();
let url = url.split('#').next().unwrap_or(url);
if url.ends_with('/') {
url.to_string()
} else if url.contains('.') {
url.to_string()
} else {
format!("{}/", url)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_is_draft() {
assert!(is_draft(Path::new("blog/_my-draft.md")));
assert!(is_draft(Path::new("blog/_drafts/post.md")));
assert!(!is_draft(Path::new("blog/my-post.md")));
}
#[test]
fn test_extract_title() {
let content = "# My Title\n\nSome content";
assert_eq!(extract_title(content), Some("My Title".to_string()));
}
#[test]
fn test_extract_title_no_h1() {
let content = "## Not H1\n\nSome content";
assert_eq!(extract_title(content), None);
}
#[test]
fn test_generate_url_root_index() {
let path = Path::new("/source/index.md");
let source_dir = Path::new("/source");
let url = generate_url(path, source_dir, Collection::Root);
assert_eq!(url, "/");
}
#[test]
fn test_generate_url_blog_post() {
let path = Path::new("/source/blog/2024-01-15-my-post.md");
let source_dir = Path::new("/source");
let url = generate_url(path, source_dir, Collection::Blog);
assert_eq!(url, "/blog/my-post/");
}
#[test]
fn test_normalize_url() {
assert_eq!(normalize_url("/blog/post"), "/blog/post/");
assert_eq!(normalize_url("/blog/post/"), "/blog/post/");
assert_eq!(normalize_url("/blog/post#section"), "/blog/post/");
}
}