mbr_core/

repo.rs

use std::{
    ops::Deref,
    path::{Component, Path, PathBuf},
    sync::{
        Arc,
        atomic::{AtomicBool, Ordering},
    },
    time::{Instant, UNIX_EPOCH},
};

use papaya::{HashMap, HashSet};
use rayon::prelude::*;
use serde::{Serialize, Serializer, ser::SerializeSeq};

use crate::Config;
use crate::config::{RelationType, TagSource};
use crate::errors::RepoError;
use crate::relationships::{NoteRelInput, RawRelationship, RelationshipIndex};
use crate::tag_index::{TagIndex, TaggedPage};
use crate::vault::{Entry, LocalVault, MountPolicy, Vault, VaultError, VaultPath};
use crate::wikilink_index::WikilinkIndex;

mod snapshot;
pub use snapshot::{FileStamp, RepoDiff, RepoSnapshot, SNAPSHOT_FORMAT, SnapshotError};

#[derive(Clone, Serialize)]
pub struct Repo {
    /// Where the repository is stored. Every read the scan makes goes through
    /// it; see [`crate::vault`].
    #[serde(skip)]
    vault: Arc<dyn Vault>,
    /// Repository root, **always canonicalized**: it is [`Vault::root`] of
    /// [`Self::vault`], which a [`LocalVault`] canonicalizes at construction.
    ///
    /// This must stay canonical. `scan_folder` walks from the vault's
    /// canonicalization of each folder, and every path it lists is relativized
    /// back against this same value. If the two were
    /// allowed to differ — a raw root here and a canonicalized one there —
    /// `pathdiff::diff_paths` would find no common prefix and hand back an
    /// effectively absolute path, so every `url_path` would embed the whole
    /// filesystem path.
    ///
    /// That is not hypothetical: on Windows `canonicalize` returns a verbatim
    /// prefix (`\\?\D:\x`), and `Prefix::VerbatimDisk != Prefix::Disk`, so a
    /// raw `D:\x` base silently failed to match. On Unix the same thing happens
    /// through a symlinked temp dir (`/tmp` vs `/private/tmp`).
    ///
    /// Note this is deliberately *not* the same value as `Config::root_dir`,
    /// which stays as the user supplied it (it feeds template globs and
    /// user-facing output, where a verbatim prefix would cause its own
    /// problems).
    #[serde(skip)]
    canonical_root: PathBuf,
    #[serde(skip)]
    static_folder: String,
    /// The static overlay, as a vault of its own, when it legitimately
    /// resolves *outside* `canonical_root` — `static_folder = "../static"` for
    /// the `repo/content` + `repo/static` layout, or `"../../static"` for a
    /// framework layout whose markdown root is pinned two levels down
    /// (SvelteKit's `<project>/src/routes` alongside `<project>/static`). Its
    /// [`Vault::root`] is the overlay's canonical directory.
    ///
    /// A separate vault because a [`VaultPath`] cannot leave its vault: the
    /// overlay is a second root, not a path under the first. `None` when the
    /// overlay is disabled, resolves inside the root, or was refused by the
    /// policy — so this is the only storage besides [`Self::vault`] that the
    /// scanner may descend into. Decided once, by
    /// [`crate::config::resolve_static_overlay`], which is the same call
    /// `Config::validate` makes: the scanner must never accept a root the
    /// validator would have refused.
    #[serde(skip)]
    static_vault: Option<Arc<dyn Vault>>,
    #[serde(skip)]
    markdown_extensions: Vec<String>,
    /// The configured index file name (e.g., "index.md" or "_index.md").
    /// Exposed in site.json for frontend use.
    pub index_file: String,
    #[serde(skip)]
    ignore_dirs: Vec<String>,
    #[serde(skip)]
    #[allow(dead_code)] // Kept for debugging/logging; compiled version used for matching
    ignore_globs: Vec<String>,
    #[serde(skip)]
    compiled_ignore_globs: Vec<glob::Pattern>,
    /// Hidden directories the leading-dot rule must let through, as absolute
    /// paths under `canonical_root`. Rebased once in
    /// [`Repo::with_explicit_hidden_dirs`] from the root-relative form
    /// `Config` carries, so the per-entry check is a plain path comparison
    /// against the keys the scan already builds. Empty for almost every run.
    #[serde(skip)]
    exempt_hidden_dirs: Vec<PathBuf>,
    /// [`Self::exempt_hidden_dirs`] as vault paths, for the path resolver's
    /// hidden-file rule ([`crate::vault::has_hidden_segment`]).
    #[serde(skip)]
    exempt_hidden_vault_paths: Vec<VaultPath>,
    /// Every folder walked so far, by its **canonical** key, so a directory
    /// reached twice (a symlink to a folder that is also walked directly) is
    /// listed once.
    #[serde(skip)]
    pub scanned_folders: HashSet<PathBuf>,
    /// Folders found but not walked yet, by key (the folder's path as listed,
    /// *not* canonicalized — the next hop canonicalizes it).
    #[serde(skip)]
    pub queued_folders: HashMap<PathBuf, QueuedFolder>,
    /// Folders whose listing failed [`VaultError::NotDownloaded`] — on iOS, an
    /// iCloud Drive folder nothing has listed yet — by canonical key, as of
    /// the last scan or [`Repo::record_diff`]. Their download was requested;
    /// their files are indexed by the refresh that first lists them.
    #[serde(skip)]
    not_downloaded_folders: HashSet<PathBuf>,
    pub markdown_files: MarkdownFiles,
    pub other_files: OtherFiles,
    /// Thread-safe index of tagged pages.
    #[serde(skip)]
    pub tag_index: Arc<TagIndex>,
    /// Thread-safe index of typed relationships between notes.
    #[serde(skip)]
    pub relationship_index: Arc<RelationshipIndex>,
    /// Thread-safe global name index for body-wikilink (`[[Name]]`) resolution.
    /// Always built (independent of relationship tracking).
    #[serde(skip)]
    pub wikilink_index: Arc<WikilinkIndex>,
    /// Configured tag sources for frontmatter extraction.
    #[serde(skip)]
    tag_sources: Vec<TagSource>,
    /// Whether text extraction has been performed for searchable files.
    #[serde(skip)]
    text_extracted: Arc<AtomicBool>,
    /// Whether media metadata population has *finished* (phase 2 of scan).
    ///
    /// Readers (`is_media_populated`, `wait_for_media`, the `media.json`
    /// handler) treat this as "the metadata is there", so it is published only
    /// once every entry has been probed.
    #[serde(skip)]
    media_populated: Arc<AtomicBool>,
    /// Whether media metadata population has *started* (run-once guard).
    ///
    /// Deliberately separate from `media_populated`: using one flag for both
    /// meant the very first instruction of `populate_media_metadata` announced
    /// completion, so a `media.json` request arriving during the probing window
    /// skipped the wait and returned entries with no duration or dimensions.
    #[serde(skip)]
    media_population_started: Arc<AtomicBool>,
    /// Whether initial scan_all has completed.
    #[serde(skip)]
    scan_complete: Arc<AtomicBool>,
    /// Notifier for scan completion (signals waiters when scan_all finishes).
    #[serde(skip)]
    scan_notify: Arc<tokio::sync::Notify>,
    /// Notifier for media metadata population (signals waiters when populate_media_metadata finishes).
    #[serde(skip)]
    media_notify: Arc<tokio::sync::Notify>,
    /// Per-file [`FileStamp`]s, recorded by the scan only with
    /// [`Repo::with_change_tracking`] (the mobile app, which has no watcher
    /// and diffs on foreground instead). `None` on the desktop, which then
    /// pays one branch per listed file and nothing else.
    #[serde(skip)]
    stamps: Option<Arc<HashMap<PathBuf, FileStamp>>>,
}

#[derive(Clone)]
pub struct MarkdownFiles(HashMap<PathBuf, MarkdownInfo>);
impl Deref for MarkdownFiles {
    type Target = HashMap<PathBuf, MarkdownInfo>;
    fn deref(&self) -> &Self::Target {
        &self.0
    }
}

/// Serializes in `url_path` order rather than map order.
///
/// `papaya::HashMap` uses a randomly seeded `RandomState`, so iteration order
/// differs between processes: two builds of the same repository produced
/// byte-different `site.json`/`media.json`, which churns committed `build/`
/// directories and invalidates content-hash/ETag caches on a no-op rebuild.
/// `raw_path` breaks ties, because distinct files can share a `url_path`
/// (`docs/index.md` and `docs.md` both render at `/docs/`).
impl Serialize for MarkdownFiles {
    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
    where
        S: Serializer,
    {
        let pin = self.pin();
        let mut entries: Vec<&MarkdownInfo> = pin.iter().map(|(_, v)| v).collect();
        entries.sort_unstable_by(|a, b| {
            a.url_path
                .cmp(&b.url_path)
                .then_with(|| a.raw_path.cmp(&b.raw_path))
        });

        let mut s = serializer.serialize_seq(Some(entries.len()))?;
        for v in entries {
            s.serialize_element(v)?;
        }
        s.end()
    }
}

#[derive(Clone)]
pub struct OtherFiles(HashMap<PathBuf, OtherFileInfo>);
impl Deref for OtherFiles {
    type Target = HashMap<PathBuf, OtherFileInfo>;
    fn deref(&self) -> &Self::Target {
        &self.0
    }
}
/// Serializes in `url_path` order — see [`MarkdownFiles`]'s impl for why.
impl Serialize for OtherFiles {
    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
    where
        S: Serializer,
    {
        let pin = self.pin();
        let mut entries: Vec<&OtherFileInfo> = pin.iter().map(|(_, v)| v).collect();
        entries.sort_unstable_by(|a, b| {
            a.url_path
                .cmp(&b.url_path)
                .then_with(|| a.raw_path.cmp(&b.raw_path))
        });

        let mut s = serializer.serialize_seq(Some(entries.len()))?;
        for v in entries {
            s.serialize_element(v)?;
        }
        s.end()
    }
}

#[derive(Clone, Serialize)]
pub struct MarkdownInfo {
    /// Path to the source file, **relative to the repo root**.
    ///
    /// Relative rather than absolute for two reasons: it is serialized into
    /// `site.json`, where an absolute path would leak the build machine's
    /// directory layout into every published static site; and consumers
    /// (including `.mbr/` customizations) split it on `/` to recover the file
    /// name. It is serialized through [`crate::url_path::serialize_as_url`] so
    /// the JSON stays `/`-separated even on Windows.
    ///
    /// Join it with the repo root before doing any file I/O — see
    /// `SearchEngine::search_file_content`. The `markdown_files` map is still
    /// keyed by absolute path.
    #[serde(serialize_with = "crate::url_path::serialize_as_url")]
    pub raw_path: PathBuf,
    pub url_path: String,
    pub created: u64,
    pub modified: u64,
    /// The simplified frontmatter, complete in memory (search matches on all
    /// of it) but serialized **without** contact details — `emails`, `phones`,
    /// `urls`, `social`, `im`, `addresses` and their dot-key variants. This
    /// struct is what both server and static `site.json` serialize, so the one
    /// attribute covers both; see [`crate::contact::is_public_frontmatter_key`].
    #[serde(serialize_with = "crate::contact::serialize_public_frontmatter")]
    pub frontmatter: Option<crate::markdown::SimpleMetadata>,
    /// Typed relationships declared in frontmatter (unresolved endpoints).
    /// Skipped in serialization — resolved relationships are exposed via the
    /// relationship index in site.json/links.json instead.
    #[serde(skip)]
    pub relationships: Vec<RawRelationship>,
}

#[derive(Clone, Serialize)]
pub struct OtherFileInfo {
    #[serde(skip)]
    pub raw_path: PathBuf,
    pub url_path: String,
    metadata: StaticFileMetadata,
    /// Extracted text content for searchable files (PDFs, text files).
    /// Only populated for files under the size limit.
    #[serde(skip)]
    pub extracted_text: Option<String>,
}

/// Maximum file size (in bytes) for text extraction (10 MB).
const MAX_TEXT_EXTRACTION_SIZE: u64 = 10 * 1024 * 1024;

impl OtherFileInfo {
    /// Returns the file type as a string for search results.
    pub fn filetype(&self) -> &'static str {
        match &self.metadata.kind {
            StaticFileKind::Pdf { .. } => "pdf",
            StaticFileKind::Image { .. } => "image",
            StaticFileKind::Video { .. } => "video",
            StaticFileKind::Audio { .. } => "audio",
            StaticFileKind::Text => "text",
            StaticFileKind::Other => "other",
        }
    }

    /// Returns true if this file type is searchable (has extractable text).
    pub fn is_searchable(&self) -> bool {
        matches!(
            &self.metadata.kind,
            StaticFileKind::Pdf { .. } | StaticFileKind::Text
        )
    }

    /// Extract text content from the file if it's a searchable type.
    /// Respects file size limit for performance.
    ///
    /// `vault`/`path` locate the file ([`Repo::locate`]); `raw_path` is only
    /// used to name it in logs.
    fn extract_text(&self, vault: &dyn Vault, path: &VaultPath) -> Option<String> {
        // Check file size first
        if let Some(size) = self.metadata.file_size_bytes
            && size > MAX_TEXT_EXTRACTION_SIZE
        {
            tracing::debug!(
                "Skipping text extraction for {:?}: file too large ({} bytes)",
                self.raw_path,
                size
            );
            return None;
        }

        match &self.metadata.kind {
            StaticFileKind::Pdf { .. } => self.extract_pdf_text(vault, path),
            StaticFileKind::Text => self.extract_plain_text(vault, path),
            _ => None,
        }
    }

    /// Extract text from a PDF file using lopdf.
    ///
    /// Loaded from its local path when there is one (lopdf reads it lazily);
    /// otherwise from bytes read through the vault.
    fn extract_pdf_text(&self, vault: &dyn Vault, path: &VaultPath) -> Option<String> {
        let loaded = match vault.local_path(path) {
            Some(local) => lopdf::Document::load(local),
            None => match vault.read(path) {
                Ok(bytes) => lopdf::Document::load_mem(&bytes),
                Err(e) => {
                    tracing::debug!("Failed to read PDF {:?}: {}", self.raw_path, e);
                    return None;
                }
            },
        };
        let doc = match loaded {
            Ok(doc) => doc,
            Err(e) => {
                tracing::debug!("Failed to load PDF {:?}: {}", self.raw_path, e);
                return None;
            }
        };

        let page_numbers: Vec<u32> = doc.get_pages().keys().copied().collect();
        if page_numbers.is_empty() {
            return None;
        }

        match doc.extract_text(&page_numbers) {
            Ok(text) => {
                let text = text.trim().to_string();
                if text.is_empty() { None } else { Some(text) }
            }
            Err(e) => {
                tracing::debug!("Failed to extract PDF text from {:?}: {}", self.raw_path, e);
                None
            }
        }
    }

    /// Extract text from a plain text file.
    fn extract_plain_text(&self, vault: &dyn Vault, path: &VaultPath) -> Option<String> {
        match vault.read_to_string(path) {
            Ok(text) => {
                let text = text.trim().to_string();
                if text.is_empty() { None } else { Some(text) }
            }
            Err(e) => {
                tracing::debug!("Failed to read text file {:?}: {}", self.raw_path, e);
                None
            }
        }
    }
}

#[derive(Clone, Default, Serialize)]
pub struct StaticFileMetadata {
    /// Absolute path to the source file.
    ///
    /// Used internally to probe the file for kind-specific metadata (see
    /// `populate_basic` / `populate_full`), so it must stay absolute.
    ///
    /// **Not serialized.** It reaches both `site.json` (static builds) and
    /// `/.mbr/media.json` (server mode), where an absolute path would publish
    /// the build machine's directory layout to every visitor. Nothing consumes
    /// it on the wire — the frontend reads only `kind`, `created`, `modified`
    /// and `file_size_bytes`.
    #[serde(skip)]
    path: PathBuf,
    created: Option<u64>,
    modified: Option<u64>,
    file_size_bytes: Option<u64>,
    kind: StaticFileKind,
}

#[derive(Clone, Default, Serialize)]
#[serde(tag = "type", rename_all = "lowercase")]
enum StaticFileKind {
    Pdf {
        description: Option<String>,
        title: Option<String>,
        author: Option<String>,
        subject: Option<String>,
        num_pages: Option<usize>,
    },
    Image {
        width: Option<u32>,
        height: Option<u32>,
    },
    Video {
        width: Option<u32>,
        height: Option<u32>,
        duration: Option<String>,
        title: Option<String>,
        genre: Option<String>,
        album: Option<String>,
    },
    Audio {
        duration: Option<String>,
        title: Option<String>,
    },
    Text,
    #[default]
    Other,
}

/* impl Serialize for Repo {
    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
    where
        S: Serializer,
    {
        let mut s = serializer.serialize_struct("Site", 2)?;
        s.serialize_field("markdown", &self.markdown_files)?;
        s.serialize_field("other", &self.other_files)?;
        s.end()
    }
}

impl Serialize for papaya::HashMap<PathBuf, MarkdownInfo> {
    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
    where
        S: Serializer,
    {
        let mut map = serializer.serialize_map(Some(self.x.len()))?;
        for (k, v) in &self.x {
            map.serialize_entry(&k.to_string(), &v)?;
        }
        map.end()
    }
} */

impl StaticFileMetadata {
    pub fn empty<P: Into<std::path::PathBuf>>(file: P) -> Self {
        let file: PathBuf = file.into();
        // We'll silently ignore errors and always return something
        match file
            .extension()
            .map(|x| x.to_ascii_lowercase().to_string_lossy().to_string())
            .as_deref()
        {
            Some("pdf") => Self {
                path: file,
                kind: StaticFileKind::Pdf {
                    description: None,
                    title: None,
                    author: None,
                    subject: None,
                    num_pages: None,
                },
                ..Default::default()
            },
            Some("jpg") | Some("jpeg") | Some("png") | Some("webp") | Some("gif") | Some("bmp")
            | Some("tif") | Some("tiff") => Self {
                path: file,
                kind: StaticFileKind::Image {
                    width: None,
                    height: None,
                },
                ..Default::default()
            },
            Some("aiff") | Some("mp3") | Some("aac") | Some("m4a") | Some("ogg") | Some("oga")
            | Some("opus") | Some("wma") | Some("flac") | Some("wav") | Some("aif") | Some("") => {
                Self {
                    path: file,
                    kind: StaticFileKind::Audio {
                        duration: None,
                        title: None,
                    },
                    ..Default::default()
                }
            }
            Some("mp4") | Some("m4v") | Some("mov") | Some("webm") | Some("flv") | Some("mpg")
            | Some("mpeg") | Some("avi") | Some("3gp") | Some("wmv") => Self {
                path: file,
                kind: StaticFileKind::Video {
                    width: None,
                    height: None,
                    duration: None,
                    title: None,
                    genre: None,
                    album: None,
                },
                ..Default::default()
            },
            Some("txt") | Some("css") | Some("vtt") | Some("srt") | Some("toml") | Some("json")
            | Some("js") | Some("ts") => Self {
                path: file,
                kind: StaticFileKind::Text,
                ..Default::default()
            },
            _ => Self {
                path: file,
                kind: StaticFileKind::Other,
                ..Default::default()
            },
        }
    }

    /// Populate basic file metadata (size, timestamps) without expensive media extraction.
    ///
    /// Reads `path` from the local filesystem. The repository's own pass,
    /// [`Repo::populate_basic_metadata`], reads through its vault instead.
    pub fn populate_basic(self) -> Self {
        let file_details_start = Instant::now();
        let details = file_details_from_path(&self.path).ok();
        tracing::debug!(
            "populate file_details for {:?}: {:?}",
            self.path,
            file_details_start.elapsed()
        );
        self.with_details(details)
    }

    /// Sets `(size, created, modified)`; `None` clears all three, which is
    /// what a failed `stat` has always recorded.
    fn with_details(mut self, details: Option<(u64, u64, u64)>) -> Self {
        let (size, created, modified) = match details {
            Some((size, created, modified)) => (Some(size), Some(created), Some(modified)),
            None => (None, None, None),
        };
        self.file_size_bytes = size;
        self.created = created;
        self.modified = modified;
        self
    }

    /// Populate media-specific metadata (ffmpeg, lopdf) - expensive operation.
    #[cfg(feature = "media-metadata")]
    pub fn populate_media(self) -> Self {
        let mut me = self;
        let media_start = Instant::now();
        let kind_name = match &me.kind {
            StaticFileKind::Pdf { .. } => "pdf",
            StaticFileKind::Image { .. } => "image",
            StaticFileKind::Video { .. } => "video",
            StaticFileKind::Audio { .. } => "audio",
            StaticFileKind::Text => "text",
            StaticFileKind::Other => "other",
        };
        me.kind = match me.kind {
            StaticFileKind::Pdf { .. } => match crate::pdf_metadata::probe_pdf(&me.path) {
                Ok(meta) => StaticFileKind::Pdf {
                    title: meta.title,
                    author: meta.author,
                    subject: meta.subject,
                    description: None,
                    num_pages: Some(meta.num_pages as usize),
                },
                Err(e) => {
                    tracing::debug!("Failed to extract PDF metadata from {:?}: {}", me.path, e);
                    me.kind
                }
            },
            StaticFileKind::Image { .. } => {
                let metadata = metadata::media_file::MediaFileMetadata::new(&me.path).ok();

                StaticFileKind::Image {
                    width: metadata.as_ref().and_then(|m| m.width),
                    height: metadata.as_ref().and_then(|m| m.height),
                }
            }
            StaticFileKind::Audio { .. } => {
                let metadata = metadata::media_file::MediaFileMetadata::new(&me.path).ok();
                StaticFileKind::Audio {
                    duration: metadata.as_ref().and_then(|m| m.duration.clone()),
                    title: metadata.as_ref().and_then(|m| m.title.clone()),
                }
            }
            StaticFileKind::Video { .. } => {
                let metadata = metadata::media_file::MediaFileMetadata::new(&me.path).ok();

                // Extract genre from tags (case-insensitive search)
                let genre = metadata.as_ref().and_then(|m| {
                    m.tags
                        .iter()
                        .find(|(k, _)| k.eq_ignore_ascii_case("genre"))
                        .map(|(_, v)| v.clone())
                });

                // Extract album from tags (case-insensitive search)
                let album = metadata.as_ref().and_then(|m| {
                    m.tags
                        .iter()
                        .find(|(k, _)| k.eq_ignore_ascii_case("album"))
                        .map(|(_, v)| v.clone())
                });

                StaticFileKind::Video {
                    width: metadata.as_ref().and_then(|m| m.width),
                    height: metadata.as_ref().and_then(|m| m.height),
                    duration: metadata.as_ref().and_then(|m| m.duration.clone()),
                    title: metadata.as_ref().and_then(|m| m.title.clone()),
                    genre,
                    album,
                }
            }
            _ => me.kind,
        };
        tracing::debug!(
            "populate media metadata ({}) for {:?}: {:?}",
            kind_name,
            me.path,
            media_start.elapsed()
        );
        me
    }

    /// Full populate: basic + media metadata (used by build mode).
    pub fn populate(self) -> Self {
        let me = self.populate_basic();
        #[cfg(feature = "media-metadata")]
        let me = me.populate_media();
        me
    }

    pub fn from<P: Into<std::path::PathBuf>>(file: P) -> Self {
        let empty = Self::empty(file);
        empty.populate()
    }
}

impl Repo {
    pub fn init_from_config(c: &Config) -> Self {
        Self::init(
            c.root_dir.clone(),
            c.static_folder.clone(),
            &c.markdown_extensions[..],
            &c.ignore_dirs[..],
            &c.ignore_globs[..],
            c.index_file.clone(),
            &c.tag_sources[..],
            &c.relationship_types[..],
        )
        .with_explicit_hidden_dirs(&c.explicit_hidden_dirs)
        .with_symlink_mounts(!c.no_symlink_mounts)
    }

    /// Turns read-only symlink mounts off (`enabled == false`): the
    /// operator's `--no-symlink-mounts` / `MBR_NO_SYMLINK_MOUNTS`. The vaults
    /// are rebuilt as plain [`LocalVault`]s on the same (canonical) roots, so a
    /// link out of the repository or the overlay is `OutsideRoot` — 404 on the
    /// server, not placed by a build — exactly as before mounts. Hidden-file
    /// blocking is unaffected. Logs one line when it disables anything.
    ///
    /// Call before scanning: it replaces the storage the scan reads through.
    #[must_use]
    pub fn with_symlink_mounts(mut self, enabled: bool) -> Self {
        if !enabled {
            tracing::info!(
                "Symlink mounts are disabled (--no-symlink-mounts / {}): links out of the \
                 repository are not served or built",
                crate::config::NO_SYMLINK_MOUNTS_ENV
            );
            self.vault = Arc::new(LocalVault::new(self.vault.root()));
            self.static_vault = self
                .static_vault
                .map(|overlay| Arc::new(LocalVault::new(overlay.root())) as Arc<dyn Vault>);
        }
        self
    }

    /// Re-admits the hidden directories the user named on the command line
    /// (`Config::explicit_hidden_dirs`) to the scan.
    ///
    /// A builder rather than an [`Self::init`] parameter: `init` already carries
    /// eight positional arguments, and every caller but the two live entry
    /// points wants the default of "no exemptions".
    ///
    /// Rebasing happens here, once, and onto `canonical_root` rather than the
    /// configured root — the scan descends from the canonicalized root, so an
    /// exemption built from the *configured* spelling would silently never match
    /// wherever the two differ (`/tmp` vs `/private/tmp` on macOS).
    #[must_use]
    pub fn with_explicit_hidden_dirs(mut self, explicit_hidden_dirs: &[PathBuf]) -> Self {
        let absolute = explicit_hidden_dirs
            .iter()
            .map(|relative| self.canonical_root.join(relative))
            .collect();
        self.exempt_hidden_dirs = absolute;
        self.exempt_hidden_vault_paths = explicit_hidden_dirs
            .iter()
            .filter_map(|relative| VaultPath::from_relative_native(relative).ok())
            .collect();
        self
    }

    /// The hidden directories the user named on the command line, as vault
    /// paths of [`Self::vault`] — the exemption to "hidden files are never
    /// served" (see [`crate::vault::has_hidden_segment`]).
    pub fn exempt_hidden_dirs(&self) -> &[VaultPath] {
        &self.exempt_hidden_vault_paths
    }

    /// A repository on the local filesystem at `root_dir`, stored through a
    /// [`LocalVault`] (plus a second one for an external static overlay).
    ///
    /// Both vaults serve symlinks out of them as read-only **mounts**
    /// ([`crate::vault::MountPolicy`]), each protecting the other's root, so
    /// the server and the static build — which both start here — mount
    /// exactly the same folders and refuse exactly the same targets.
    #[allow(clippy::too_many_arguments)]
    pub fn init<S: Into<String>, P: Into<std::path::PathBuf>>(
        root_dir: P,
        static_folder: S,
        markdown_extensions: &[String],
        ignore_dirs: &[String],
        ignore_globs: &[String],
        index_file: S,
        tag_sources: &[TagSource],
        relationship_types: &[RelationType],
    ) -> Self {
        let root_dir: PathBuf = root_dir.into();
        let canonical_root = root_dir.canonicalize().unwrap_or_else(|_| root_dir.clone());
        let static_folder = static_folder.into();
        // Ask the config policy, rather than re-deriving containment here, so a
        // refused overlay is never scannable. A policy error is not re-reported:
        // `Config::validate` already aborted startup on it, and the only callers
        // that reach here with a bad value are tests constructing a `Repo`
        // directly.
        let overlay_dir =
            match crate::config::resolve_static_overlay(&canonical_root, &static_folder) {
                Ok(crate::config::StaticOverlay::External(dir)) => Some(dir),
                Ok(crate::config::StaticOverlay::WithinRoot) => None,
                Err(e) => {
                    tracing::warn!("Not indexing static_folder {static_folder:?}: {e}");
                    None
                }
            };
        let vault: Arc<dyn Vault> = Arc::new(LocalVault::with_mounts(
            root_dir,
            MountPolicy::new(overlay_dir.iter().cloned().collect()),
        ));
        let static_vault: Option<Arc<dyn Vault>> = overlay_dir.map(|dir| {
            Arc::new(LocalVault::with_mounts(
                dir,
                MountPolicy::new(vec![canonical_root.clone()])
                    .with_markdown_root(canonical_root.clone()),
            )) as Arc<dyn Vault>
        });

        Self::init_with_vault(
            vault,
            static_folder,
            markdown_extensions,
            ignore_dirs,
            ignore_globs,
            index_file.into(),
            tag_sources,
            relationship_types,
        )
        .with_static_vault(static_vault)
    }

    /// A repository stored in `vault`, which may be anything — the desktop's
    /// [`LocalVault`], a test's [`crate::vault::MemVault`], a platform vault.
    ///
    /// `static_folder` is honoured when it names a folder *inside* the vault;
    /// an external overlay is a second vault, given with
    /// [`Self::with_static_vault`] ([`Self::init`] does both for a local
    /// repository).
    #[allow(clippy::too_many_arguments)]
    pub fn init_with_vault<S: Into<String>>(
        vault: Arc<dyn Vault>,
        static_folder: S,
        markdown_extensions: &[String],
        ignore_dirs: &[String],
        ignore_globs: &[String],
        index_file: S,
        tag_sources: &[TagSource],
        relationship_types: &[RelationType],
    ) -> Self {
        // Pre-compile glob patterns for efficient matching during scans
        let compiled_ignore_globs: Vec<glob::Pattern> = ignore_globs
            .iter()
            .filter_map(|pat| {
                glob::Pattern::new(pat)
                    .map_err(|e| tracing::warn!("Invalid ignore glob pattern '{}': {}", pat, e))
                    .ok()
            })
            .collect();

        let canonical_root = vault.root().to_path_buf();

        Self {
            vault,
            canonical_root,
            static_folder: static_folder.into(),
            static_vault: None,
            markdown_extensions: markdown_extensions.to_vec(),
            ignore_dirs: ignore_dirs.to_vec(),
            ignore_globs: ignore_globs.to_vec(),
            compiled_ignore_globs,
            exempt_hidden_dirs: Vec::new(),
            exempt_hidden_vault_paths: Vec::new(),
            index_file: index_file.into(),
            scanned_folders: HashSet::new(),
            queued_folders: HashMap::new(),
            not_downloaded_folders: HashSet::new(),
            markdown_files: MarkdownFiles(HashMap::new()),
            other_files: OtherFiles(HashMap::new()),
            tag_index: Arc::new(TagIndex::new()),
            relationship_index: Arc::new(RelationshipIndex::from_relation_types(
                relationship_types,
            )),
            wikilink_index: Arc::new(WikilinkIndex::new()),
            tag_sources: tag_sources.to_vec(),
            text_extracted: Arc::new(AtomicBool::new(false)),
            media_populated: Arc::new(AtomicBool::new(false)),
            media_population_started: Arc::new(AtomicBool::new(false)),
            scan_complete: Arc::new(AtomicBool::new(false)),
            scan_notify: Arc::new(tokio::sync::Notify::new()),
            media_notify: Arc::new(tokio::sync::Notify::new()),
            stamps: None,
        }
    }

    /// Sets the vault of an *external* static overlay (see the field docs).
    /// Its assets are indexed with the overlay prefix stripped from their
    /// URLs, and markdown inside it is skipped.
    #[must_use]
    pub fn with_static_vault(mut self, static_vault: Option<Arc<dyn Vault>>) -> Self {
        self.static_vault = static_vault;
        self
    }

    /// The repository's storage.
    pub fn vault(&self) -> &Arc<dyn Vault> {
        &self.vault
    }

    /// The external static overlay's storage, when there is one.
    pub fn static_vault(&self) -> Option<&Arc<dyn Vault>> {
        self.static_vault.as_ref()
    }

    /// The external static overlay's canonical directory, when there is one.
    pub fn canonical_static_root(&self) -> Option<&Path> {
        self.static_vault.as_deref().map(|vault| vault.root())
    }

    /// Which vault holds the file behind an index key, and where in it.
    ///
    /// `None` for a key under neither — which no scan produces, but the watcher
    /// can report (a configured template folder outside the root).
    pub fn locate(&self, key: &Path) -> Option<(&dyn Vault, VaultPath)> {
        if let Ok(path) = self.vault.vault_path(key) {
            return Some((self.vault.as_ref(), path));
        }
        let overlay = self.static_vault.as_deref()?;
        overlay.vault_path(key).ok().map(|path| (overlay, path))
    }

    /// [`Self::locate`], with the vault shared rather than borrowed — for a
    /// caller that reads it from another task (a streamed response body).
    pub fn locate_shared(&self, key: &Path) -> Option<(Arc<dyn Vault>, VaultPath)> {
        if let Ok(path) = self.vault.vault_path(key) {
            return Some((Arc::clone(&self.vault), path));
        }
        let overlay = self.static_vault.as_ref()?;
        overlay
            .vault_path(key)
            .ok()
            .map(|path| (Arc::clone(overlay), path))
    }

    /// The symlink mounts both vaults have accepted so far, each with the index
    /// key of its location (where the link is). Grows as the first scan
    /// discovers links; see [`crate::vault::Mount`].
    pub fn mounts(&self) -> Vec<(PathBuf, crate::vault::Mount)> {
        std::iter::once(&self.vault)
            .chain(self.static_vault.as_ref())
            .flat_map(|vault| {
                vault
                    .mounts()
                    .into_iter()
                    .map(|mount| (vault.key(&mount.location), mount))
            })
            .collect()
    }

    /// The URL of the static file behind an index key: inside an external
    /// overlay, its path in the overlay vault; anywhere else,
    /// [`build_static_url_path`] against the root.
    fn static_url_for_key(&self, key: &Path) -> String {
        match self.static_vault.as_deref() {
            Some(overlay) if !key.starts_with(&self.canonical_root) => {
                match overlay.vault_path(key) {
                    Ok(path) => format!("/{path}"),
                    Err(_) => build_static_url_path(key, &self.canonical_root, &self.static_folder),
                }
            }
            _ => build_static_url_path(key, &self.canonical_root, &self.static_folder),
        }
    }

    /// Size and `(created, modified)` seconds for the file behind `key`, read
    /// through its vault; `None` when it cannot be stat'ed or located.
    fn basic_details(&self, key: &Path) -> Option<(u64, u64, u64)> {
        let (vault, path) = self.locate(key)?;
        let entry = vault.stat(&path).ok()??;
        let (created, modified) = entry.epoch_secs()?;
        Some((entry.size, created, modified))
    }

    /// The vault a queued folder belongs to.
    fn vault_at(&self, location: ScanLocation) -> Option<&Arc<dyn Vault>> {
        match location {
            ScanLocation::WithinRoot => Some(&self.vault),
            ScanLocation::StaticOverlay => self.static_vault.as_ref(),
        }
    }

    /// Converts an absolute file path into the repo-relative form stored in
    /// [`MarkdownInfo::raw_path`].
    ///
    /// Uses `diff_paths` rather than `strip_prefix` so a file that somehow sits
    /// outside the root still yields a *relative* result (`../outside.md`)
    /// instead of an absolute one — `site.json` must never contain an absolute
    /// host path. Both inputs are absolute in practice, so the fallback is
    /// unreachable.
    fn relative_to_root(&self, abs_path: &Path) -> PathBuf {
        pathdiff::diff_paths(abs_path, &self.canonical_root)
            .unwrap_or_else(|| abs_path.to_path_buf())
    }

    /// The static folder as a path inside the root vault, when it is one
    /// (see [`crate::vault::configured_folder`]).
    fn static_folder_path(&self) -> Option<VaultPath> {
        crate::vault::configured_folder(self.vault.as_ref(), &self.static_folder)
    }

    /// The vault holding the static folder and the folder's path in it: the
    /// external overlay from its root, or the configured folder inside the
    /// repository. `None` when there is no static folder.
    pub fn static_folder_vault(&self) -> Option<(&Arc<dyn Vault>, VaultPath)> {
        match &self.static_vault {
            Some(overlay) => Some((overlay, VaultPath::root())),
            None => Some((&self.vault, self.static_folder_path()?)),
        }
    }

    /// The canonical key of the static folder, wherever it lives.
    fn static_folder_key(&self) -> Option<PathBuf> {
        match &self.static_vault {
            Some(overlay) => overlay
                .canonicalize(&VaultPath::root())
                .ok()
                .map(|root| overlay.key(&root)),
            None => {
                let path = self.static_folder_path()?;
                let canonical = self.vault.canonicalize(&path).ok()?;
                Some(self.vault.key(&canonical))
            }
        }
    }

    /// Scans one folder of the root vault (non-recursively: subfolders are
    /// queued in [`Self::queued_folders`], not walked).
    ///
    /// `relative_folder_path` is relative to the root. One that climbs out of
    /// it is skipped with a warning, as a folder that *resolves* out of it is.
    pub fn scan_folder<P: AsRef<Path>>(&self, relative_folder_path: &P) -> Result<(), RepoError> {
        let relative = relative_folder_path.as_ref();
        match VaultPath::from_relative_native(relative) {
            Ok(folder) => self.scan_dir(ScanLocation::WithinRoot, &folder),
            Err(e) => {
                tracing::warn!("Skipping {relative:?}: {e}");
                Ok(())
            }
        }
    }

    /// [`Self::scan_folder`] for a folder given as a vault path.
    pub fn scan_vault_folder(&self, folder: &VaultPath) -> Result<(), RepoError> {
        self.scan_dir(ScanLocation::WithinRoot, folder)
    }

    /// Scans one folder of the vault at `location`.
    fn scan_dir(&self, location: ScanLocation, folder: &VaultPath) -> Result<(), RepoError> {
        let Some(vault) = self.vault_at(location) else {
            return Ok(());
        };

        // A directory symlink can resolve outside the vault, and canonicalizing
        // re-roots the walk at its target. Every file found below it would then
        // relativize to `../…`, which `url_path::path_to_url` deliberately
        // preserves — so the URL escapes the site and, in build mode,
        // `output_dir.join(url_path)` writes pages outside `--output`. Refuse to
        // descend instead.
        //
        // The validated static overlay is the one exception, and a narrow one:
        // it is a specific directory the config policy already approved, scanned
        // through a vault of its own — not a general loosening. A link out of
        // the overlay is refused by that vault the same way.
        //
        // A link out of a vault built with mounts (every production vault) was
        // judged by the vault itself as it canonicalized: accepted, it comes
        // back as a path inside the vault and is walked like any folder;
        // refused, the vault has already logged why, once per link.
        let start = match vault.canonicalize(folder) {
            Ok(start) => start,
            Err(VaultError::OutsideRoot { path }) => {
                tracing::debug!(
                    "Skipping {:?}: it resolves to {}, outside the repository root {}",
                    folder.as_str(),
                    path,
                    vault.root().display()
                );
                return Ok(());
            }
            Err(e) => {
                return Err(RepoError::CanonicalizeFailed {
                    path: vault.key(folder),
                    source: e.into(),
                });
            }
        };
        let start_key = vault.key(&start);

        // Skip if already scanned
        if self.scanned_folders.pin().contains(&start_key) {
            return Ok(());
        }
        tracing::debug!("Scanning folder: {:?}", folder.as_str());
        self.scanned_folders.pin().insert(start_key.clone());

        // One listing per folder, every child already stat'ed (kind, size,
        // times). A folder that cannot be listed contributes nothing, as the
        // old `WalkDir` loop's error entries did.
        let list_start = Instant::now();
        let entries = match vault.list_dir(&start) {
            Ok(entries) => {
                self.not_downloaded_folders.pin().remove(&start_key);
                entries
            }
            // A file provider's folder whose listing is not on the device
            // (iCloud Drive on iOS keeps a folder *dataless* until something
            // lists it on a thread allowed to wait, and mbr's threads are
            // not). Ask for it, and let the folder be walked again: the
            // refresh that first lists it indexes its files.
            Err(VaultError::NotDownloaded { .. }) => {
                tracing::info!(
                    "Folder {} is not downloaded yet; requested it",
                    start_key.display()
                );
                vault.request_download(std::slice::from_ref(&start));
                self.not_downloaded_folders.pin().insert(start_key.clone());
                self.scanned_folders.pin().remove(&start_key);
                Vec::new()
            }
            // A folder that cannot be listed contributes nothing, as the old
            // `WalkDir` loop's error entries did.
            Err(e) => {
                tracing::debug!("Could not list {}: {e}", start_key.display());
                Vec::new()
            }
        };

        if start.is_root() {
            // What the root lists decides everything below it, and a file
            // provider can list less than its own UI shows: say what we got.
            let folders = entries.iter().filter(|e| e.is_dir()).count();
            tracing::info!(
                "Root {} lists {} entries ({} folders, {} files)",
                start_key.display(),
                entries.len(),
                folders,
                entries.len() - folders
            );
        }

        let mut markdown: Vec<(PathBuf, VaultPath, MarkdownInfo)> = Vec::new();
        let mut other: Vec<(PathBuf, OtherFileInfo)> = Vec::new();

        for entry in entries {
            // The key is the path the old walk produced: the canonical folder
            // plus the child's own name, so a symlinked *file* keeps its own
            // name, and ignore globs keep matching the same absolute paths.
            let key = vault.key(&entry.path);
            if should_ignore_compiled(
                &key,
                entry.is_dir(),
                &self.ignore_dirs,
                &self.compiled_ignore_globs,
                &self.exempt_hidden_dirs,
            ) {
                continue;
            }
            let extension = entry.path.extension().unwrap_or("");

            if entry.is_dir() {
                // Queue subdirectory for later scanning. Whether it may be
                // walked is decided on the next hop, when it is canonicalized.
                self.queued_folders.pin().insert(
                    key,
                    QueuedFolder {
                        location,
                        path: entry.path,
                    },
                );
            } else if location == ScanLocation::StaticOverlay
                && is_markdown_extension(extension, &self.markdown_extensions)
            {
                // Markdown inside an *external* static overlay has no
                // representable URL: `build_markdown_url_path` relativizes it to
                // `../static/…`, `url_path::path_to_url` preserves the `..` on
                // purpose, and build mode then joins that onto `output_dir` —
                // writing the page outside `--output`. That escape is exactly
                // what this pass fixed in `build.rs`, so the file is skipped
                // rather than indexed with a broken URL. Static assets are
                // unaffected: their URL strips the overlay prefix.
                tracing::warn!(
                    "Skipping markdown file {} in the external static folder {:?}: \
                     markdown outside the repository root has no valid URL",
                    key.display(),
                    self.static_folder
                );
            } else if is_markdown_extension(extension, &self.markdown_extensions) {
                // Process markdown file
                self.record_stamp(&key, &entry);
                if let Some((created, modified)) = entry.epoch_secs() {
                    let url = build_markdown_url_path(&key, &self.canonical_root, &self.index_file);
                    let mdfile = MarkdownInfo {
                        raw_path: self.relative_to_root(&key),
                        url_path: url,
                        created,
                        modified,
                        frontmatter: None,
                        relationships: Vec::new(),
                    };
                    markdown.push((key, entry.path, mdfile));
                } else {
                    tracing::warn!("Couldn't process markdown file at {:?}", key);
                }
            } else {
                // Process static file
                //
                // Inside the overlay the URL is the path within the overlay
                // vault — the *resolved* directory stripped, never the
                // configured string. `build_static_url_path` strips the raw
                // value with a component-wise `strip_prefix`, so `../../static`
                // and the equivalent `./../../static` — both accepted by the
                // config policy — do not both match, and the second would leave
                // a `..` in the URL.
                let url = match location {
                    ScanLocation::StaticOverlay => format!(
                        "/{}",
                        crate::url_path::path_to_url(
                            key.strip_prefix(vault.root()).unwrap_or(&key)
                        )
                    ),
                    ScanLocation::WithinRoot => {
                        build_static_url_path(&key, &self.canonical_root, &self.static_folder)
                    }
                };
                // Basic metadata (size, timestamps) stays deferred to
                // `populate_basic_metadata()`, as it always was: the static
                // build never runs that pass, so recording the listing's
                // numbers here would change its `media.json`.
                self.record_stamp(&key, &entry);
                let other_file = OtherFileInfo {
                    raw_path: key.clone(),
                    url_path: url,
                    metadata: StaticFileMetadata::empty(&key),
                    extracted_text: None,
                };
                other.push((key, other_file));
            }
        }
        tracing::debug!(
            "scan_folder listing for {:?}: {} markdown, {} other files in {:?}",
            folder.as_str(),
            markdown.len(),
            other.len(),
            list_start.elapsed()
        );

        // Parallel processing: extract frontmatter from markdown files and build tag index
        let frontmatter_start = Instant::now();
        markdown.into_par_iter().for_each(
            |(key, path, mddetails): (PathBuf, VaultPath, MarkdownInfo)| {
                let file_meta =
                    crate::markdown::extract_metadata_from_vault(vault.as_ref(), &path).ok();
                let details = if let Some(file_meta) = file_meta {
                    let frontmatter = file_meta.metadata;
                    let relationships = file_meta.relationships;
                    // Extract tags from frontmatter for each configured tag source
                    let title = get_page_title(&frontmatter, &mddetails.raw_path);
                    let description = frontmatter
                        .get("description")
                        .and_then(|v| v.as_str())
                        .map(|s| s.to_string());

                    for tag_source in &self.tag_sources {
                        // Look up the field (supports dot notation like "taxonomy.tags")
                        if let Some(tag_value_json) = frontmatter.get(&tag_source.field) {
                            // Extract tag values (handles both arrays and comma-separated strings)
                            for tag_value in extract_tag_values(tag_value_json) {
                                let page = if let Some(ref desc) = description {
                                    TaggedPage::with_description(
                                        &mddetails.url_path,
                                        &title,
                                        desc,
                                        &tag_value,
                                    )
                                } else {
                                    TaggedPage::new(&mddetails.url_path, &title, &tag_value)
                                };
                                self.tag_index.add_page(&tag_source.field, &tag_value, page);
                            }
                        }
                    }

                    MarkdownInfo {
                        frontmatter: Some(frontmatter),
                        relationships,
                        ..mddetails
                    }
                } else {
                    mddetails
                };
                self.markdown_files.pin().insert(key, details);
            },
        );
        tracing::debug!(
            "scan_folder frontmatter extraction for {:?}: {:?}",
            folder.as_str(),
            frontmatter_start.elapsed()
        );

        let static_insert_start = Instant::now();
        for (file, other_file) in other {
            self.other_files.pin().insert(file, other_file);
        }
        tracing::debug!(
            "scan_folder static file registration for {:?}: {:?}",
            folder.as_str(),
            static_insert_start.elapsed()
        );

        Ok(())
    }

    /// Walks every queued folder, one parallel batch per directory level,
    /// until nothing is queued.
    fn drain_queued_folders(&self) {
        while !self.queued_folders.is_empty() {
            // TODO: make sure this doesn't deadlock
            let batch: Vec<QueuedFolder> = self
                .queued_folders
                .pin()
                .iter()
                .map(|(_, queued)| queued.clone())
                .collect();
            self.queued_folders.pin().clear();
            tracing::debug!("Parallel batch: {:?}", &batch);
            batch.into_par_iter().for_each(|queued| {
                self.scan_dir(queued.location, &queued.path)
                    .unwrap_or_else(|e| {
                        tracing::error!("Failed to scan folder {:?}: {e}", queued.path.as_str())
                    }) // ignores errors
            });
        }
    }

    pub fn scan_all(&self) -> Result<(), RepoError> {
        let scan_all_start = Instant::now();

        // Pre-mark static folder as scanned to skip it during content scan.
        // This defers static file registration to scan_static_folder() so
        // mark_scan_complete() fires faster (search only needs markdown).
        let static_deferred = self.static_folder_key().inspect(|key| {
            self.scanned_folders.pin().insert(key.clone());
        });

        self.scan_dir(ScanLocation::WithinRoot, &VaultPath::root())?;
        self.drain_queued_folders();

        // Un-mark static folder so scan_static_folder() can scan it later
        if let Some(ref sp) = static_deferred {
            self.scanned_folders.pin().remove(sp);
        }

        // Log file counts and type breakdown
        let markdown_count = self.markdown_files.len();
        let other_count = self.other_files.len();
        let other_pin = self.other_files.pin();
        let mut pdf_count: usize = 0;
        let mut image_count: usize = 0;
        let mut video_count: usize = 0;
        let mut audio_count: usize = 0;
        let mut text_count: usize = 0;
        let mut misc_count: usize = 0;
        for (_, info) in other_pin.iter() {
            match info.filetype() {
                "pdf" => pdf_count += 1,
                "image" => image_count += 1,
                "video" => video_count += 1,
                "audio" => audio_count += 1,
                "text" => text_count += 1,
                _ => misc_count += 1,
            }
        }

        tracing::info!(
            "scan_all completed in {:?}: {} markdown files, {} other files in {} folders, {} not downloaded (pdf={}, image={}, video={}, audio={}, text={}, other={})",
            scan_all_start.elapsed(),
            markdown_count,
            other_count,
            self.scanned_folders.len(),
            self.not_downloaded_folders.len(),
            pdf_count,
            image_count,
            video_count,
            audio_count,
            text_count,
            misc_count,
        );

        Ok(())
    }

    /// Scan the static folder and its subdirectories.
    /// Deferred from scan_all() so mark_scan_complete() fires faster.
    pub fn scan_static_folder(&self) -> Result<(), RepoError> {
        let start = Instant::now();
        // An external overlay is its own vault, scanned from its root; an
        // in-root static folder is a folder of the root vault.
        let (location, folder) = if self.static_vault.is_some() {
            (ScanLocation::StaticOverlay, VaultPath::root())
        } else {
            match self.static_folder_path() {
                Some(folder) => (ScanLocation::WithinRoot, folder),
                None => return Ok(()),
            }
        };
        if !self
            .vault_at(location)
            .is_some_and(|vault| vault.is_dir(&folder))
        {
            return Ok(());
        }

        self.scan_dir(location, &folder)?;
        self.drain_queued_folders();

        let other_count = self.other_files.len();
        tracing::info!(
            "scan_static_folder completed in {:?}: {} other files total",
            start.elapsed(),
            other_count,
        );
        Ok(())
    }

    /// Mark the initial scan as complete and notify any waiters.
    pub fn mark_scan_complete(&self) {
        self.scan_complete.store(true, Ordering::SeqCst);
        self.scan_notify.notify_waiters();
    }

    /// Drop every cached view of the repository and rebuild it from disk.
    ///
    /// Used by the watcher when a change batch is too large to invalidate file
    /// by file. Blocking; call from `spawn_blocking`.
    ///
    /// **No step may return early.** `clear()` resets `media_populated`, so any
    /// path that skips [`Repo::notify_media_populated`] leaves every later
    /// [`Repo::wait_for_media`] — that is, `/.mbr/media.json` — blocked for the
    /// life of the process. A failed scan is logged and the pass continues on
    /// whatever was indexed: `clear()` already discarded the previous contents,
    /// so the real choice is between serving partial data and hanging.
    pub fn full_rescan(&self) {
        self.clear();
        if let Err(e) = self.scan_all() {
            tracing::error!("Rescan failed, continuing with partial repository state: {e}");
        }
        self.build_relationship_index();
        self.build_wikilink_index();
        if let Err(e) = self.scan_static_folder() {
            tracing::error!("Static folder rescan failed: {e}");
        }
        self.populate_basic_metadata();
        self.populate_media_metadata();
        self.notify_media_populated();
        self.ensure_text_extracted();
    }

    /// Populate basic file metadata (size, timestamps) for all other files.
    /// Deferred from scan_folder() to avoid blocking scan_all() completion,
    /// so search (which only needs markdown files) can proceed sooner.
    pub fn populate_basic_metadata(&self) {
        let start = Instant::now();
        let pin = self.other_files.pin();
        let keys: Vec<PathBuf> = pin
            .iter()
            .filter(|(_, info)| info.metadata.file_size_bytes.is_none())
            .map(|(k, _)| k.clone())
            .collect();
        let count = keys.len();
        drop(pin);

        keys.into_par_iter().for_each(|key| {
            let pin = self.other_files.pin();
            if let Some(info) = pin.get(&key) {
                let updated = OtherFileInfo {
                    metadata: info.metadata.clone().with_details(self.basic_details(&key)),
                    ..info.clone()
                };
                drop(pin);
                self.other_files.pin().insert(key, updated);
            }
        });

        tracing::info!(
            "populate_basic_metadata completed for {} files in {:?}",
            count,
            start.elapsed()
        );
    }

    /// Returns true if the initial scan has completed.
    pub fn is_scan_complete(&self) -> bool {
        self.scan_complete.load(Ordering::SeqCst)
    }

    /// Wait for the initial scan to complete. Returns immediately if already done.
    ///
    /// `mark_scan_complete()` fires exactly once and `Notify::notify_waiters()`
    /// stores no permit, so a notification observed between a flag check and the
    /// registration of the waiter would be lost forever — and `/.mbr/site.json`
    /// would block for the life of the process. Registering interest *before*
    /// re-checking closes that window (same pattern as the in-flight waits in
    /// `server.rs`).
    pub async fn wait_for_scan(&self) {
        loop {
            let notified = self.scan_notify.notified();
            tokio::pin!(notified);
            notified.as_mut().enable();

            if self.is_scan_complete() {
                return;
            }

            // Re-checked by the loop: `notified()` also returns on spurious wakeups.
            notified.await;
        }
    }

    /// Returns true if media metadata has been populated.
    pub fn is_media_populated(&self) -> bool {
        self.media_populated.load(Ordering::Acquire)
    }

    /// Notify waiters that media metadata population is complete.
    pub fn notify_media_populated(&self) {
        self.media_notify.notify_waiters();
    }

    /// Wait for media metadata to be populated. Returns immediately if already done.
    ///
    /// Registers interest before re-checking the flag, for the same reason as
    /// [`Repo::wait_for_scan`].
    pub async fn wait_for_media(&self) {
        loop {
            let notified = self.media_notify.notified();
            tokio::pin!(notified);
            notified.as_mut().enable();

            if self.is_media_populated() {
                return;
            }

            notified.await;
        }
    }

    pub fn to_json(&self) -> serde_json::Result<String> {
        serde_json::to_string(self)
    }

    /// Populate media metadata (ffmpeg/lopdf) for all static files.
    /// This is deferred from initial scan to avoid blocking the first site.json response.
    pub fn populate_media_metadata(&self) {
        if self.media_population_started.swap(true, Ordering::SeqCst) {
            return; // Already running, or already finished
        }
        let start = Instant::now();
        let pin = self.other_files.pin();
        let entries: Vec<_> = pin.iter().map(|(k, v)| (k.clone(), v.clone())).collect();
        drop(pin);

        // ffmpeg and pdfium open files by path, so only files with a
        // `local_path` can be probed; the rest keep their basic metadata.
        #[cfg(feature = "media-metadata")]
        {
            entries.into_par_iter().for_each(|(key, info)| {
                let Some(local) = self
                    .locate(&key)
                    .and_then(|(vault, path)| vault.local_path(&path))
                else {
                    return;
                };
                let updated = OtherFileInfo {
                    metadata: StaticFileMetadata {
                        path: local,
                        ..info.metadata
                    }
                    .populate_media(),
                    ..info
                };
                self.other_files.pin().insert(key, updated);
            });
        }
        #[cfg(not(feature = "media-metadata"))]
        let _ = entries;

        // Publish "finished" only now that every entry carries its metadata.
        // Callers invoke `notify_media_populated()` immediately after this
        // returns; waiters re-read the flag after being woken, so the store must
        // happen first.
        self.media_populated.store(true, Ordering::Release);

        tracing::info!("populate_media_metadata completed in {:?}", start.elapsed());
    }

    /// Extract text from searchable files (PDFs, text files) for search indexing.
    /// Deferred from initial scan since text is only needed for search.
    pub fn ensure_text_extracted(&self) {
        if self.text_extracted.swap(true, Ordering::SeqCst) {
            return; // Already extracted
        }
        let start = Instant::now();
        let pin = self.other_files.pin();
        let entries: Vec<_> = pin
            .iter()
            .filter(|(_, info)| info.is_searchable() && info.extracted_text.is_none())
            .map(|(k, v)| (k.clone(), v.clone()))
            .collect();
        drop(pin);

        entries.into_par_iter().for_each(|(key, mut info)| {
            info.extracted_text = self
                .locate(&key)
                .and_then(|(vault, path)| info.extract_text(vault, &path));
            self.other_files.pin().insert(key, info);
        });

        tracing::info!("ensure_text_extracted completed in {:?}", start.elapsed());
    }

    /// Clear all cached data, forcing a full rescan on next scan_all() call.
    ///
    /// Call this when files are added, removed, or modified to ensure
    /// the next scan picks up the changes.
    pub fn clear(&self) {
        self.scanned_folders.pin().clear();
        self.not_downloaded_folders.pin().clear();
        self.markdown_files.pin().clear();
        self.other_files.pin().clear();
        self.queued_folders.pin().clear();
        self.tag_index.clear();
        self.relationship_index.clear();
        self.wikilink_index.clear();
        if let Some(stamps) = &self.stamps {
            stamps.pin().clear();
        }
        self.text_extracted.store(false, Ordering::SeqCst);
        self.media_populated.store(false, Ordering::SeqCst);
        // Must be reset too, or the run-once guard would make the rescan's
        // `populate_media_metadata()` a no-op and leave `media_populated` false
        // forever, hanging every later `wait_for_media()`.
        self.media_population_started.store(false, Ordering::SeqCst);
        // Note: scan_complete is NOT reset here. It tracks whether the initial background
        // scan finished. After clear(), scan_all() will re-scan synchronously in handlers.
    }

    /// The vault, vault path and `(created, modified)` seconds of the markdown
    /// file behind `key`, when it can be located and stat'ed — the
    /// `file_details_from_path` check `invalidate_file` used to make.
    fn markdown_source(&self, key: &Path) -> Option<(&dyn Vault, VaultPath, u64, u64)> {
        let (vault, path) = self.locate(key)?;
        let entry = vault.stat(&path).ok()??;
        let (created, modified) = entry.epoch_secs()?;
        Some((vault, path, created, modified))
    }

    /// Surgically invalidate a single file, updating only the affected cache entries.
    ///
    /// Much cheaper than `clear()` + `scan_all()` for small batches of file changes.
    pub fn invalidate_file(&self, abs_path: &Path, event: &crate::change_event::ChangeEventType) {
        self.invalidate_file_entry(abs_path, event);
        self.restamp(abs_path, event);
    }

    /// Keeps a change-tracked file's stamp in step with an applied change —
    /// a watcher event, the server's own write, a refresh — so the next
    /// [`Self::diff_vault`] does not report it again. One `stat`, and only
    /// with change tracking.
    fn restamp(&self, key: &Path, event: &crate::change_event::ChangeEventType) {
        let Some(stamps) = &self.stamps else {
            return;
        };
        let entry = match event {
            crate::change_event::ChangeEventType::Deleted => None,
            _ => self
                .locate(key)
                .and_then(|(vault, path)| vault.stat(&path).ok().flatten())
                .filter(Entry::is_file),
        };
        match entry {
            Some(entry)
                if self.markdown_files.pin().contains_key(key)
                    || self.other_files.pin().contains_key(key) =>
            {
                stamps
                    .pin()
                    .insert(key.to_path_buf(), FileStamp::of(&entry));
            }
            _ => {
                stamps.pin().remove(key);
            }
        }
    }

    fn invalidate_file_entry(&self, abs_path: &Path, event: &crate::change_event::ChangeEventType) {
        let extension = abs_path.extension().and_then(|x| x.to_str()).unwrap_or("");
        let is_markdown = is_markdown_extension(extension, &self.markdown_extensions);
        // Markdown outside the root (in an external overlay, or a mount inside
        // one) has no page URL, so the scanner never indexes it; neither may an
        // event. The same goes for any hidden path.
        if (is_markdown && !abs_path.starts_with(&self.canonical_root))
            || self.locate(abs_path).is_some_and(|(_, path)| {
                crate::vault::has_hidden_segment(&path, self.exempt_hidden_dirs())
            })
        {
            return;
        }

        match event {
            crate::change_event::ChangeEventType::Deleted => {
                if is_markdown {
                    self.markdown_files.pin().remove(abs_path);
                } else {
                    self.other_files.pin().remove(abs_path);
                }
            }
            crate::change_event::ChangeEventType::Created => {
                if is_markdown {
                    if let Some((vault, path, created, modified)) = self.markdown_source(abs_path) {
                        let url = build_markdown_url_path(
                            abs_path,
                            &self.canonical_root,
                            &self.index_file,
                        );
                        let file_meta =
                            crate::markdown::extract_metadata_from_vault(vault, &path).ok();
                        let (frontmatter, relationships) = match file_meta {
                            Some(fm) => (Some(fm.metadata), fm.relationships),
                            None => (None, Vec::new()),
                        };

                        // Add tags from frontmatter
                        if let Some(ref fm) = frontmatter {
                            let title = get_page_title(fm, abs_path);
                            let description = fm
                                .get("description")
                                .and_then(|v| v.as_str())
                                .map(|s| s.to_string());
                            for tag_source in &self.tag_sources {
                                if let Some(tag_value_json) = fm.get(&tag_source.field) {
                                    for tag_value in extract_tag_values(tag_value_json) {
                                        let page = if let Some(ref desc) = description {
                                            TaggedPage::with_description(
                                                &url, &title, desc, &tag_value,
                                            )
                                        } else {
                                            TaggedPage::new(&url, &title, &tag_value)
                                        };
                                        self.tag_index.add_page(
                                            &tag_source.field,
                                            &tag_value,
                                            page,
                                        );
                                    }
                                }
                            }
                        }

                        let info = MarkdownInfo {
                            raw_path: self.relative_to_root(abs_path),
                            url_path: url,
                            created,
                            modified,
                            frontmatter,
                            relationships,
                        };
                        self.markdown_files
                            .pin()
                            .insert(abs_path.to_path_buf(), info);
                    }
                } else {
                    let url = self.static_url_for_key(abs_path);
                    let info = OtherFileInfo {
                        raw_path: abs_path.to_path_buf(),
                        url_path: url,
                        metadata: StaticFileMetadata::empty(abs_path)
                            .with_details(self.basic_details(abs_path)),
                        extracted_text: None,
                    };
                    self.other_files.pin().insert(abs_path.to_path_buf(), info);
                }
            }
            crate::change_event::ChangeEventType::Modified => {
                if is_markdown {
                    // Re-extract frontmatter and update
                    if let Some((vault, path, created, modified)) = self.markdown_source(abs_path) {
                        let url = build_markdown_url_path(
                            abs_path,
                            &self.canonical_root,
                            &self.index_file,
                        );
                        let file_meta =
                            crate::markdown::extract_metadata_from_vault(vault, &path).ok();
                        let (frontmatter, relationships) = match file_meta {
                            Some(fm) => (Some(fm.metadata), fm.relationships),
                            None => (None, Vec::new()),
                        };
                        let info = MarkdownInfo {
                            raw_path: self.relative_to_root(abs_path),
                            url_path: url,
                            created,
                            modified,
                            frontmatter,
                            relationships,
                        };
                        self.markdown_files
                            .pin()
                            .insert(abs_path.to_path_buf(), info);
                    }
                } else {
                    // Update basic metadata for modified static files
                    let url = self.static_url_for_key(abs_path);
                    let info = OtherFileInfo {
                        raw_path: abs_path.to_path_buf(),
                        url_path: url,
                        metadata: StaticFileMetadata::empty(abs_path)
                            .with_details(self.basic_details(abs_path)),
                        extracted_text: None,
                    };
                    self.other_files.pin().insert(abs_path.to_path_buf(), info);
                }
            }
        }
    }

    /// Rebuild the tag index from the current cached markdown files.
    ///
    /// Call this after surgical file invalidation when tags may have changed
    /// (e.g., after deleting or modifying markdown files).
    /// Returns the configured tag sources.
    pub fn tag_sources(&self) -> &[TagSource] {
        &self.tag_sources
    }

    /// Rebuild the relationship index from the current cached markdown files.
    ///
    /// Must run after a scan (or file-change invalidation) once all note titles
    /// are known, since endpoint resolution matches on titles and filename
    /// stems across the whole repo. Modelled on [`Self::rebuild_tag_index`].
    pub fn build_relationship_index(&self) {
        // Short-circuit when no note declares any relationship: avoid the
        // per-note NoteRelInput cloning and the O(n log n) sort / name-index /
        // rebuild in `build_relationship_map` (mirrors the gated tag-index
        // rebuild). An empty rebuild is otherwise pure overhead.
        if !self
            .markdown_files
            .pin()
            .iter()
            .any(|(_, info)| !info.relationships.is_empty())
        {
            self.relationship_index.clear();
            return;
        }
        let notes = self.collect_note_inputs();
        self.relationship_index
            .rebuild(&notes, &self.markdown_extensions);
    }

    /// Rebuild the global wikilink name index from the current cached markdown
    /// files.
    ///
    /// Unlike [`Self::build_relationship_index`] this is **ungated** — the index
    /// powers Obsidian-style `[[Name]]` body-link resolution for every repo, so
    /// it is rebuilt whenever the repo is (re)scanned or a file changes. Must run
    /// after a scan once all note titles/stems are known.
    pub fn build_wikilink_index(&self) {
        let notes = self.collect_note_inputs();
        self.wikilink_index.rebuild(&notes);
    }

    /// Assembles one [`NoteRelInput`] per cached markdown file (url, title,
    /// filename stem, frontmatter aliases, index flag, declared relationships).
    ///
    /// Shared by [`Self::build_relationship_index`] and
    /// [`Self::build_wikilink_index`] so the per-note title/stem/alias derivation
    /// lives in one place.
    fn collect_note_inputs(&self) -> Vec<NoteRelInput> {
        let pin = self.markdown_files.pin();
        pin.iter()
            .map(|(_, info)| {
                let title = info
                    .frontmatter
                    .as_ref()
                    .map(|fm| get_page_title(fm, &info.raw_path))
                    .unwrap_or_else(|| {
                        info.raw_path
                            .file_stem()
                            .and_then(|s| s.to_str())
                            .unwrap_or("Untitled")
                            .to_string()
                    });
                let stem = info
                    .raw_path
                    .file_stem()
                    .and_then(|s| s.to_str())
                    .unwrap_or_default()
                    .to_string();
                // Alternate names (e.g. maiden names) that also resolve to this
                // note — labeled ones included; see `contact::alias_names`.
                let aliases = info
                    .frontmatter
                    .as_ref()
                    .map(crate::contact::alias_names_in)
                    .unwrap_or_default();
                let is_index = info
                    .raw_path
                    .file_name()
                    .and_then(|f| f.to_str())
                    .is_some_and(|f| f == self.index_file);
                NoteRelInput {
                    url: info.url_path.clone(),
                    title,
                    stem,
                    aliases,
                    is_index,
                    relationships: info.relationships.clone(),
                }
            })
            .collect()
    }

    pub fn rebuild_tag_index(&self) {
        self.tag_index.clear();
        let pin = self.markdown_files.pin();
        for (_, info) in pin.iter() {
            if let Some(ref fm) = info.frontmatter {
                let title = get_page_title(fm, &info.raw_path);
                let description = fm
                    .get("description")
                    .and_then(|v| v.as_str())
                    .map(|s| s.to_string());
                for tag_source in &self.tag_sources {
                    if let Some(tag_value_json) = fm.get(&tag_source.field) {
                        for tag_value in extract_tag_values(tag_value_json) {
                            let page = if let Some(ref desc) = description {
                                TaggedPage::with_description(
                                    &info.url_path,
                                    &title,
                                    desc,
                                    &tag_value,
                                )
                            } else {
                                TaggedPage::new(&info.url_path, &title, &tag_value)
                            };
                            self.tag_index.add_page(&tag_source.field, &tag_value, page);
                        }
                    }
                }
            }
        }
    }
}

/// Returns file_size, created_secs, modified_secs
pub fn file_details_from_path<P: AsRef<Path>>(path: P) -> Result<(u64, u64, u64), RepoError> {
    let path = path.as_ref();
    let metadata = std::fs::metadata(path).map_err(|source| RepoError::MetadataFailed {
        path: path.to_path_buf(),
        source,
    })?;

    let file_size = metadata.len();

    // Modified time
    let modified = metadata
        .modified()
        .map_err(|source| RepoError::MetadataFailed {
            path: path.to_path_buf(),
            source,
        })?;
    let modified_secs = modified.duration_since(UNIX_EPOCH)?.as_secs();

    // Created time (birth time) is not available on all filesystems (e.g. older Linux
    // kernels, some NFS mounts). Fall back to the modified time rather than failing, so
    // the file is never silently dropped from listings/search when btime is unavailable.
    let created_secs = metadata
        .created()
        .ok()
        .and_then(|created| created.duration_since(UNIX_EPOCH).ok())
        .map(|d| d.as_secs())
        .unwrap_or(modified_secs);

    Ok((file_size, created_secs, modified_secs))
}

// ============================================================================
// Pure helper functions for repo scanning (extracted for testability)
// ============================================================================

/// Returns the repo-relative form of `abs_path`, or `None` when it resolves
/// *outside* `root`.
///
/// Containment is decided component-wise, not with `Path::starts_with`: the
/// relative path is what the rest of the pipeline consumes, and a single
/// `ParentDir` in it means the file is not in the repository at all. A
/// `RootDir`/`Prefix` component means [`pathdiff::diff_paths`] could not
/// relativize the two at all (mixed absolute/relative inputs, or different
/// Windows drives) and handed back something absolute.
///
/// This is the scan-side half of the containment guard: `url_path::path_to_url`
/// preserves `..` on purpose, so an escaping path stays escaped all the way into
/// `site.json` URLs and into `output_dir.join(url_path)` during a static build.
/// Which of the scanner's two legitimate roots a path belongs to.
///
/// There are exactly two: the repository root, and the validated external static
/// overlay. Anything else is not scannable at all, so this enum has no third
/// variant by design.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ScanLocation {
    /// The repository's own vault.
    WithinRoot,
    /// The external static overlay's vault.
    StaticOverlay,
}

/// A folder found by a scan but not walked yet: which vault it is in, and
/// where. See [`Repo::queued_folders`].
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct QueuedFolder {
    pub location: ScanLocation,
    pub path: VaultPath,
}

pub fn repo_relative_within_root(root: &Path, abs_path: &Path) -> Option<PathBuf> {
    let relative = pathdiff::diff_paths(abs_path, root)?;

    let escapes = relative.components().any(|component| {
        matches!(
            component,
            Component::ParentDir | Component::RootDir | Component::Prefix(_)
        )
    });

    (!escapes).then_some(relative)
}

/// The leading-dot rule, and its one exemption.
///
/// Hidden entries are skipped because a repository's dot directories are tooling
/// state rather than content — `.git`, `.direnv`, `.obsidian`. The exemption is
/// for the case where that inference is provably wrong: the user *named* the
/// directory on the command line. `mbr -s .scratch` still roots at the enclosing
/// checkout (root discovery walks upward, and must — see
/// [`crate::config::find_root_dir`]), so without this the scan would index the
/// whole repository *except* the one folder that was asked for.
///
/// Only the chain from the root down to that target is exempt. A dot directory
/// nested *inside* it was not named and gets no waiver: naming `.scratch` says
/// something about `.scratch`, not about a `.git` that happens to live under it.
///
/// `exempt_hidden_dirs` is empty for every invocation that named no hidden
/// directory, and non-empty ones hold one or two entries. `starts_with('.')`
/// short-circuits ahead of the scan, so on the per-entry hot path this costs the
/// same character comparison it always did — no allocation, no canonicalization.
#[inline]
fn is_ignored_hidden(path: &Path, file_name: &str, exempt_hidden_dirs: &[PathBuf]) -> bool {
    file_name.starts_with('.') && !exempt_hidden_dirs.iter().any(|dir| dir == path)
}

/// Checks if a path should be ignored based on the given rules.
///
/// A path is ignored if:
/// - Its name starts with '.', unless it is one of `exempt_hidden_dirs`
///   (see [`is_ignored_hidden`])
/// - It's a directory matching one of the ignore_dirs
/// - It matches one of the ignore_globs patterns
///
/// `exempt_hidden_dirs` must be expressed in the same base as `path`: callers
/// that pass absolute paths pass absolute exemptions, and the watcher — which
/// works entirely in repo-relative space so its glob matching stays anchored at
/// the root — passes repo-relative ones.
pub fn should_ignore(
    path: &Path,
    ignore_dirs: &[String],
    ignore_globs: &[String],
    exempt_hidden_dirs: &[PathBuf],
) -> bool {
    let file_name = path.file_name().and_then(|x| x.to_str()).unwrap_or("");

    // Hidden files/dirs (starting with .)
    if is_ignored_hidden(path, file_name, exempt_hidden_dirs) {
        return true;
    }

    // Directory matching ignore list
    if path.is_dir() && ignore_dirs.iter().any(|x| x.as_str() == file_name) {
        return true;
    }

    // Glob pattern match
    ignore_globs.iter().any(|pat| {
        glob::Pattern::new(pat)
            .map(|pat| pat.matches_path(path))
            .unwrap_or(false)
    })
}

/// Checks if a path should be ignored using pre-compiled glob patterns.
/// This is more efficient than `should_ignore` when processing many files.
///
/// `is_dir` comes from the listing that produced `path` (the target's kind,
/// symlinks followed), so the check costs no `stat` of its own.
fn should_ignore_compiled(
    path: &Path,
    is_dir: bool,
    ignore_dirs: &[String],
    compiled_patterns: &[glob::Pattern],
    exempt_hidden_dirs: &[PathBuf],
) -> bool {
    let file_name = path.file_name().and_then(|x| x.to_str()).unwrap_or("");

    // Hidden files/dirs (starting with .)
    if is_ignored_hidden(path, file_name, exempt_hidden_dirs) {
        return true;
    }

    // Directory matching ignore list
    if is_dir && ignore_dirs.iter().any(|x| x.as_str() == file_name) {
        return true;
    }

    // Pre-compiled glob pattern match
    compiled_patterns.iter().any(|pat| pat.matches_path(path))
}

/// Builds a URL path for a markdown file.
///
/// Converts a filesystem path relative to root into a URL path:
/// - Ensures leading slash
/// - Removes index file from path (e.g., /docs/index.md → /docs/)
/// - Replaces file extension with trailing slash
pub fn build_markdown_url_path(path: &Path, root_dir: &Path, index_file: &str) -> String {
    // `path_to_url` (not `to_string_lossy`) so the separators below are `/` on
    // every platform; otherwise the `rsplit('/')` and `contains('/')` checks
    // that follow silently stop matching on Windows.
    let relative = pathdiff::diff_paths(path, root_dir).unwrap_or_default();
    let mut url = format!("/{}", crate::url_path::path_to_url(&relative));

    // Remove index file from path — only when the final path component (file name)
    // exactly equals the index file, not merely when it's a suffix substring.
    // Otherwise "docs/myindex.md" would be wrongly truncated for index_file "index.md".
    if url.rsplit('/').next() == Some(index_file) {
        url.truncate(url.len() - index_file.len());
    }

    // Replace extension with trailing slash
    if let Some(dot_pos) = url.rfind('.')
        && !url[dot_pos..].contains('/')
    {
        url.truncate(dot_pos);
        url.push('/');
    }

    url
}

/// Builds a URL path for a static file.
///
/// Converts a filesystem path relative to root into a URL path:
/// - Removes static folder prefix
/// - Ensures leading slash
pub fn build_static_url_path(path: &Path, root_dir: &Path, static_folder: &str) -> String {
    let relative = pathdiff::diff_paths(path, root_dir).unwrap_or_default();

    // Strip only a *leading* `{static_folder}/` path component, not any substring
    // occurrence of the folder name. A plain `.replacen` would corrupt paths like
    // "notes/static-analysis/img.png" (with static_folder="static").
    let stripped = relative.strip_prefix(static_folder).unwrap_or(&relative);

    format!("/{}", crate::url_path::path_to_url(stripped))
}

/// Checks if a file has a markdown extension.
pub fn is_markdown_extension(extension: &str, markdown_extensions: &[String]) -> bool {
    markdown_extensions.iter().any(|x| x.as_str() == extension)
}

/// Parses a comma-separated string of tag values into individual tags.
///
/// Handles whitespace around commas and filters empty values.
///
/// # Examples
///
/// ```
/// use mbr_core::repo::parse_tag_values;
///
/// let tags: Vec<String> = parse_tag_values("rust, programming, web dev").collect();
/// assert_eq!(tags, vec!["rust", "programming", "web dev"]);
/// ```
pub fn parse_tag_values(values: &str) -> impl Iterator<Item = String> + '_ {
    values
        .split(',')
        .map(|s| s.trim().to_string())
        .filter(|s| !s.is_empty())
}

/// Extract tag values from a serde_json::Value (supports both arrays and comma-separated strings).
///
/// # Examples
///
/// ```
/// use mbr_core::repo::extract_tag_values;
///
/// // From array
/// let val = serde_json::json!(["rust", "python"]);
/// assert_eq!(extract_tag_values(&val), vec!["rust", "python"]);
///
/// // From comma-separated string
/// let val = serde_json::json!("rust, python");
/// assert_eq!(extract_tag_values(&val), vec!["rust", "python"]);
/// ```
pub fn extract_tag_values(value: &serde_json::Value) -> Vec<String> {
    match value {
        serde_json::Value::String(s) => parse_tag_values(s).collect(),
        serde_json::Value::Array(arr) => arr
            .iter()
            .filter_map(|v| v.as_str())
            .map(|s| s.to_string())
            .collect(),
        _ => vec![],
    }
}

/// Gets the page title from frontmatter or falls back to filename.
///
/// Priority:
/// 1. `title` field in frontmatter
/// 2. Filename stem (without extension)
fn get_page_title(frontmatter: &crate::markdown::SimpleMetadata, path: &Path) -> String {
    frontmatter
        .get("title")
        .and_then(|v| v.as_str())
        .map(|s| s.to_string())
        .unwrap_or_else(|| {
            path.file_stem()
                .and_then(|s| s.to_str())
                .unwrap_or("Untitled")
                .to_string()
        })
}

#[cfg(test)]
mod tests {
    use super::*;

    /// `Repo` must canonicalize the root it is handed, so that the base used to
    /// relativize scanned paths is identical to the one the scan starts from.
    /// If these drift, `diff_paths` returns an effectively absolute path and
    /// every `url_path` embeds the whole filesystem path.
    #[test]
    fn test_repo_canonicalizes_its_root() {
        let tmp = tempfile::tempdir().unwrap();
        let canonical = tmp.path().canonicalize().unwrap();

        let repo = Repo::init(
            tmp.path().to_path_buf(),
            "static".to_string(),
            &["md".to_string()],
            &[],
            &[],
            "index.md".to_string(),
            &[],
            &[],
        );

        assert_eq!(
            repo.canonical_root, canonical,
            "Repo must store the canonical root, not the path as supplied"
        );
    }

    /// Canonicalization must not lose a root that does not exist yet; falling
    /// back to the given path keeps the two sides consistent.
    #[test]
    fn test_canonicalize_root_falls_back_when_missing() {
        let missing = PathBuf::from("/definitely/does/not/exist/anywhere");
        assert_eq!(LocalVault::new(missing.clone()).root(), missing);
        assert_eq!(test_repo(&missing).canonical_root, missing);
    }

    #[test]
    fn test_should_ignore_hidden_file() {
        let path = Path::new(".hidden");
        assert!(should_ignore(path, &[], &[], &[]));
    }

    #[test]
    fn test_should_ignore_hidden_dir() {
        let path = Path::new(".git");
        assert!(should_ignore(path, &[], &[], &[]));
    }

    #[test]
    fn test_should_ignore_normal_file() {
        let path = Path::new("readme.md");
        assert!(!should_ignore(path, &[], &[], &[]));
    }

    #[test]
    fn test_should_ignore_glob_pattern() {
        let path = Path::new("test.log");
        let globs = vec!["*.log".to_string()];
        assert!(should_ignore(path, &[], &globs, &[]));
    }

    #[test]
    fn test_should_ignore_glob_no_match() {
        let path = Path::new("test.md");
        let globs = vec!["*.log".to_string()];
        assert!(!should_ignore(path, &[], &globs, &[]));
    }

    // ---- the leading-dot exemption ------------------------------------------

    /// The whole point: a hidden directory the user named is not tooling state.
    #[test]
    fn test_should_ignore_exempts_an_explicitly_named_hidden_dir() {
        let exempt = vec![PathBuf::from("/repo/.scratch")];
        assert!(!should_ignore(
            Path::new("/repo/.scratch"),
            &[],
            &[],
            &exempt
        ));
    }

    /// Naming `.scratch` says nothing about a dot directory *inside* it, which
    /// stays tooling state and stays skipped.
    #[test]
    fn test_should_ignore_still_skips_a_dot_dir_nested_in_an_exempt_one() {
        let exempt = vec![PathBuf::from("/repo/.scratch")];
        assert!(should_ignore(
            Path::new("/repo/.scratch/.git"),
            &[],
            &[],
            &exempt
        ));
    }

    /// Exemption is by exact path, not by name: a same-named hidden directory
    /// elsewhere in the repository was never on the target's chain.
    #[test]
    fn test_should_ignore_exemption_does_not_match_a_same_named_dir_elsewhere() {
        let exempt = vec![PathBuf::from("/repo/.scratch")];
        assert!(should_ignore(
            Path::new("/repo/docs/.scratch"),
            &[],
            &[],
            &exempt
        ));
    }

    /// The exemption waives *only* the leading-dot rule; the other two rules are
    /// untouched, so an exempt directory still loses to an ignore glob.
    #[test]
    fn test_should_ignore_exemption_does_not_override_ignore_globs() {
        let exempt = vec![PathBuf::from("/repo/.scratch")];
        let globs = vec!["**/.scratch".to_string()];
        assert!(should_ignore(
            Path::new("/repo/.scratch"),
            &[],
            &globs,
            &exempt
        ));
    }

    #[test]
    fn test_build_markdown_url_path_simple() {
        let root = Path::new("/root");
        let path = Path::new("/root/readme.md");
        assert_eq!(build_markdown_url_path(path, root, "index.md"), "/readme/");
    }

    #[test]
    fn test_build_markdown_url_path_nested() {
        let root = Path::new("/root");
        let path = Path::new("/root/docs/guide.md");
        assert_eq!(
            build_markdown_url_path(path, root, "index.md"),
            "/docs/guide/"
        );
    }

    #[test]
    fn test_build_markdown_url_path_index() {
        let root = Path::new("/root");
        let path = Path::new("/root/docs/index.md");
        assert_eq!(build_markdown_url_path(path, root, "index.md"), "/docs/");
    }

    #[test]
    fn test_build_markdown_url_path_root_index() {
        let root = Path::new("/root");
        let path = Path::new("/root/index.md");
        assert_eq!(build_markdown_url_path(path, root, "index.md"), "/");
    }

    #[test]
    fn test_build_static_url_path_in_static() {
        let root = Path::new("/root");
        let path = Path::new("/root/static/image.png");
        assert_eq!(build_static_url_path(path, root, "static"), "/image.png");
    }

    #[test]
    fn test_build_static_url_path_not_in_static() {
        let root = Path::new("/root");
        let path = Path::new("/root/assets/image.png");
        assert_eq!(
            build_static_url_path(path, root, "static"),
            "/assets/image.png"
        );
    }

    // Regression (Bug #8): the static folder name appearing as a substring of a
    // path component must NOT be stripped. Only a leading `{static_folder}/`
    // component is removed. Previously `.replacen` corrupted these paths.
    #[test]
    fn test_build_static_url_path_preserves_static_substring() {
        let root = Path::new("/root");

        // "static" appears inside a directory name, not as a leading component.
        let path = Path::new("/root/notes/static-analysis/img.png");
        assert_eq!(
            build_static_url_path(path, root, "static"),
            "/notes/static-analysis/img.png"
        );

        // "static" appears inside a nested (non-leading) directory name.
        let nested = Path::new("/root/my-static/image.png");
        assert_eq!(
            build_static_url_path(nested, root, "static"),
            "/my-static/image.png"
        );

        // Correctly-prefixed paths still have the leading component stripped.
        let prefixed = Path::new("/root/static/static-report.png");
        assert_eq!(
            build_static_url_path(prefixed, root, "static"),
            "/static-report.png"
        );
    }

    // Regression (Bug #9): a file whose name merely ends with the index file
    // name (e.g. "myindex.md" for index_file "index.md") must NOT be treated as
    // an index file. Previously `ends_with` truncated it to "/docs/my/".
    #[test]
    fn test_build_markdown_url_path_myindex_not_treated_as_index() {
        let root = Path::new("/root");

        let path = Path::new("/root/docs/myindex.md");
        assert_eq!(
            build_markdown_url_path(path, root, "index.md"),
            "/docs/myindex/"
        );

        // A genuine index file is still collapsed to a trailing slash.
        let real_index = Path::new("/root/docs/index.md");
        assert_eq!(
            build_markdown_url_path(real_index, root, "index.md"),
            "/docs/"
        );
    }

    // Regression (Bug #10): file_details_from_path must not fail (which would drop
    // the file from listings/search) when birth time is unavailable. It falls back
    // to the modified time. On platforms that do support btime this still succeeds.
    #[test]
    fn test_created_falls_back_to_modified() {
        let dir = tempfile::tempdir().expect("create temp dir");
        let file_path = dir.path().join("note.md");
        std::fs::write(&file_path, b"# hello").expect("write temp file");

        let (size, created, modified) =
            file_details_from_path(&file_path).expect("file details must not fail");

        assert_eq!(size, 7);
        // Both timestamps must be populated; created is either the real btime or,
        // when unavailable, the modified time (never causing an error/drop).
        assert!(modified > 0, "modified time should be populated");
        assert!(
            created > 0,
            "created time should be populated (btime or fallback)"
        );
    }

    #[test]
    fn test_is_markdown_extension_true() {
        let extensions = vec!["md".to_string(), "markdown".to_string()];
        assert!(is_markdown_extension("md", &extensions));
        assert!(is_markdown_extension("markdown", &extensions));
    }

    #[test]
    fn test_is_markdown_extension_false() {
        let extensions = vec!["md".to_string()];
        assert!(!is_markdown_extension("txt", &extensions));
        assert!(!is_markdown_extension("html", &extensions));
    }

    #[test]
    fn test_parse_tag_values_basic() {
        let tags: Vec<String> = parse_tag_values("rust, programming, web dev").collect();
        assert_eq!(tags, vec!["rust", "programming", "web dev"]);
    }

    #[test]
    fn test_parse_tag_values_single() {
        let tags: Vec<String> = parse_tag_values("rust").collect();
        assert_eq!(tags, vec!["rust"]);
    }

    #[test]
    fn test_parse_tag_values_whitespace() {
        let tags: Vec<String> = parse_tag_values("  rust  ,  python  ").collect();
        assert_eq!(tags, vec!["rust", "python"]);
    }

    #[test]
    fn test_parse_tag_values_empty() {
        let tags: Vec<String> = parse_tag_values("").collect();
        assert!(tags.is_empty());
    }

    #[test]
    fn test_parse_tag_values_empty_between() {
        let tags: Vec<String> = parse_tag_values("rust,,python").collect();
        assert_eq!(tags, vec!["rust", "python"]);
    }

    #[test]
    fn test_get_page_title_from_frontmatter() {
        let mut frontmatter = crate::markdown::SimpleMetadata::new();
        frontmatter.insert(
            "title".to_string(),
            serde_json::Value::String("My Page Title".to_string()),
        );
        let path = Path::new("/docs/readme.md");
        assert_eq!(get_page_title(&frontmatter, path), "My Page Title");
    }

    #[test]
    fn test_get_page_title_from_filename() {
        let frontmatter = crate::markdown::SimpleMetadata::new();
        let path = Path::new("/docs/rust-guide.md");
        assert_eq!(get_page_title(&frontmatter, path), "rust-guide");
    }

    #[test]
    fn test_get_page_title_fallback() {
        let frontmatter = crate::markdown::SimpleMetadata::new();
        let path = Path::new("/");
        assert_eq!(get_page_title(&frontmatter, path), "Untitled");
    }

    #[test]
    fn test_extract_tag_values_from_array() {
        let val = serde_json::json!(["rust", "python"]);
        let tags = extract_tag_values(&val);
        assert_eq!(tags, vec!["rust", "python"]);
    }

    #[test]
    fn test_extract_tag_values_from_comma_string() {
        let val = serde_json::json!("rust, python");
        let tags = extract_tag_values(&val);
        assert_eq!(tags, vec!["rust", "python"]);
    }

    #[test]
    fn test_extract_tag_values_from_single_string() {
        let val = serde_json::json!("rust");
        let tags = extract_tag_values(&val);
        assert_eq!(tags, vec!["rust"]);
    }

    #[test]
    fn test_extract_tag_values_from_number() {
        let val = serde_json::json!(42);
        let tags = extract_tag_values(&val);
        assert!(tags.is_empty());
    }

    // ==================== Root containment ====================

    /// Builds a `Repo` rooted at `root` with the defaults these tests need.
    fn test_repo(root: &Path) -> Repo {
        Repo::init(
            root.to_path_buf(),
            "static".to_string(),
            &["md".to_string()],
            &[],
            &[],
            "index.md".to_string(),
            &[],
            &[],
        )
    }

    /// The bug this exemption exists for: `mbr -s .scratch` roots at the
    /// enclosing repository (correctly), and the scan then skipped the one
    /// directory the user asked for — `site.json` and the directory listing for
    /// `/.scratch/` both came back empty while `/.scratch/alpha/` served 200,
    /// because `path_resolver` hits the disk and only the *index* was blind.
    #[test]
    fn test_scan_indexes_an_explicitly_named_hidden_dir() {
        let root = tempfile::tempdir().expect("temp dir");
        let scratch = root.path().join(".scratch");
        std::fs::create_dir(&scratch).expect("create .scratch");
        std::fs::write(scratch.join("alpha.md"), "# Alpha").expect("write alpha");
        std::fs::write(scratch.join("beta.md"), "# Beta").expect("write beta");
        // Nested, and therefore never named: still tooling state, still skipped.
        let nested = scratch.join(".obsidian");
        std::fs::create_dir(&nested).expect("create nested dot dir");
        std::fs::write(nested.join("workspace.md"), "# Tooling").expect("write nested");
        // A sibling hidden directory the user did not name.
        let other = root.path().join(".private");
        std::fs::create_dir(&other).expect("create .private");
        std::fs::write(other.join("secret.md"), "# Secret").expect("write secret");

        let repo = test_repo(root.path()).with_explicit_hidden_dirs(&[PathBuf::from(".scratch")]);
        repo.scan_all().expect("scan must succeed");

        let markdown = repo.markdown_files.pin();
        let urls: Vec<&str> = markdown.iter().map(|(_, i)| i.url_path.as_str()).collect();

        assert!(
            urls.contains(&"/.scratch/alpha/") && urls.contains(&"/.scratch/beta/"),
            "the named hidden directory must be indexed, got {urls:?}"
        );
        assert!(
            !urls.iter().any(|url| url.contains("workspace")),
            "a dot directory nested inside the named one must stay ignored, got {urls:?}"
        );
        assert!(
            !urls.iter().any(|url| url.contains("secret")),
            "an unnamed sibling hidden directory must stay ignored, got {urls:?}"
        );
    }

    /// Without the exemption the same tree indexes nothing — this is the
    /// pre-fix behaviour, kept so the test above cannot pass vacuously.
    #[test]
    fn test_scan_skips_a_hidden_dir_that_was_not_named() {
        let root = tempfile::tempdir().expect("temp dir");
        let scratch = root.path().join(".scratch");
        std::fs::create_dir(&scratch).expect("create .scratch");
        std::fs::write(scratch.join("alpha.md"), "# Alpha").expect("write alpha");

        let repo = test_repo(root.path());
        repo.scan_all().expect("scan must succeed");

        assert!(
            repo.markdown_files.pin().is_empty(),
            "a hidden directory nobody named must not be indexed"
        );
    }

    #[test]
    fn test_repo_relative_within_root_accepts_contained_paths() {
        let root = Path::new("/repo");

        assert_eq!(
            repo_relative_within_root(root, &root.join("docs").join("guide.md")),
            Some(PathBuf::from("docs").join("guide.md"))
        );
        // The root itself relativizes to the empty path, which is contained.
        assert_eq!(repo_relative_within_root(root, root), Some(PathBuf::new()));
    }

    #[test]
    fn test_repo_relative_within_root_rejects_escapes() {
        let root = Path::new("/repo");

        // A sibling directory: relativizes to `../elsewhere/secret.md`.
        assert_eq!(
            repo_relative_within_root(root, Path::new("/elsewhere/secret.md")),
            None
        );
        // An unresolved upward traversal.
        assert_eq!(
            repo_relative_within_root(root, &root.join("..").join("secret.md")),
            None
        );
        // Mixed absolute/relative inputs: `diff_paths` hands back something
        // absolute, which must not be mistaken for a repo-relative path.
        assert_eq!(
            repo_relative_within_root(Path::new("repo"), Path::new("/repo/a.md")),
            None
        );
    }

    /// A visible temp dir: `tempfile`'s default `.tmpXXXX` name is hidden, and
    /// a symlink target under a hidden directory is refused as a mount.
    #[cfg(unix)]
    fn visible_tempdir() -> tempfile::TempDir {
        tempfile::Builder::new()
            .prefix("mbr-repo-test-")
            .tempdir()
            .expect("temp dir")
    }

    /// A directory symlinked out of the repository is a read-only **mount**:
    /// indexed at the link's location, with in-root URLs and raw paths.
    ///
    /// The escape this used to guard against still cannot happen. The walk was
    /// once re-rooted at the symlink target, so every file underneath
    /// relativized to `../…`, `path_to_url` preserved that, and the static
    /// builder joined it onto `--output` — writing pages outside the requested
    /// output directory. A mount's paths are the link's, so nothing climbs.
    /// (Until mounts, the link was skipped outright.)
    #[cfg(unix)]
    #[test]
    fn test_scan_indexes_a_directory_symlinked_outside_root_as_a_mount() {
        let outside = visible_tempdir();
        let outside_docs = outside.path().join("docs");
        std::fs::create_dir_all(outside_docs.join(".private")).expect("create outside dir");
        std::fs::write(outside_docs.join("secret.md"), "# Linked").expect("write note");
        std::fs::write(outside_docs.join("secret.png"), b"not really a png").expect("write asset");
        std::fs::write(outside_docs.join(".private/plan.md"), "# Hidden").expect("write hidden");
        std::fs::write(outside_docs.join(".env"), "SECRET=1").expect("write dotfile");

        let root = tempfile::tempdir().expect("temp dir");
        std::fs::write(root.path().join("index.md"), "# Home").expect("write index");
        std::os::unix::fs::symlink(&outside_docs, root.path().join("work")).expect("symlink");

        let repo = test_repo(root.path());
        repo.scan_all().expect("scan must succeed");

        let markdown = repo.markdown_files.pin();
        for (_, info) in markdown.iter() {
            assert!(
                !info.url_path.contains(".."),
                "markdown url_path escaped the root: {}",
                info.url_path
            );
            assert!(
                !info
                    .raw_path
                    .components()
                    .any(|c| matches!(c, Component::ParentDir)),
                "markdown raw_path escaped the root: {}",
                info.raw_path.display()
            );
        }
        let linked = markdown
            .iter()
            .find(|(_, i)| i.url_path == "/work/secret/")
            .map(|(_, i)| i.raw_path.clone());
        assert_eq!(linked, Some(PathBuf::from("work/secret.md")));
        assert!(
            markdown.iter().all(|(_, i)| !i.url_path.contains("plan")),
            "hidden directories inside a mount are not indexed"
        );
        assert!(markdown.iter().any(|(_, i)| i.url_path == "/"));

        let other = repo.other_files.pin();
        let urls: Vec<&str> = other.iter().map(|(_, i)| i.url_path.as_str()).collect();
        assert_eq!(urls, vec!["/work/secret.png"], "{urls:?}");
        assert_eq!(repo.mounts().len(), 1);
        assert!(
            repo.vault()
                .is_read_only(&VaultPath::new("work/secret.md").unwrap())
        );
    }

    /// A link to a refused target (here: a hidden directory) still
    /// contributes nothing.
    #[cfg(unix)]
    #[test]
    fn test_scan_skips_a_directory_symlinked_to_a_refused_target() {
        let outside = visible_tempdir();
        let hidden = outside.path().join(".config/app");
        std::fs::create_dir_all(&hidden).expect("create");
        std::fs::write(hidden.join("secret.md"), "# Secret").expect("write");
        let root = tempfile::tempdir().expect("temp dir");
        std::fs::write(root.path().join("index.md"), "# Home").expect("write index");
        std::os::unix::fs::symlink(&hidden, root.path().join("work")).expect("symlink");
        std::os::unix::fs::symlink(root.path(), root.path().join("self")).expect("symlink");

        let repo = test_repo(root.path());
        repo.scan_all().expect("scan must succeed");
        let urls: Vec<String> = repo
            .markdown_files
            .pin()
            .iter()
            .map(|(_, i)| i.url_path.clone())
            .collect();
        assert_eq!(urls, vec!["/".to_string()]);
        assert!(repo.mounts().is_empty());
    }

    // ==================== External static overlay ====================

    /// Builds `<tmp>/project/{content,static}` and a `Repo` rooted at `content`
    /// with `static_folder = "../static"` — the `repo/content` + `repo/static`
    /// layout. Returns the temp dir (kept alive), the project dir and the repo.
    fn peer_overlay_repo() -> (tempfile::TempDir, PathBuf, Repo) {
        let tmp = tempfile::tempdir().expect("temp dir");
        let project = tmp.path().to_path_buf();
        let content = project.join("content");
        std::fs::create_dir_all(content.join(".mbr")).expect("create content");
        std::fs::write(content.join("README.md"), "# Home").expect("write readme");
        std::fs::create_dir_all(project.join("static/videos")).expect("create static");
        std::fs::write(project.join("static/pic.png"), b"PNG bytes").expect("write pic");
        std::fs::write(project.join("static/videos/demo.mp4"), b"MP4 bytes").expect("write video");

        let repo = Repo::init(
            content,
            "../static".to_string(),
            &["md".to_string()],
            &[],
            &[],
            "index.md".to_string(),
            &[],
            &[],
        );
        (tmp, project, repo)
    }

    /// The user's layout: `static_folder = "../static"` with
    /// `static/videos -> <elsewhere>`. The link is a mount of the *overlay*
    /// vault, so its files are indexed at overlay URLs (`/videos/clip.mp4`),
    /// hidden files in it are not, and a watcher event for one keeps its URL.
    #[cfg(unix)]
    #[test]
    fn test_scan_indexes_a_mount_inside_the_external_overlay() {
        let (_tmp, project, repo) = peer_overlay_repo();
        let movies = visible_tempdir();
        std::fs::create_dir(movies.path().join(".secret")).unwrap();
        std::fs::write(movies.path().join("clip.mp4"), b"clip").unwrap();
        std::fs::write(movies.path().join(".dotfile"), b"x").unwrap();
        std::fs::write(movies.path().join(".secret/key"), b"k").unwrap();
        std::fs::write(movies.path().join("notes.md"), b"# skipped").unwrap();
        std::os::unix::fs::symlink(movies.path(), project.join("static/films")).unwrap();

        repo.scan_all().expect("scan");
        repo.scan_static_folder().expect("static scan");

        let urls = |repo: &Repo| {
            let mut urls: Vec<String> = repo
                .other_files
                .pin()
                .iter()
                .map(|(_, i)| i.url_path.clone())
                .collect();
            urls.sort();
            urls
        };
        assert_eq!(
            urls(&repo),
            vec!["/films/clip.mp4", "/pic.png", "/videos/demo.mp4"]
        );
        assert!(
            repo.markdown_files
                .pin()
                .iter()
                .all(|(_, i)| !i.url_path.contains("notes")),
            "markdown in an external overlay (or a mount in one) has no page URL"
        );
        let mounts = repo.mounts();
        assert_eq!(mounts.len(), 1);
        let (location_key, mount) = &mounts[0];
        assert_eq!(
            *location_key,
            repo.canonical_static_root().unwrap().join("films")
        );
        assert_eq!(mount.target, movies.path().canonicalize().unwrap());

        // A file appearing in the target, reported at the link's location (as
        // the watcher translates it), is indexed at the overlay URL.
        std::fs::write(movies.path().join("new.mp4"), b"new").unwrap();
        repo.invalidate_file(
            &location_key.join("new.mp4"),
            &crate::change_event::ChangeEventType::Created,
        );
        repo.invalidate_file(
            &location_key.join(".secret").join("later"),
            &crate::change_event::ChangeEventType::Created,
        );
        assert_eq!(
            urls(&repo),
            vec![
                "/films/clip.mp4",
                "/films/new.mp4",
                "/pic.png",
                "/videos/demo.mp4"
            ]
        );
    }

    /// The regression: a peer static folder serves over HTTP but used to index
    /// nothing, so its assets were invisible to `media.json`, the media browser,
    /// the editor's media picker, search, and the media-metadata pass.
    #[test]
    fn test_scan_indexes_assets_in_external_static_overlay() {
        let (_tmp, _project, repo) = peer_overlay_repo();

        repo.scan_all().expect("scan must succeed");
        repo.scan_static_folder().expect("static scan must succeed");

        let other = repo.other_files.pin();
        let mut urls: Vec<_> = other.iter().map(|(_, i)| i.url_path.clone()).collect();
        urls.sort();
        assert_eq!(
            urls,
            vec!["/pic.png".to_string(), "/videos/demo.mp4".to_string()],
            "a peer static folder's assets must be indexed, with the overlay prefix stripped"
        );
    }

    /// Markdown inside an external overlay has no representable URL — it would
    /// relativize to `../static/…`, which `path_to_url` preserves and the static
    /// builder joins onto `--output`. It is skipped rather than indexed with a
    /// broken URL, so no `..` can reach `site.json`.
    #[test]
    fn test_scan_skips_markdown_in_external_static_overlay() {
        let (_tmp, project, repo) = peer_overlay_repo();
        std::fs::write(project.join("static/stray.md"), "# Stray").expect("write stray");

        repo.scan_all().expect("scan must succeed");
        repo.scan_static_folder().expect("static scan must succeed");

        let markdown = repo.markdown_files.pin();
        assert!(
            markdown.iter().all(|(_, i)| !i.url_path.contains("stray")),
            "markdown in an external static overlay must not be indexed"
        );
        for (_, info) in markdown.iter() {
            assert!(
                !info.url_path.contains(".."),
                "markdown url_path escaped the root: {}",
                info.url_path
            );
            assert!(
                !info
                    .raw_path
                    .components()
                    .any(|c| matches!(c, Component::ParentDir)),
                "markdown raw_path escaped the root: {}",
                info.raw_path.display()
            );
        }
        assert!(
            markdown.iter().any(|(_, i)| i.url_path == "/README/"),
            "the in-root markdown must still be indexed"
        );

        // The assets alongside it are still indexed, with clean URLs.
        let other = repo.other_files.pin();
        for (_, info) in other.iter() {
            assert!(
                !info.url_path.contains(".."),
                "static url_path escaped: {}",
                info.url_path
            );
        }
    }

    /// The overlay exception must stay an exception. A directory symlinked out
    /// of the root is still refused even when an external overlay is configured,
    /// so the "build writes pages outside --output" fix cannot regress through
    /// the new second scan root.
    #[cfg(unix)]
    #[test]
    fn test_external_overlay_does_not_admit_other_escaping_directories() {
        let (_tmp, project, repo) = peer_overlay_repo();

        let elsewhere = tempfile::tempdir().expect("temp dir");
        std::fs::write(elsewhere.path().join("secret.md"), "# Secret").expect("write secret");
        std::fs::write(elsewhere.path().join("secret.png"), b"secret").expect("write asset");
        // One escape from inside the root, one from inside the overlay itself.
        std::os::unix::fs::symlink(elsewhere.path(), project.join("content/work"))
            .expect("symlink into root");
        std::os::unix::fs::symlink(elsewhere.path(), project.join("static/leak"))
            .expect("symlink into overlay");

        repo.scan_all().expect("scan must succeed");
        repo.scan_static_folder().expect("static scan must succeed");

        let markdown = repo.markdown_files.pin();
        assert!(
            markdown.iter().all(|(_, i)| !i.url_path.contains("secret")),
            "a directory symlinked out of the repo must contribute no markdown"
        );
        let other = repo.other_files.pin();
        assert!(
            other.iter().all(|(_, i)| !i.url_path.contains("secret")),
            "a directory symlinked out of the repo (or out of the overlay) must contribute no assets"
        );
        for (_, info) in other.iter() {
            assert!(
                !info.url_path.contains(".."),
                "static url_path escaped: {}",
                info.url_path
            );
        }
    }

    /// A `static_folder` the config policy would refuse must not become a scan
    /// root either — the scanner and the validator share one decision.
    #[test]
    fn test_refused_static_folder_is_not_scannable() {
        let tmp = tempfile::tempdir().expect("temp dir");
        let content = tmp.path().join("project/content");
        std::fs::create_dir_all(&content).expect("create content");

        // Two shapes the policy refuses for different reasons: an ancestor of
        // the markdown root, and a target past the ascent budget. Both must
        // leave the scanner with nothing extra to walk.
        for value in ["../..", "../../../elsewhere"] {
            let repo = Repo::init(
                content.clone(),
                value.to_string(),
                &["md".to_string()],
                &[],
                &[],
                "index.md".to_string(),
                &[],
                &[],
            );

            assert_eq!(
                repo.canonical_static_root(),
                None,
                "a static_folder the validator refuses must not be a scan root: {value:?}"
            );
        }
    }

    /// Builds `<tmp>/project/{src/routes,static}` and a `Repo` rooted at
    /// `src/routes` — the SvelteKit shape, where the overlay is two levels up.
    fn two_deep_overlay_repo(static_folder: &str) -> (tempfile::TempDir, PathBuf, Repo) {
        let tmp = tempfile::tempdir().expect("temp dir");
        let project = tmp.path().join("project");
        let routes = project.join("src/routes");
        std::fs::create_dir_all(routes.join(".mbr")).expect("create routes");
        std::fs::write(routes.join("README.md"), "# Home").expect("write readme");
        std::fs::create_dir_all(project.join("static/videos")).expect("create static");
        std::fs::write(project.join("static/pic.png"), b"PNG bytes").expect("write pic");
        std::fs::write(project.join("static/videos/demo.mp4"), b"MP4 bytes").expect("write video");

        let repo = Repo::init(
            routes,
            static_folder.to_string(),
            &["md".to_string()],
            &[],
            &[],
            "index.md".to_string(),
            &[],
            &[],
        );
        (tmp, project, repo)
    }

    /// A two-level overlay must index exactly like a peer one: the whole
    /// overlay prefix stripped, no `..` left in any URL. This is what catches a
    /// `strip_prefix` regression on a two-component prefix, which a one-level
    /// fixture cannot distinguish from a working implementation.
    #[test]
    fn test_two_deep_overlay_is_scannable_with_clean_urls() {
        let (_tmp, project, repo) = two_deep_overlay_repo("../../static");

        assert_eq!(
            repo.canonical_static_root().map(Path::to_path_buf),
            Some(project.join("static").canonicalize().expect("canonicalize")),
            "a two-level overlay the validator accepts must be a scan root"
        );

        repo.scan_all().expect("scan must succeed");
        repo.scan_static_folder().expect("static scan must succeed");

        let other = repo.other_files.pin();
        let mut urls: Vec<_> = other.iter().map(|(_, i)| i.url_path.clone()).collect();
        urls.sort();
        assert_eq!(
            urls,
            vec!["/pic.png".to_string(), "/videos/demo.mp4".to_string()],
            "a two-level static folder's assets must be indexed with the overlay prefix stripped"
        );
    }

    /// `../../static` and `./../../static` name the same directory, and the
    /// config policy accepts both — so indexing must not depend on which
    /// spelling was written. Stripping the raw configured string does depend on
    /// it, because `Path::strip_prefix` compares components and keeps a leading
    /// `.`, which used to leave a `..` in the URL.
    #[test]
    fn test_overlay_urls_survive_a_noncanonical_static_folder_spelling() {
        let (_tmp, _project, repo) = two_deep_overlay_repo("./../../static");

        repo.scan_all().expect("scan must succeed");
        repo.scan_static_folder().expect("static scan must succeed");

        let other = repo.other_files.pin();
        let mut urls: Vec<_> = other.iter().map(|(_, i)| i.url_path.clone()).collect();
        urls.sort();
        assert_eq!(
            urls,
            vec!["/pic.png".to_string(), "/videos/demo.mp4".to_string()],
            "URLs must come from the resolved overlay, not the configured spelling"
        );
    }

    // ==================== Non-local vaults ====================

    /// A repository in a `MemVault`: no `local_path`, so every read the scan
    /// makes has to go through the vault.
    fn mem_repo(vault: crate::vault::MemVault, tag_sources: &[TagSource]) -> Repo {
        Repo::init_with_vault(
            Arc::new(vault),
            "static".to_string(),
            &["md".to_string()],
            &["node_modules".to_string()],
            &["*.tmp".to_string()],
            "index.md".to_string(),
            tag_sources,
            &[],
        )
    }

    fn sample_mem_vault() -> crate::vault::MemVault {
        crate::vault::MemVault::new()
            .with_file("index.md", "---\ntitle: Home\ntags: [start]\n---\n# Home\n")
            .with_file("docs/guide.md", "# The Guide\n\nBody.\n")
            .with_file("docs/index.md", "---\ntitle: Docs\n---\n")
            .with_file("docs/notes.txt", "plain text body")
            .with_file("docs/scratch.tmp", "ignored by glob")
            .with_file(".hidden/secret.md", "# Secret")
            .with_file("node_modules/pkg/readme.md", "# Dependency")
            .with_file("static/img.png", "PNG bytes")
            .with_file("static/css/site.css", "body{}")
    }

    #[test]
    fn test_scan_through_a_mem_vault_indexes_everything_a_disk_scan_would() {
        let tags = [TagSource {
            field: "tags".to_string(),
            label: None,
            label_plural: None,
        }];
        let repo = mem_repo(sample_mem_vault(), &tags);
        let root = PathBuf::from(crate::vault::MemVault::DEFAULT_ROOT);
        assert_eq!(repo.canonical_root, root);

        repo.scan_all().expect("scan");
        let markdown_urls: Vec<String> = sorted(
            repo.markdown_files
                .pin()
                .iter()
                .map(|(_, info)| info.url_path.clone())
                .collect(),
        );
        assert_eq!(markdown_urls, vec!["/", "/docs/", "/docs/guide/"]);

        let pin = repo.markdown_files.pin();
        let home = pin
            .get(&root.join("index.md"))
            .expect("keyed under the vault root");
        assert_eq!(home.raw_path, PathBuf::from("index.md"));
        let fm = home
            .frontmatter
            .as_ref()
            .expect("frontmatter read via the vault");
        assert_eq!(fm.get("title").and_then(|v| v.as_str()), Some("Home"));
        let guide = pin
            .get(&root.join("docs").join("guide.md"))
            .expect("nested file");
        assert_eq!(
            guide
                .frontmatter
                .as_ref()
                .and_then(|fm| fm.get("title"))
                .and_then(|v| v.as_str()),
            Some("The Guide"),
            "the first-H1 fallback works off vault bytes too"
        );
        assert!(guide.modified > 0 && guide.created > 0);
        assert_eq!(repo.tag_index.get_pages("tags", "start").len(), 1);
        drop(pin);

        // The static folder is deferred to its own pass, as on disk.
        assert!(
            repo.other_files
                .pin()
                .iter()
                .all(|(_, info)| !info.url_path.starts_with("/img")),
            "static assets wait for scan_static_folder"
        );
        repo.scan_static_folder().expect("static scan");
        let other_urls: Vec<String> = sorted(
            repo.other_files
                .pin()
                .iter()
                .map(|(_, info)| info.url_path.clone())
                .collect(),
        );
        assert_eq!(
            other_urls,
            vec!["/css/site.css", "/docs/notes.txt", "/img.png"]
        );
    }

    /// Basic metadata stays deferred, as on disk, and the deferred pass reads
    /// it through the vault — there is no file to `stat`.
    #[test]
    fn test_mem_vault_basic_metadata_is_populated_through_the_vault() {
        let repo = mem_repo(sample_mem_vault(), &[]);
        repo.scan_all().expect("scan");
        repo.scan_static_folder().expect("static scan");
        let key = repo.canonical_root.join("static").join("img.png");
        let size = |repo: &Repo| {
            repo.other_files
                .pin()
                .get(&key)
                .and_then(|info| info.metadata.file_size_bytes)
        };
        assert_eq!(size(&repo), None, "deferred, exactly as before the vault");
        repo.populate_basic_metadata();
        assert_eq!(size(&repo), Some(9));
    }

    #[test]
    fn test_mem_vault_text_extraction_reads_through_the_vault() {
        let repo = mem_repo(sample_mem_vault(), &[]);
        repo.scan_all().expect("scan");
        repo.ensure_text_extracted();
        let key = repo.canonical_root.join("docs").join("notes.txt");
        let text = repo
            .other_files
            .pin()
            .get(&key)
            .and_then(|info| info.extracted_text.clone());
        assert_eq!(text.as_deref(), Some("plain text body"));
    }

    /// Watcher-driven invalidation reads the changed file through the vault.
    #[test]
    fn test_mem_vault_invalidate_file_reads_through_the_vault() {
        use crate::change_event::ChangeEventType;
        let vault = Arc::new(sample_mem_vault());
        let repo = Repo::init_with_vault(
            vault.clone(),
            "static".to_string(),
            &["md".to_string()],
            &[],
            &[],
            "index.md".to_string(),
            &[],
            &[],
        );
        repo.scan_all().expect("scan");
        let key = repo.canonical_root.join("docs").join("new.md");
        let title = |repo: &Repo| {
            repo.markdown_files.pin().get(&key).and_then(|info| {
                info.frontmatter
                    .as_ref()?
                    .get("title")?
                    .as_str()
                    .map(str::to_string)
            })
        };

        vault.insert_file("docs/new.md", "---\ntitle: Fresh\n---\n");
        repo.invalidate_file(&key, &ChangeEventType::Created);
        assert_eq!(title(&repo).as_deref(), Some("Fresh"));

        vault.insert_file("docs/new.md", "---\ntitle: Edited\n---\n");
        repo.invalidate_file(&key, &ChangeEventType::Modified);
        assert_eq!(title(&repo).as_deref(), Some("Edited"));

        repo.invalidate_file(&key, &ChangeEventType::Deleted);
        assert!(repo.markdown_files.pin().get(&key).is_none());

        // A key outside the vault cannot be read through it and is not indexed.
        let outside = PathBuf::from("/somewhere/else/x.md");
        repo.invalidate_file(&outside, &ChangeEventType::Created);
        assert!(repo.markdown_files.pin().get(&outside).is_none());
    }

    /// A listed-but-not-downloaded note is still indexed (its URL and times
    /// come from the listing); only its frontmatter is missing until it is
    /// readable.
    #[test]
    fn test_not_downloaded_markdown_is_indexed_without_frontmatter() {
        let vault = sample_mem_vault();
        vault.set_availability("docs/guide.md", crate::vault::Availability::NotDownloaded);
        let repo = mem_repo(vault, &[]);
        repo.scan_all().expect("scan");
        let key = repo.canonical_root.join("docs").join("guide.md");
        let pin = repo.markdown_files.pin();
        let info = pin.get(&key).expect("indexed from the listing");
        assert_eq!(info.url_path, "/docs/guide/");
        assert!(info.frontmatter.is_none());
    }

    /// The server's non-recursive directory scan queues subfolders with their
    /// vault paths.
    #[test]
    fn test_scan_folder_queues_subfolders_with_vault_paths() {
        let repo = mem_repo(sample_mem_vault(), &[]);
        repo.scan_folder(&PathBuf::from("docs")).expect("scan");
        assert!(repo.queued_folders.is_empty(), "docs has no subfolders");
        repo.scan_folder(&PathBuf::from(".")).expect("scan root");
        let queued: Vec<String> = sorted(
            repo.queued_folders
                .pin()
                .iter()
                .map(|(_, queued)| queued.path.to_string())
                .collect(),
        );
        // `.hidden` and `node_modules` are ignored; `docs` is already scanned
        // but still listed as a child.
        assert_eq!(queued, vec!["docs", "static"]);
        assert!(
            repo.scan_folder(&PathBuf::from("../escape")).is_ok(),
            "a folder climbing out of the root is skipped, not an error"
        );
    }

    /// On disk, the scan itself leaves other files' basic metadata unset — the
    /// static build, which never runs `populate_basic_metadata`, publishes
    /// exactly that in `media.json` — and the deferred pass fills it in.
    #[test]
    fn test_local_scan_defers_static_file_details() {
        let dir = tempfile::tempdir().expect("temp dir");
        std::fs::create_dir(dir.path().join("static")).unwrap();
        std::fs::write(dir.path().join("static/a.png"), b"12345").unwrap();
        std::fs::write(dir.path().join("b.pdf"), b"123").unwrap();
        let repo = test_repo(dir.path());
        repo.scan_all().expect("scan");
        repo.scan_static_folder().expect("static scan");
        let sizes = |repo: &Repo| -> Vec<(String, Option<u64>)> {
            let pin = repo.other_files.pin();
            let mut v: Vec<_> = pin
                .iter()
                .map(|(_, i)| (i.url_path.clone(), i.metadata.file_size_bytes))
                .collect();
            v.sort();
            v
        };
        assert_eq!(
            sizes(&repo),
            vec![("/a.png".to_string(), None), ("/b.pdf".to_string(), None)]
        );
        repo.populate_basic_metadata();
        assert_eq!(
            sizes(&repo),
            vec![
                ("/a.png".to_string(), Some(5)),
                ("/b.pdf".to_string(), Some(3))
            ]
        );
    }

    // ==================== Media metadata flags ====================

    /// Claiming the run-once guard must not announce completion.
    ///
    /// `media_populated` is what `is_media_populated()`/`wait_for_media()` and
    /// the `media.json` handler read as "the metadata is there". While it
    /// doubled as the "already started" guard, a request arriving during the
    /// probing window skipped the wait and got entries with no duration or
    /// dimensions.
    #[test]
    fn test_in_flight_media_population_does_not_report_as_complete() {
        let dir = tempfile::tempdir().expect("temp dir");
        let repo = test_repo(dir.path());

        // Exactly the state a reader sees mid-population: guard claimed, work
        // not finished.
        repo.media_population_started.store(true, Ordering::SeqCst);

        assert!(
            !repo.is_media_populated(),
            "population in flight must not be reported as complete"
        );
    }

    #[test]
    fn test_media_populated_is_published_after_population_and_reset_by_clear() {
        let dir = tempfile::tempdir().expect("temp dir");
        std::fs::write(dir.path().join("index.md"), "# Home").expect("write index");
        std::fs::write(dir.path().join("paper.pdf"), b"not really a pdf").expect("write paper");

        let repo = test_repo(dir.path());
        repo.scan_all().expect("scan must succeed");

        assert!(!repo.is_media_populated());
        assert!(!repo.media_population_started.load(Ordering::SeqCst));

        repo.populate_media_metadata();
        assert!(repo.media_population_started.load(Ordering::SeqCst));
        assert!(
            repo.is_media_populated(),
            "flag must be published once population finished"
        );

        // Run-once guard: a repeat call is a no-op and must not unset "finished".
        repo.populate_media_metadata();
        assert!(repo.is_media_populated());

        // `clear()` must reset both flags, otherwise the full-rescan path would
        // skip population and leave every later `wait_for_media()` blocked.
        repo.clear();
        assert!(!repo.is_media_populated());
        assert!(!repo.media_population_started.load(Ordering::SeqCst));

        repo.populate_media_metadata();
        assert!(
            repo.is_media_populated(),
            "a rescan must be able to re-populate media metadata"
        );
    }

    // ==================== Scan/media waiters ====================

    /// `Notify::notify_waiters()` stores no permit, so a waiter created after the
    /// single completion notification has already fired must still return.
    #[tokio::test]
    async fn test_wait_for_scan_returns_immediately_when_already_complete() {
        let dir = tempfile::tempdir().expect("temp dir");
        let repo = test_repo(dir.path());
        repo.mark_scan_complete();

        tokio::time::timeout(std::time::Duration::from_secs(5), repo.wait_for_scan())
            .await
            .expect("a waiter started after completion must not block");
    }

    #[tokio::test]
    async fn test_wait_for_scan_is_woken_by_later_completion() {
        let dir = tempfile::tempdir().expect("temp dir");
        let repo = Arc::new(test_repo(dir.path()));

        let waiting = Arc::clone(&repo);
        let waiter = tokio::spawn(async move { waiting.wait_for_scan().await });
        // Let the waiter register its interest before completion fires.
        tokio::task::yield_now().await;

        repo.mark_scan_complete();

        tokio::time::timeout(std::time::Duration::from_secs(5), waiter)
            .await
            .expect("waiter must be woken by mark_scan_complete")
            .expect("waiter task panicked");
    }

    #[tokio::test]
    async fn test_wait_for_media_returns_immediately_when_already_populated() {
        let dir = tempfile::tempdir().expect("temp dir");
        let repo = test_repo(dir.path());
        repo.populate_media_metadata();
        repo.notify_media_populated();

        tokio::time::timeout(std::time::Duration::from_secs(5), repo.wait_for_media())
            .await
            .expect("a waiter started after population must not block");
    }

    /// A notification that arrives with the flag still unset is spurious: the
    /// waiter must go back to waiting rather than report populated metadata.
    #[tokio::test]
    async fn test_wait_for_media_ignores_spurious_notify_then_wakes() {
        let dir = tempfile::tempdir().expect("temp dir");
        let repo = Arc::new(test_repo(dir.path()));

        let waiting = Arc::clone(&repo);
        let mut waiter = tokio::spawn(async move { waiting.wait_for_media().await });
        tokio::task::yield_now().await;

        // Notification without population: must not release the waiter.
        repo.notify_media_populated();
        assert!(
            tokio::time::timeout(std::time::Duration::from_millis(100), &mut waiter)
                .await
                .is_err(),
            "a notification with the flag unset must not release the waiter"
        );

        repo.populate_media_metadata();
        repo.notify_media_populated();

        tokio::time::timeout(std::time::Duration::from_secs(5), waiter)
            .await
            .expect("waiter must be woken once media metadata is populated")
            .expect("waiter task panicked");
    }

    /// A rescan whose scan step fails must still release `wait_for_media()`.
    ///
    /// `full_rescan()` calls `clear()` first, which resets `media_populated`.
    /// When the scan error caused an early return, `notify_media_populated()`
    /// never ran and `/.mbr/media.json` blocked for the life of the process —
    /// one unreadable repository root wedged the endpoint permanently.
    #[tokio::test]
    async fn test_full_rescan_releases_media_waiters_even_when_scan_fails() {
        let dir = tempfile::tempdir().expect("temp dir");
        let repo = Arc::new(test_repo(dir.path()));

        // Populate once so the flag starts set, as it would on a live server.
        repo.populate_media_metadata();
        repo.notify_media_populated();
        assert!(repo.is_media_populated());

        // Delete the root out from under the repo so `scan_all()` errors. The
        // canonical root was resolved at construction, so it still points here.
        dir.close().expect("remove repo root");

        let rescanning = Arc::clone(&repo);
        let rescan = tokio::task::spawn_blocking(move || rescanning.full_rescan());

        tokio::time::timeout(std::time::Duration::from_secs(5), rescan)
            .await
            .expect("rescan must not hang")
            .expect("rescan task panicked");

        tokio::time::timeout(std::time::Duration::from_secs(5), repo.wait_for_media())
            .await
            .expect("a failed rescan must still release media waiters");
    }

    // ==================== Deterministic serialization ====================

    /// Two builds of an unchanged repository must serialize to identical bytes.
    ///
    /// `papaya::HashMap` seeds its `RandomState` per instance, so iteration
    /// order differs between processes: `site.json` churned in `git diff` and
    /// content-hash/ETag caches invalidated on no-op rebuilds. Two
    /// independently built maps stand in for two builds.
    #[test]
    fn test_markdown_files_serialize_deterministically_in_url_path_order() {
        let build_files = || {
            let files = MarkdownFiles(HashMap::new());
            let pin = files.pin();
            for i in 0..64 {
                pin.insert(
                    PathBuf::from(format!("/repo/notes/note{i:02}.md")),
                    MarkdownInfo {
                        raw_path: PathBuf::from(format!("notes/note{i:02}.md")),
                        url_path: format!("/notes/note{i:02}/"),
                        created: 1,
                        modified: 2,
                        frontmatter: None,
                        relationships: Vec::new(),
                    },
                );
            }
            drop(pin);
            files
        };

        let first = serde_json::to_string(&build_files()).expect("serialize");
        let second = serde_json::to_string(&build_files()).expect("serialize");
        assert_eq!(
            first, second,
            "serialized bytes must not depend on map iteration order"
        );

        assert_eq!(
            url_paths_of(&first),
            sorted(url_paths_of(&first)),
            "entries must be emitted in url_path order"
        );
    }

    #[test]
    fn test_other_files_serialize_deterministically_in_url_path_order() {
        let build_files = || {
            let files = OtherFiles(HashMap::new());
            let pin = files.pin();
            for i in 0..64 {
                let raw_path = PathBuf::from(format!("/repo/images/img{i:02}.png"));
                pin.insert(
                    raw_path.clone(),
                    OtherFileInfo {
                        url_path: format!("/images/img{i:02}.png"),
                        metadata: StaticFileMetadata::empty(&raw_path),
                        raw_path,
                        extracted_text: None,
                    },
                );
            }
            drop(pin);
            files
        };

        let first = serde_json::to_string(&build_files()).expect("serialize");
        let second = serde_json::to_string(&build_files()).expect("serialize");
        assert_eq!(
            first, second,
            "serialized bytes must not depend on map iteration order"
        );

        assert_eq!(
            url_paths_of(&first),
            sorted(url_paths_of(&first)),
            "entries must be emitted in url_path order"
        );
    }

    /// Extracts the `url_path` of every entry in a serialized file sequence.
    fn url_paths_of(json: &str) -> Vec<String> {
        serde_json::from_str::<Vec<serde_json::Value>>(json)
            .expect("serialized files must be a JSON array")
            .iter()
            .map(|entry| {
                entry["url_path"]
                    .as_str()
                    .expect("every entry has a url_path")
                    .to_string()
            })
            .collect()
    }

    fn sorted(mut values: Vec<String>) -> Vec<String> {
        values.sort();
        values
    }
}

#[cfg(test)]
mod proptests {
    use super::*;
    use proptest::prelude::*;

    // Strategy for valid file/directory names (no path separators or special chars)
    fn valid_name_strategy() -> impl Strategy<Value = String> {
        "[a-zA-Z0-9_-]{1,20}"
    }

    // Strategy for valid extensions
    fn extension_strategy() -> impl Strategy<Value = String> {
        "[a-z]{1,10}"
    }

    proptest! {
        /// should_ignore is deterministic - same input always gives same output
        #[test]
        fn prop_should_ignore_deterministic(
            name in valid_name_strategy(),
            ignore_dirs in proptest::collection::vec(valid_name_strategy(), 0..3),
            ignore_globs in proptest::collection::vec("[*][.][a-z]{1,5}", 0..3),
        ) {
            let path = Path::new(&name);
            let result1 = should_ignore(path, &ignore_dirs, &ignore_globs, &[]);
            let result2 = should_ignore(path, &ignore_dirs, &ignore_globs, &[]);
            prop_assert_eq!(result1, result2);
        }

        /// Hidden files (starting with .) are always ignored
        #[test]
        fn prop_hidden_files_always_ignored(name in "[.][a-zA-Z0-9]{1,15}") {
            let path = Path::new(&name);
            prop_assert!(should_ignore(path, &[], &[], &[]));
        }

        /// Non-hidden files without matching globs are not ignored
        #[test]
        fn prop_normal_files_not_ignored(name in "[a-zA-Z][a-zA-Z0-9]{0,15}") {
            let path = Path::new(&name);
            // No ignore patterns configured
            prop_assert!(!should_ignore(path, &[], &[], &[]));
        }

        /// is_markdown_extension is deterministic
        #[test]
        fn prop_is_markdown_extension_deterministic(
            ext in extension_strategy(),
            extensions in proptest::collection::vec(extension_strategy(), 1..5)
        ) {
            let result1 = is_markdown_extension(&ext, &extensions);
            let result2 = is_markdown_extension(&ext, &extensions);
            prop_assert_eq!(result1, result2);
        }

        /// Extension in list returns true
        #[test]
        fn prop_extension_in_list_returns_true(
            extensions in proptest::collection::vec(extension_strategy(), 1..5)
        ) {
            // Pick the first extension from the list
            if let Some(ext) = extensions.first() {
                prop_assert!(is_markdown_extension(ext, &extensions));
            }
        }

        /// build_markdown_url_path always returns path starting with /
        #[test]
        fn prop_markdown_url_starts_with_slash(
            subpath in proptest::collection::vec(valid_name_strategy(), 1..4),
            filename in valid_name_strategy(),
        ) {
            let root = PathBuf::from("/root");
            let mut full_path = root.clone();
            for component in &subpath {
                full_path.push(component);
            }
            full_path.push(format!("{}.md", filename));

            let url = build_markdown_url_path(&full_path, &root, "index.md");
            prop_assert!(url.starts_with('/'), "URL should start with /: {}", url);
            prop_assert!(!url.contains('\\'), "URL must not contain a backslash: {}", url);
        }

        /// build_markdown_url_path always returns path ending with /
        #[test]
        fn prop_markdown_url_ends_with_slash(
            subpath in proptest::collection::vec(valid_name_strategy(), 0..4),
            filename in valid_name_strategy(),
        ) {
            let root = PathBuf::from("/root");
            let mut full_path = root.clone();
            for component in &subpath {
                full_path.push(component);
            }
            full_path.push(format!("{}.md", filename));

            let url = build_markdown_url_path(&full_path, &root, "index.md");
            prop_assert!(url.ends_with('/'), "URL should end with /: {}", url);
            prop_assert!(!url.contains('\\'), "URL must not contain a backslash: {}", url);
        }

        /// build_static_url_path always returns path starting with /
        #[test]
        fn prop_static_url_starts_with_slash(
            subpath in proptest::collection::vec(valid_name_strategy(), 0..4),
            filename in valid_name_strategy(),
            ext in extension_strategy(),
        ) {
            let root = PathBuf::from("/root");
            let mut full_path = root.clone();
            for component in &subpath {
                full_path.push(component);
            }
            full_path.push(format!("{}.{}", filename, ext));

            let url = build_static_url_path(&full_path, &root, "static");
            prop_assert!(url.starts_with('/'), "URL should start with /: {}", url);
            prop_assert!(!url.contains('\\'), "URL must not contain a backslash: {}", url);
        }

        /// URL paths don't contain double slashes
        #[test]
        fn prop_no_double_slashes_in_markdown_url(
            subpath in proptest::collection::vec(valid_name_strategy(), 0..4),
            filename in valid_name_strategy(),
        ) {
            let root = PathBuf::from("/root");
            let mut full_path = root.clone();
            for component in &subpath {
                full_path.push(component);
            }
            full_path.push(format!("{}.md", filename));

            let url = build_markdown_url_path(&full_path, &root, "index.md");
            prop_assert!(!url.contains("//"), "URL should not contain //: {}", url);
            prop_assert!(!url.contains('\\'), "URL must not contain a backslash: {}", url);
        }
    }
}