siteforge 0.1.19

use std::collections::HashSet;
use std::fs::{self, File, OpenOptions};
use std::io::{BufWriter, Write};
use std::path::{Path, PathBuf};
use std::time::{Duration, Instant};

use futures_util::stream::{FuturesUnordered, StreamExt};
use globset::{Glob, GlobSet, GlobSetBuilder};
use regex::Regex;
use serde::{Deserialize, Serialize};
use tokio::sync::mpsc;
use url::Url;

use crate::archive::{
    archive_id_from_seed, new_manifest, read_manifest, stable_id, validate_archive_id,
    write_checksums_streamed, write_manifest, write_thesa_sidecar, ArchiveLayout, ArchiveManifest,
    ArchiveWriteBudget, CrawlStats,
};
use crate::config::Config;
use crate::errors::{Result, SiteforgeError};
use crate::fetch::{bytes_to_string, is_html, FetchResult, HttpFetcher};
use crate::ocr::{backend_from_config, likely_contains_code, OcrBackend};
use crate::parse::{normalize_link, parse_html, AssetRef, ImageDimensions, ParsedPage};
use crate::render::{likely_needs_js_render, render_js_dom, BrowserRenderOptions};
use crate::storage::{FetchedAssetRecord, FrontierItem, Storage, StoredAsset, StoredPage};

const CRAWL_INDEX_PAGE_BATCH: usize = 500;
const CRAWL_INDEX_ASSET_BATCH: usize = 500;
const PROGRESS_EMIT_INTERVAL: Duration = Duration::from_millis(250);
const README_ASSET_LIMIT: usize = 200;

#[derive(Debug, Clone)]
/// Options for starting a new archive run.
///
/// These are per-run controls. Defaults such as timeouts, size caps, output
/// profile, JS rendering, and include/exclude patterns live in [`Config`].
pub struct ArchiveRunOptions {
    /// Seed URLs to enqueue at depth zero.
    pub seeds: Vec<Url>,
    /// Follow discovered in-scope links when true; archive seeds only when false.
    pub full_site: bool,
    /// Restrict discovery to the seed host and path prefix when true.
    pub same_domain: bool,
    /// Maximum crawl depth for discovered links.
    pub max_depth: usize,
    /// Maximum number of page fetches to attempt.
    pub max_pages: usize,
    /// Concurrent page fetch count for this run.
    pub concurrency: usize,
    /// Politeness delay between requests to the same origin, in milliseconds.
    pub delay_ms: u64,
    /// Additional HTTP headers for authorized archives.
    pub headers: Vec<(String, String)>,
    /// Optional `Cookie` header value for authorized archives.
    pub cookie: Option<String>,
    /// Optional stable archive ID. If omitted, one is generated from the first seed.
    pub archive_id: Option<String>,
}

impl ArchiveRunOptions {
    /// Create seed-only archive options from explicit URLs and configuration defaults.
    ///
    /// The returned options archive the provided seeds without following discovered
    /// links. Use [`Self::full_site`] for a crawl that follows in-scope links.
    pub fn new(seeds: impl IntoIterator<Item = Url>, config: &Config) -> Self {
        Self {
            seeds: seeds.into_iter().collect(),
            full_site: false,
            same_domain: true,
            max_depth: 0,
            max_pages: config.max_pages,
            concurrency: config.default_concurrency.max(1),
            delay_ms: config.default_delay_ms,
            headers: Vec::new(),
            cookie: None,
            archive_id: None,
        }
    }

    /// Create options for archiving exactly one URL.
    pub fn single_page(seed: Url, config: &Config) -> Self {
        Self::new([seed], config).with_max_pages(1)
    }

    /// Parse a URL and create options for archiving exactly one URL.
    pub fn single_page_url(seed: impl AsRef<str>, config: &Config) -> Result<Self> {
        Ok(Self::single_page(Url::parse(seed.as_ref())?, config))
    }

    /// Create options for a same-domain crawl starting at one seed URL.
    pub fn full_site(seed: Url, config: &Config) -> Self {
        let mut options = Self::new([seed], config);
        options.full_site = true;
        options.max_depth = config.max_depth;
        options
    }

    /// Parse a URL and create options for a same-domain crawl.
    pub fn full_site_url(seed: impl AsRef<str>, config: &Config) -> Result<Self> {
        Ok(Self::full_site(Url::parse(seed.as_ref())?, config))
    }

    /// Set a stable archive ID for the run.
    pub fn with_archive_id(mut self, archive_id: impl Into<String>) -> Self {
        self.archive_id = Some(archive_id.into());
        self
    }

    /// Set whether discovered URLs are followed.
    pub fn with_full_site(mut self, full_site: bool) -> Self {
        self.full_site = full_site;
        self
    }

    /// Set whether discovery is restricted to the seed host and path prefix.
    pub fn with_same_domain(mut self, same_domain: bool) -> Self {
        self.same_domain = same_domain;
        self
    }

    /// Allow discovered URLs outside the seed domain/path scope.
    pub fn allowing_cross_domain(self) -> Self {
        self.with_same_domain(false)
    }

    /// Set maximum crawl depth.
    pub fn with_max_depth(mut self, max_depth: usize) -> Self {
        self.max_depth = max_depth;
        self
    }

    /// Set maximum page fetch attempts.
    pub fn with_max_pages(mut self, max_pages: usize) -> Self {
        self.max_pages = max_pages;
        self
    }

    /// Set concurrent page fetch count. Values below one are normalized to one.
    pub fn with_concurrency(mut self, concurrency: usize) -> Self {
        self.concurrency = concurrency.max(1);
        self
    }

    /// Set the politeness delay between requests to the same origin, in milliseconds.
    pub fn with_delay_ms(mut self, delay_ms: u64) -> Self {
        self.delay_ms = delay_ms;
        self
    }

    /// Add one HTTP header for authorized archives.
    pub fn with_header(mut self, name: impl Into<String>, value: impl Into<String>) -> Self {
        self.headers.push((name.into(), value.into()));
        self
    }

    /// Replace HTTP headers for authorized archives.
    pub fn with_headers<I, K, V>(mut self, headers: I) -> Self
    where
        I: IntoIterator<Item = (K, V)>,
        K: Into<String>,
        V: Into<String>,
    {
        self.headers = headers
            .into_iter()
            .map(|(name, value)| (name.into(), value.into()))
            .collect();
        self
    }

    /// Set the optional `Cookie` header value for authorized archives.
    pub fn with_cookie(mut self, cookie: impl Into<String>) -> Self {
        self.cookie = Some(cookie.into());
        self
    }
}

#[derive(Debug, Clone, Serialize, Deserialize)]
/// Summary returned after an archive run completes or resumes successfully.
pub struct ArchiveSummary {
    /// Archive ID used for the run.
    pub archive_id: String,
    /// Absolute path to the archive directory.
    pub archive_path: PathBuf,
    /// Final crawl counters.
    pub stats: CrawlStats,
}

impl ArchiveSummary {
    /// Path to the archive `manifest.json`.
    pub fn manifest_path(&self) -> PathBuf {
        self.archive_path.join("manifest.json")
    }

    /// Path to the siteforge `checksums.json`.
    pub fn checksums_path(&self) -> PathBuf {
        self.archive_path.join("checksums.json")
    }

    /// Path to the top-level `AGENTS.md`.
    pub fn agents_path(&self) -> PathBuf {
        self.archive_path.join("AGENTS.md")
    }

    /// Path to the machine-readable `agent-index.json`.
    pub fn agent_index_path(&self) -> PathBuf {
        self.archive_path.join("agent-index.json")
    }

    /// Path to `.thesa/manifest.json`.
    pub fn thesa_manifest_path(&self) -> PathBuf {
        self.archive_path.join(".thesa").join("manifest.json")
    }

    /// Path to `.thesa/checksums.blake3`.
    pub fn thesa_checksums_path(&self) -> PathBuf {
        self.archive_path.join(".thesa").join("checksums.blake3")
    }
}

/// Sender type used by archive/resume operations to emit progress events.
pub type ArchiveProgressSender = mpsc::Sender<ArchiveProgress>;

#[derive(Debug, Clone)]
/// Point-in-time archive progress event.
pub struct ArchiveProgress {
    /// Archive ID associated with the run.
    pub archive_id: String,
    /// Current high-level run phase.
    pub phase: ArchiveProgressPhase,
    /// URL currently being processed, when available.
    pub active_url: Option<String>,
    /// Best-known crawl counters at the time of emission.
    pub stats: CrawlStats,
    /// Configured maximum page count for progress scaling.
    pub max_pages: usize,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
/// High-level archive lifecycle phase.
pub enum ArchiveProgressPhase {
    /// Archive layout, manifest, and frontier are being initialized.
    Starting,
    /// Sitemap and seed discovery is running.
    Discovering,
    /// Page fetch tasks are active.
    Fetching,
    /// A fetched page is being parsed, written, or indexed.
    Processing,
    /// Archive completed normally.
    Completed,
    /// Archive was interrupted and can be resumed.
    Interrupted,
}

impl ArchiveProgressPhase {
    /// Stable lowercase label for status displays and logs.
    pub fn as_str(self) -> &'static str {
        match self {
            ArchiveProgressPhase::Starting => "starting",
            ArchiveProgressPhase::Discovering => "discovering",
            ArchiveProgressPhase::Fetching => "fetching",
            ArchiveProgressPhase::Processing => "processing",
            ArchiveProgressPhase::Completed => "completed",
            ArchiveProgressPhase::Interrupted => "interrupted",
        }
    }
}

#[derive(Debug, Clone)]
/// URL scope predicate used by the crawler.
pub struct CrawlScope {
    seeds: Vec<SeedScope>,
    same_domain: bool,
    max_depth: usize,
    include: GlobSet,
    exclude: GlobSet,
}

#[derive(Debug, Clone)]
struct SeedScope {
    host: String,
    path_prefix: String,
}

impl CrawlScope {
    /// Build a crawl scope from seeds, same-domain policy, depth, and config globs.
    pub fn new(
        seeds: &[Url],
        same_domain: bool,
        max_depth: usize,
        config: &Config,
    ) -> Result<Self> {
        Ok(Self {
            seeds: seeds.iter().map(seed_scope).collect(),
            same_domain,
            max_depth,
            include: globset_from_patterns(&config.include_url_patterns)?,
            exclude: globset_from_patterns(&config.exclude_url_patterns)?,
        })
    }

    /// Return whether `url` is allowed at the given crawl depth.
    pub fn allows(&self, url: &Url, depth: usize) -> bool {
        if depth > self.max_depth || !matches!(url.scheme(), "http" | "https") {
            return false;
        }
        let url_string = url.as_str();
        if self.exclude.is_match(url_string) {
            return false;
        }
        if !self.include.is_empty() && !self.include.is_match(url_string) {
            return false;
        }
        if !self.same_domain {
            return true;
        }
        let Some(host) = url.host_str() else {
            return false;
        };
        let host = host.trim_start_matches("www.");
        self.seeds.iter().any(|seed| {
            seed.host == host
                && (url.path().starts_with(&seed.path_prefix) || seed.path_prefix == "/")
        })
    }
}

pub async fn run_archive(config: Config, options: ArchiveRunOptions) -> Result<ArchiveSummary> {
    run_archive_inner(config, options, None).await
}

pub async fn run_archive_with_progress(
    config: Config,
    options: ArchiveRunOptions,
    progress: ArchiveProgressSender,
) -> Result<ArchiveSummary> {
    run_archive_inner(config, options, Some(progress)).await
}

async fn run_archive_inner(
    config: Config,
    options: ArchiveRunOptions,
    progress: Option<ArchiveProgressSender>,
) -> Result<ArchiveSummary> {
    if options.seeds.is_empty() {
        return Err(SiteforgeError::MissingInput("URL or --input"));
    }
    let root = config.resolved_archive_root()?;
    let archive_id = match options.archive_id.as_deref() {
        Some(archive_id) => {
            validate_archive_id(archive_id)?;
            archive_id.to_string()
        }
        None => archive_id_from_seed(&options.seeds[0]),
    };
    let layout = ArchiveLayout::new(root, archive_id.clone());
    if layout.base().exists() {
        return Err(SiteforgeError::message(format!(
            "archive {archive_id:?} already exists; use `siteforge resume {archive_id}` or choose a different --archive-id"
        )));
    }
    layout.ensure()?;

    let mut manifest = new_manifest(
        archive_id.clone(),
        &options.seeds,
        options.full_site,
        options.same_domain,
        options.max_depth,
        &config,
    );
    manifest.config.concurrency = options.concurrency;
    manifest.config.delay_ms = options.delay_ms;
    manifest.config.max_pages = options.max_pages;
    manifest.config.archive_profile = config.archive_profile;
    manifest.config.render_js = config.render_js;
    manifest.config.render_js_auto = config.render_js_auto;
    write_manifest(&layout, &manifest)?;

    let storage = Storage::open_compact(&layout.database())?;
    storage.create_or_update_job(&archive_id, &archive_id, "running")?;
    storage.log("info", "archive job started")?;
    for seed in &options.seeds {
        storage.enqueue_url(seed.as_str(), 0, None)?;
    }
    fs::write(layout.chunks_jsonl(), "")?;
    let mut progress_emitter = ProgressEmitter::new(progress.as_ref(), manifest.stats.clone());
    progress_emitter.emit(
        &manifest,
        &storage,
        options.max_pages,
        ArchiveProgressPhase::Starting,
        None,
    );
    let mut write_budget = ArchiveWriteBudget::new(&config, &layout)?;

    run_loop(
        &config,
        &layout,
        &storage,
        &mut manifest,
        &options,
        &mut progress_emitter,
        &mut write_budget,
    )
    .await?;
    let stats = storage.crawl_stats()?;
    storage.update_job(&archive_id, "completed", None)?;
    storage.log("info", "archive job completed")?;
    manifest.stats = stats.clone();
    write_manifest(&layout, &manifest)?;
    write_readable_index(&config, &layout, &storage, &manifest, &mut write_budget)?;
    storage.checkpoint()?;
    write_checksums_streamed(&layout)?;
    write_thesa_sidecar(&layout, &manifest, true)?;
    progress_emitter.emit(
        &manifest,
        &storage,
        options.max_pages,
        ArchiveProgressPhase::Completed,
        None,
    );

    Ok(ArchiveSummary {
        archive_id,
        archive_path: layout.base(),
        stats,
    })
}

pub async fn resume_archive(config: Config, archive_id: &str) -> Result<ArchiveSummary> {
    resume_archive_inner(config, archive_id, None).await
}

pub async fn resume_archive_with_progress(
    config: Config,
    archive_id: &str,
    progress: ArchiveProgressSender,
) -> Result<ArchiveSummary> {
    resume_archive_inner(config, archive_id, Some(progress)).await
}

async fn resume_archive_inner(
    mut config: Config,
    archive_id: &str,
    progress: Option<ArchiveProgressSender>,
) -> Result<ArchiveSummary> {
    let root = config.resolved_archive_root()?;
    let layout = ArchiveLayout::new(root, archive_id.to_string());
    if !layout.manifest().exists() {
        return Err(SiteforgeError::ArchiveNotFound(archive_id.to_string()));
    }
    let mut manifest = read_manifest(&layout.manifest())?;
    config.archive_profile = manifest.config.archive_profile;
    config.render_js = manifest.config.render_js;
    config.render_js_auto = manifest.config.render_js_auto;
    let seeds = manifest
        .seeds
        .iter()
        .map(|seed| Url::parse(seed))
        .collect::<std::result::Result<Vec<_>, _>>()?;
    let options = ArchiveRunOptions {
        seeds,
        full_site: manifest.scope.full_site,
        same_domain: manifest.scope.same_domain,
        max_depth: manifest.scope.max_depth,
        max_pages: manifest.config.max_pages,
        concurrency: manifest.config.concurrency.max(1),
        delay_ms: manifest.config.delay_ms,
        headers: Vec::new(),
        cookie: None,
        archive_id: Some(archive_id.to_string()),
    };
    let storage = Storage::open_compact(&layout.database())?;
    storage.reset_in_progress_frontier()?;
    storage.create_or_update_job(archive_id, archive_id, "running")?;
    storage.log("info", "archive job resumed")?;
    let mut progress_emitter = ProgressEmitter::new(progress.as_ref(), manifest.stats.clone());
    progress_emitter.emit(
        &manifest,
        &storage,
        options.max_pages,
        ArchiveProgressPhase::Starting,
        None,
    );
    let mut write_budget = ArchiveWriteBudget::new(&config, &layout)?;
    run_loop(
        &config,
        &layout,
        &storage,
        &mut manifest,
        &options,
        &mut progress_emitter,
        &mut write_budget,
    )
    .await?;
    let stats = storage.crawl_stats()?;
    storage.update_job(archive_id, "completed", None)?;
    manifest.stats = stats.clone();
    write_manifest(&layout, &manifest)?;
    write_readable_index(&config, &layout, &storage, &manifest, &mut write_budget)?;
    storage.checkpoint()?;
    write_checksums_streamed(&layout)?;
    write_thesa_sidecar(&layout, &manifest, true)?;
    progress_emitter.emit(
        &manifest,
        &storage,
        options.max_pages,
        ArchiveProgressPhase::Completed,
        None,
    );
    Ok(ArchiveSummary {
        archive_id: archive_id.to_string(),
        archive_path: layout.base(),
        stats,
    })
}

async fn run_loop(
    config: &Config,
    layout: &ArchiveLayout,
    storage: &Storage,
    manifest: &mut ArchiveManifest,
    options: &ArchiveRunOptions,
    progress: &mut ProgressEmitter<'_>,
    write_budget: &mut ArchiveWriteBudget,
) -> Result<()> {
    let fetcher = HttpFetcher::new(
        config,
        options.delay_ms,
        &options.headers,
        options.cookie.as_deref(),
    )?;
    let scope = CrawlScope::new(
        &options.seeds,
        options.same_domain,
        options.max_depth,
        config,
    )?;
    let ocr_backend = backend_from_config(config);

    let mut interrupt = Box::pin(tokio::signal::ctrl_c());

    if options.full_site {
        progress.emit(
            manifest,
            storage,
            options.max_pages,
            ArchiveProgressPhase::Discovering,
            None,
        );
        tokio::select! {
            biased;
            _ = &mut interrupt => {
                finish_interrupted(storage, layout, manifest, options, progress)?;
                return Err(SiteforgeError::ArchiveInterrupted(manifest.archive_id.clone()));
            }
            result = enqueue_sitemap_urls(storage, &fetcher, &scope, &options.seeds, options.max_pages) => result?,
        }
    }

    loop {
        let stats = storage.crawl_stats()?;
        if stats.pages_fetched >= options.max_pages {
            storage.log("warn", "max page limit reached")?;
            break;
        }
        let remaining = options.max_pages.saturating_sub(stats.pages_fetched).max(1);
        let mut pending = storage.pending_frontier(options.concurrency.min(remaining).max(1))?;
        if pending.is_empty() {
            break;
        }
        progress.emit(
            manifest,
            storage,
            options.max_pages,
            ArchiveProgressPhase::Fetching,
            None,
        );

        storage.mark_frontier_batch(
            pending.iter().map(|item| item.url.as_str()),
            "in_progress",
            None,
        )?;

        let mut futures = FuturesUnordered::new();
        for item in pending.drain(..) {
            let fetcher = fetcher.clone();
            let max_page_size_bytes = config.max_page_size_bytes;
            futures.push(async move {
                let result = match Url::parse(&item.url) {
                    Ok(url) => fetcher.fetch_with_limit(url, max_page_size_bytes).await,
                    Err(err) => Err(SiteforgeError::from(err)),
                };
                (item, result)
            });
        }

        loop {
            let next = tokio::select! {
                biased;
                _ = &mut interrupt => {
                    finish_interrupted(storage, layout, manifest, options, progress)?;
                    return Err(SiteforgeError::ArchiveInterrupted(manifest.archive_id.clone()));
                }
                next = futures.next() => next,
            };
            let Some((item, result)) = next else {
                break;
            };
            storage.update_job(&manifest.archive_id, "running", Some(&item.url))?;
            progress.emit(
                manifest,
                storage,
                options.max_pages,
                ArchiveProgressPhase::Processing,
                Some(&item.url),
            );
            match result {
                Ok(fetch_result) => {
                    let page_result = tokio::select! {
                        biased;
                        _ = &mut interrupt => {
                            finish_interrupted(storage, layout, manifest, options, progress)?;
                            return Err(SiteforgeError::ArchiveInterrupted(manifest.archive_id.clone()));
                        }
                        result = handle_page(
                            config,
                            layout,
                            storage,
                            manifest,
                            options,
                            &scope,
                            &fetcher,
                            ocr_backend
                                .as_ref()
                                .map(|backend| backend as &dyn OcrBackend),
                            item,
                            fetch_result,
                            write_budget,
                        ) => result,
                    };
                    if let Err(err) = page_result {
                        storage.log("error", &format!("page handling failed: {err}"))?;
                    }
                }
                Err(err @ SiteforgeError::RobotsDenied(_)) => {
                    storage.mark_frontier(&item.url, "skipped", Some(&err.to_string()))?;
                    storage.log("warn", &err.to_string())?;
                }
                Err(err) => {
                    let retries = storage.frontier_retry_count(&item.url)?;
                    if retries < config.retry_count {
                        storage.increment_frontier_retry(&item.url, &err.to_string())?;
                    } else {
                        let page_id = format!("page-{}", stable_id(&item.url));
                        storage.insert_failed_page(
                            &manifest.archive_id,
                            &page_id,
                            &item.url,
                            &item.url,
                            0,
                            item.depth,
                            &err.to_string(),
                        )?;
                        storage.mark_frontier(&item.url, "failed", Some(&err.to_string()))?;
                        storage.log("error", &format!("fetch failed for {}: {err}", item.url))?;
                    }
                }
            }
            progress.emit(
                manifest,
                storage,
                options.max_pages,
                ArchiveProgressPhase::Fetching,
                None,
            );
        }

        manifest.stats = storage.crawl_stats()?;
        write_manifest(layout, manifest)?;
    }
    storage.update_job(&manifest.archive_id, "running", None)?;
    Ok(())
}

fn finish_interrupted(
    storage: &Storage,
    layout: &ArchiveLayout,
    manifest: &mut ArchiveManifest,
    options: &ArchiveRunOptions,
    progress: &mut ProgressEmitter<'_>,
) -> Result<()> {
    storage.update_job(&manifest.archive_id, "interrupted", None)?;
    storage.log("warn", "archive job interrupted by signal")?;
    storage.reset_in_progress_frontier()?;
    manifest.stats = storage.crawl_stats()?;
    write_manifest(layout, manifest)?;
    storage.checkpoint()?;
    progress.emit(
        manifest,
        storage,
        options.max_pages,
        ArchiveProgressPhase::Interrupted,
        None,
    );
    Ok(())
}

struct ProgressEmitter<'a> {
    sender: Option<&'a ArchiveProgressSender>,
    last_emit: Option<Instant>,
    last_phase: Option<ArchiveProgressPhase>,
    stats: CrawlStats,
}

impl<'a> ProgressEmitter<'a> {
    fn new(sender: Option<&'a ArchiveProgressSender>, stats: CrawlStats) -> Self {
        Self {
            sender,
            last_emit: None,
            last_phase: None,
            stats,
        }
    }

    fn emit(
        &mut self,
        manifest: &ArchiveManifest,
        storage: &Storage,
        max_pages: usize,
        phase: ArchiveProgressPhase,
        active_url: Option<&str>,
    ) {
        let Some(sender) = self.sender else {
            return;
        };

        let now = Instant::now();
        let phase_changed = self.last_phase != Some(phase);
        let force_phase = phase_changed && !is_hot_progress_transition(self.last_phase, phase);
        let due = self
            .last_emit
            .map(|last| now.saturating_duration_since(last) >= PROGRESS_EMIT_INTERVAL)
            .unwrap_or(true);
        if !force_phase && !due {
            return;
        }

        self.stats = storage
            .crawl_stats()
            .unwrap_or_else(|_| manifest.stats.clone());
        self.last_emit = Some(now);
        self.last_phase = Some(phase);
        let _ = sender.try_send(ArchiveProgress {
            archive_id: manifest.archive_id.clone(),
            phase,
            active_url: active_url.map(str::to_string),
            stats: self.stats.clone(),
            max_pages,
        });
    }
}

fn is_hot_progress_transition(
    previous: Option<ArchiveProgressPhase>,
    next: ArchiveProgressPhase,
) -> bool {
    matches!(
        (previous, next),
        (
            Some(ArchiveProgressPhase::Fetching),
            ArchiveProgressPhase::Processing
        ) | (
            Some(ArchiveProgressPhase::Processing),
            ArchiveProgressPhase::Fetching
        )
    )
}

fn write_readable_index(
    config: &Config,
    layout: &ArchiveLayout,
    storage: &Storage,
    manifest: &ArchiveManifest,
    write_budget: &mut ArchiveWriteBudget,
) -> Result<()> {
    write_readme_index(config, layout, storage, manifest, write_budget)?;
    write_agent_entrypoints(config, layout, storage, manifest, write_budget)
}

fn write_readme_index(
    config: &Config,
    layout: &ArchiveLayout,
    storage: &Storage,
    manifest: &ArchiveManifest,
    write_budget: &mut ArchiveWriteBudget,
) -> Result<()> {
    let readme_path = layout.readable_dir().join("README.md");
    write_archive_file_streamed(config, layout, write_budget, &readme_path, |writer| {
        writeln!(writer, "# Siteforge Archive `{}`\n", manifest.archive_id)?;
        writer.write_all(b"This is the file-first index for humans and AI agents. The cleaned Markdown, structured JSON, text, chunks, and packs are the primary artifacts. SQLite is supporting crawl state/search metadata only.\n\n")?;
        writer.write_all(b"## Seeds\n\n")?;
        for seed in &manifest.seeds {
            writeln!(writer, "- {seed}")?;
        }
        writer.write_all(b"\n## Stats\n\n")?;
        write!(
            writer,
            "- Pages discovered: {}\n- Pages parsed: {}\n- Pages failed: {}\n- Assets downloaded: {}\n\n",
            manifest.stats.pages_discovered,
            manifest.stats.pages_parsed,
            manifest.stats.pages_failed,
            manifest.stats.assets_downloaded
        )?;
        writeln!(
            writer,
            "## Output Profile\n\n- {:?}\n",
            config.archive_profile
        )?;
        writer.write_all(b"## AI-Readable Artifacts\n\n")?;
        if config.archive_profile.stores_markdown() {
            writer.write_all(
                b"- `readable/markdown/`: cleaned Markdown per page with source attribution.\n",
            )?;
        }
        if config.archive_profile.stores_basic_markdown() {
            writer.write_all(b"- `readable/basic_markdown/`: plain Markdown per page with no YAML frontmatter, plus page data, links, and assets.\n")?;
        }
        if config.archive_profile.stores_json() {
            writer.write_all(b"- `readable/json/`: structured page JSON.\n")?;
        }
        if config.archive_profile.stores_text() {
            writer.write_all(b"- `readable/text/`: plain text for quick search/read.\n")?;
        }
        if config.archive_profile.stores_chunks() {
            writer
                .write_all(b"- `chunks/chunks.jsonl`: semantic chunks for RAG/AI ingestion.\n")?;
        }
        writer.write_all(b"- `packs/`: token-budgeted AI context packs.\n\n")?;
        writer.write_all(b"## Agent Entrypoints\n\n")?;
        writer.write_all(b"- `AGENTS.md`: start here for quick ingestion guidance.\n")?;
        writer.write_all(b"- `agent-index.json`: machine-readable page/asset index.\n\n")?;
        writer.write_all(b"## Pages\n\n")?;

        let mut offset = 0usize;
        loop {
            let pages = storage.list_pages_page(CRAWL_INDEX_PAGE_BATCH, offset)?;
            if pages.is_empty() {
                break;
            }
            offset = offset.saturating_add(pages.len());
            for page in pages.iter().filter(|page| page.markdown_path.is_some()) {
                let primary_path = page.markdown_path.as_deref().unwrap_or_default();
                let primary_link = primary_path
                    .strip_prefix("readable/")
                    .unwrap_or(primary_path);
                write!(
                    writer,
                    "- [{}]({}) - {}\n  - Source: {}\n",
                    page.title, primary_link, page.page_id, page.source_url
                )?;
                if let Some(basic_path) = stored_basic_markdown_path(page) {
                    if Some(basic_path.as_str()) != page.markdown_path.as_deref() {
                        let basic_link = basic_path
                            .strip_prefix("readable/")
                            .unwrap_or(basic_path.as_str());
                        writeln!(writer, "  - Basic: [{}]({})", page.title, basic_link)?;
                    }
                }
            }
        }

        let asset_count = storage.asset_count()?;
        if asset_count > 0 {
            writer.write_all(b"\n## Assets\n\n")?;
            for asset in storage.list_assets_limited(README_ASSET_LIMIT)? {
                write!(
                    writer,
                    "- {} - {}\n  - Local: {}\n",
                    asset.asset_id,
                    asset.source_url,
                    asset.local_path.as_deref().unwrap_or("not downloaded")
                )?;
            }
            if asset_count > README_ASSET_LIMIT {
                writeln!(
                    writer,
                    "\n_{} additional assets omitted from this index._",
                    asset_count - README_ASSET_LIMIT
                )?;
            }
        }
        Ok(())
    })
}

fn write_agent_entrypoints(
    config: &Config,
    layout: &ArchiveLayout,
    storage: &Storage,
    manifest: &ArchiveManifest,
    write_budget: &mut ArchiveWriteBudget,
) -> Result<()> {
    let mut agents = String::new();
    agents.push_str(&format!("# AGENTS.md for `{}`\n\n", manifest.archive_id));
    agents.push_str("This archive is optimized for direct agent ingestion. Prefer file artifacts over SQLite; the database is crawl/search state only.\n\n");
    agents.push_str("## Fast Paths\n\n");
    agents.push_str("- Start with `agent-index.json` for machine-readable page and asset paths.\n");
    agents.push_str("- Use `readable/basic_markdown/` for simple Markdown with content, metadata, links, and assets.\n");
    if config.archive_profile.stores_markdown() {
        agents.push_str(
            "- Use `readable/markdown/` when YAML frontmatter and source comments are useful.\n",
        );
    }
    agents
        .push_str("- Use `chunks/chunks.jsonl` for RAG ingestion; code chunks are never split.\n");
    agents.push_str("- Use `packs/` for token-budgeted context packs.\n");
    if config.archive_profile.stores_raw_pages() {
        agents.push_str("- Use `raw/pages/`, `raw/rendered/`, and `raw/assets/` when exact source payloads or local assets are needed.\n\n");
    } else {
        agents.push_str("- This archive was written with a lean profile, so raw payloads and downloaded assets may be omitted.\n\n");
    }
    agents.push_str("## Suggested Agent Order\n\n");
    agents.push_str("1. Read `agent-index.json`.\n");
    agents.push_str("2. Load `readable/basic_markdown/*.md` for broad understanding.\n");
    agents.push_str("3. Load `chunks/chunks.jsonl` for precise retrieval.\n");
    agents.push_str("4. Open referenced raw/assets files only when needed.\n\n");
    agents.push_str("## Archive Stats\n\n");
    agents.push_str(&format!(
        "- Pages parsed: {}\n- Pages failed: {}\n- Assets downloaded: {}\n",
        manifest.stats.pages_parsed, manifest.stats.pages_failed, manifest.stats.assets_downloaded
    ));

    let readable_agents = layout.readable_dir().join("AGENTS.md");
    write_archive_file(
        config,
        layout,
        write_budget,
        &layout.agents_md(),
        agents.as_bytes(),
    )?;
    write_archive_file(
        config,
        layout,
        write_budget,
        &readable_agents,
        agents.as_bytes(),
    )?;
    write_agent_index_json(config, layout, storage, manifest, write_budget)
}

#[derive(Serialize)]
struct AgentPathIndex<'a> {
    agents_md: &'a str,
    readable_readme: &'a str,
    basic_markdown: &'a str,
    markdown: &'a str,
    json: &'a str,
    text: &'a str,
    chunks_jsonl: &'a str,
    packs: &'a str,
    raw_pages: &'a str,
    rendered_pages: &'a str,
    raw_assets: &'a str,
}

#[derive(Serialize)]
struct AgentPageIndex<'a> {
    page_id: &'a str,
    title: &'a str,
    source_url: &'a str,
    final_url: &'a str,
    basic_markdown_path: Option<String>,
    markdown_path: Option<&'a str>,
    json_path: Option<&'a str>,
    text_path: Option<&'a str>,
    raw_path: Option<&'a str>,
    status: i64,
}

#[derive(Serialize)]
struct AgentAssetIndex<'a> {
    asset_id: &'a str,
    page_id: &'a str,
    source_url: &'a str,
    local_path: Option<&'a str>,
    mime_type: Option<&'a str>,
    size_bytes: Option<u64>,
    ocr_status: Option<&'a str>,
}

fn write_agent_index_json(
    config: &Config,
    layout: &ArchiveLayout,
    storage: &Storage,
    manifest: &ArchiveManifest,
    write_budget: &mut ArchiveWriteBudget,
) -> Result<()> {
    write_archive_file_streamed(
        config,
        layout,
        write_budget,
        &layout.agent_index_json(),
        |writer| {
            writer.write_all(b"{\n  \"archive_id\": ")?;
            serde_json::to_writer(&mut *writer, &manifest.archive_id)?;
            writer.write_all(b",\n  \"updated_at\": ")?;
            serde_json::to_writer(&mut *writer, &manifest.updated_at.to_rfc3339())?;
            writer.write_all(b",\n  \"seeds\": ")?;
            serde_json::to_writer(&mut *writer, &manifest.seeds)?;
            writer.write_all(b",\n  \"paths\": ")?;
            serde_json::to_writer(&mut *writer, &agent_path_index())?;
            writer.write_all(b",\n  \"pages\": [")?;
            write_agent_pages(writer, storage)?;
            writer.write_all(b"\n  ],\n  \"assets\": [")?;
            write_agent_assets(writer, storage)?;
            writer.write_all(b"\n  ]\n}\n")?;
            Ok(())
        },
    )
}

fn agent_path_index() -> AgentPathIndex<'static> {
    AgentPathIndex {
        agents_md: "AGENTS.md",
        readable_readme: "readable/README.md",
        basic_markdown: "readable/basic_markdown/",
        markdown: "readable/markdown/",
        json: "readable/json/",
        text: "readable/text/",
        chunks_jsonl: "chunks/chunks.jsonl",
        packs: "packs/",
        raw_pages: "raw/pages/",
        rendered_pages: "raw/rendered/",
        raw_assets: "raw/assets/",
    }
}

fn write_agent_pages(writer: &mut BufWriter<File>, storage: &Storage) -> Result<()> {
    let mut first = true;
    let mut offset = 0usize;
    loop {
        let pages = storage.list_pages_page(CRAWL_INDEX_PAGE_BATCH, offset)?;
        if pages.is_empty() {
            break;
        }
        offset = offset.saturating_add(pages.len());
        for page in &pages {
            if !first {
                writer.write_all(b",")?;
            }
            writer.write_all(b"\n    ")?;
            serde_json::to_writer(&mut *writer, &agent_page_index(page))?;
            first = false;
        }
    }
    Ok(())
}

fn write_agent_assets(writer: &mut BufWriter<File>, storage: &Storage) -> Result<()> {
    let mut first = true;
    let mut offset = 0usize;
    loop {
        let assets = storage.list_assets_page(CRAWL_INDEX_ASSET_BATCH, offset)?;
        if assets.is_empty() {
            break;
        }
        offset = offset.saturating_add(assets.len());
        for asset in &assets {
            if !first {
                writer.write_all(b",")?;
            }
            writer.write_all(b"\n    ")?;
            serde_json::to_writer(&mut *writer, &agent_asset_index(asset))?;
            first = false;
        }
    }
    Ok(())
}

fn agent_page_index(page: &StoredPage) -> AgentPageIndex<'_> {
    AgentPageIndex {
        page_id: &page.page_id,
        title: &page.title,
        source_url: &page.source_url,
        final_url: &page.final_url,
        basic_markdown_path: stored_basic_markdown_path(page),
        markdown_path: stored_markdown_path(page),
        json_path: page.json_path.as_deref(),
        text_path: page.text_path.as_deref(),
        raw_path: page.raw_path.as_deref(),
        status: page.status,
    }
}

fn stored_basic_markdown_path(page: &StoredPage) -> Option<String> {
    page.markdown_path.as_ref().map(|path| {
        if path.starts_with("readable/basic_markdown/") {
            path.clone()
        } else {
            format!("readable/basic_markdown/{}.md", page.page_id)
        }
    })
}

fn stored_markdown_path(page: &StoredPage) -> Option<&str> {
    page.markdown_path
        .as_deref()
        .filter(|path| !path.starts_with("readable/basic_markdown/"))
}

fn agent_asset_index(asset: &StoredAsset) -> AgentAssetIndex<'_> {
    AgentAssetIndex {
        asset_id: &asset.asset_id,
        page_id: &asset.page_id,
        source_url: &asset.source_url,
        local_path: asset.local_path.as_deref(),
        mime_type: asset.mime_type.as_deref(),
        size_bytes: asset.size_bytes,
        ocr_status: asset.ocr_status.as_deref(),
    }
}

#[allow(clippy::too_many_arguments)]
async fn handle_page(
    config: &Config,
    layout: &ArchiveLayout,
    storage: &Storage,
    manifest: &ArchiveManifest,
    options: &ArchiveRunOptions,
    scope: &CrawlScope,
    fetcher: &HttpFetcher,
    ocr_backend: Option<&dyn OcrBackend>,
    item: FrontierItem,
    result: FetchResult,
    write_budget: &mut ArchiveWriteBudget,
) -> Result<()> {
    let page_id = format!("page-{}", stable_id(&item.url));
    if result.status >= 400 {
        storage.insert_failed_page(
            &manifest.archive_id,
            &page_id,
            &item.url,
            result.final_url.as_str(),
            result.status as i64,
            item.depth,
            &format!("HTTP {}", result.status),
        )?;
        storage.mark_frontier(
            &item.url,
            "failed",
            Some(&format!("HTTP {}", result.status)),
        )?;
        return Ok(());
    }

    if !is_html(&result) {
        storage.insert_failed_page(
            &manifest.archive_id,
            &page_id,
            &item.url,
            result.final_url.as_str(),
            result.status as i64,
            item.depth,
            "unsupported non-HTML page response",
        )?;
        storage.mark_frontier(
            &item.url,
            "skipped",
            Some("unsupported non-HTML page response"),
        )?;
        return Ok(());
    }

    let raw_path = layout.raw_pages_dir().join(format!("{page_id}.html"));
    let raw_rel = if config.archive_profile.stores_raw_pages() {
        write_archive_file(config, layout, write_budget, &raw_path, &result.bytes)?;
        Some(relative_to_base(layout, &raw_path))
    } else {
        None
    };
    let mut html = bytes_to_string(&result.bytes);
    let mut parse_hash = result.content_hash.clone();
    let should_render =
        config.render_js && (!config.render_js_auto || likely_needs_js_render(&html));
    if should_render {
        match render_js_dom(
            BrowserRenderOptions::from_config(config),
            result.final_url.clone(),
        )
        .await
        {
            Ok(rendered_html) => {
                parse_hash = blake3::hash(rendered_html.as_bytes()).to_hex().to_string();
                if config.archive_profile.stores_rendered_pages() {
                    let rendered_path = layout.rendered_pages_dir().join(format!("{page_id}.html"));
                    write_archive_file(
                        config,
                        layout,
                        write_budget,
                        &rendered_path,
                        rendered_html.as_bytes(),
                    )?;
                }
                html = rendered_html;
                storage.log(
                    "info",
                    &format!("captured rendered DOM for {}", result.final_url),
                )?;
            }
            Err(err) => {
                storage.log(
                    "warn",
                    &format!(
                        "JS render failed for {}: {err}; using fetched HTML",
                        result.final_url
                    ),
                )?;
            }
        }
    }
    let mut parsed = parse_html(
        &manifest.archive_id,
        &page_id,
        &result.final_url,
        &html,
        &parse_hash,
        item.depth,
    );
    add_downloadable_link_assets(&mut parsed);

    if config.archive_profile.downloads_assets() {
        download_assets(
            config,
            layout,
            storage,
            fetcher,
            ocr_backend,
            &mut parsed,
            write_budget,
        )
        .await?;
    }

    let markdown_path = layout.readable_markdown_dir().join(format!("{page_id}.md"));
    let basic_markdown_path = layout
        .readable_basic_markdown_dir()
        .join(format!("{page_id}.md"));
    let json_path = layout.readable_json_dir().join(format!("{page_id}.json"));
    let text_path = layout.readable_text_dir().join(format!("{page_id}.txt"));

    let regular_markdown_rel = if config.archive_profile.stores_markdown() {
        let markdown = parsed.to_markdown()?;
        write_archive_file(
            config,
            layout,
            write_budget,
            &markdown_path,
            markdown.as_bytes(),
        )?;
        Some(relative_to_base(layout, &markdown_path))
    } else {
        None
    };
    let basic_markdown_rel = if config.archive_profile.stores_basic_markdown() {
        let basic_markdown = parsed.to_basic_markdown();
        write_archive_file(
            config,
            layout,
            write_budget,
            &basic_markdown_path,
            basic_markdown.as_bytes(),
        )?;
        Some(relative_to_base(layout, &basic_markdown_path))
    } else {
        None
    };
    let json_rel = if config.archive_profile.stores_json() {
        let json = serde_json::to_string_pretty(&parsed.to_page_json())?;
        write_archive_file(config, layout, write_budget, &json_path, json.as_bytes())?;
        Some(relative_to_base(layout, &json_path))
    } else {
        None
    };
    let text_rel = if config.archive_profile.stores_text() {
        let text = parsed.to_plain_text();
        write_archive_file(config, layout, write_budget, &text_path, text.as_bytes())?;
        Some(relative_to_base(layout, &text_path))
    } else {
        None
    };
    if config.archive_profile.stores_chunks() {
        append_chunks(config, layout, write_budget, &parsed)?;
    }

    let markdown_rel = regular_markdown_rel.or(basic_markdown_rel);
    let headers_json = serde_json::to_string(&result.headers)?;
    storage.upsert_page(
        &parsed,
        result.final_url.as_str(),
        result.status as i64,
        result.mime_type.as_deref(),
        raw_rel.as_deref(),
        markdown_rel.as_deref(),
        json_rel.as_deref(),
        text_rel.as_deref(),
        Some(&headers_json),
        None,
    )?;
    storage.mark_frontier(&item.url, "done", None)?;
    storage.log("info", &format!("archived {}", result.final_url))?;

    if options.full_site && item.depth < options.max_depth {
        enqueue_discovered_links(storage, scope, &parsed, item.depth + 1)?;
    }
    Ok(())
}

async fn download_assets(
    config: &Config,
    layout: &ArchiveLayout,
    storage: &Storage,
    fetcher: &HttpFetcher,
    ocr_backend: Option<&dyn OcrBackend>,
    parsed: &mut ParsedPage,
    write_budget: &mut ArchiveWriteBudget,
) -> Result<()> {
    let mut seen = HashSet::new();
    for index in 0..parsed.assets.len() {
        let asset_id = parsed.assets[index].asset_id.clone();
        if !seen.insert(asset_id.clone()) {
            continue;
        }
        let url = match Url::parse(&parsed.assets[index].url) {
            Ok(url) => url,
            Err(err) => {
                parsed.assets[index].ocr_status = Some(format!("asset_fetch_failed: {err}"));
                storage.upsert_asset(&parsed.page_id, &parsed.assets[index], None)?;
                continue;
            }
        };

        match fetcher
            .fetch_with_limit(url, config.max_asset_size_bytes)
            .await
        {
            Ok(result) => {
                if result.bytes.len() as u64 > config.max_asset_size_bytes {
                    parsed.assets[index].ocr_status = Some("asset_fetch_failed".to_string());
                    storage.upsert_asset(&parsed.page_id, &parsed.assets[index], None)?;
                    continue;
                }
                let extension = extension_for(&result.final_url, result.mime_type.as_deref());
                let file_name = format!("{asset_id}.{extension}");
                let asset_path = layout.raw_assets_dir().join(file_name);
                write_archive_file(config, layout, write_budget, &asset_path, &result.bytes)?;
                let rel = relative_to_base(layout, &asset_path);
                let markdown_rel = format!("../../{rel}");
                parsed.assets[index].local_path = Some(markdown_rel);
                parsed.assets[index].hash = Some(result.content_hash.clone());
                parsed.assets[index].dimensions =
                    image_dimensions(&result.bytes, result.mime_type.as_deref());
                let source_url = parsed.assets[index].url.clone();
                if is_source_text_asset(&result.final_url, result.mime_type.as_deref()) {
                    let text = bytes_to_string(&result.bytes);
                    if !text.trim().is_empty() {
                        parsed.add_source_asset_code(
                            &asset_id,
                            &source_url,
                            text,
                            language_for_source_asset(
                                &result.final_url,
                                result.mime_type.as_deref(),
                            ),
                        );
                    }
                }
                let fetched = FetchedAssetRecord {
                    final_url: result.final_url.to_string(),
                    mime_type: result.mime_type.clone(),
                    size_bytes: result.bytes.len() as u64,
                    content_hash: Some(result.content_hash.clone()),
                };

                if let Some(backend) = ocr_backend {
                    if likely_contains_code(&parsed.assets[index]) {
                        match backend.extract(&asset_path) {
                            Ok(output) if !output.text.trim().is_empty() => {
                                parsed.set_asset_ocr(
                                    &asset_id,
                                    "ok".to_string(),
                                    Some(output.text),
                                );
                            }
                            Ok(_) => {
                                parsed.set_asset_ocr(&asset_id, "empty".to_string(), None);
                            }
                            Err(SiteforgeError::OcrUnavailable(message)) => {
                                parsed.set_asset_ocr(&asset_id, "unavailable".to_string(), None);
                                storage.log("warn", &format!("OCR unavailable: {message}"))?;
                            }
                            Err(err) => {
                                parsed.set_asset_ocr(&asset_id, format!("failed: {err}"), None);
                            }
                        }
                    }
                }

                if let Some(asset) = parsed
                    .assets
                    .iter()
                    .find(|asset| asset.asset_id == asset_id)
                {
                    storage.upsert_asset(&parsed.page_id, asset, Some(&fetched))?;
                }
            }
            Err(err) => {
                parsed.assets[index].ocr_status = Some("asset_fetch_failed".to_string());
                storage.upsert_asset(&parsed.page_id, &parsed.assets[index], None)?;
                storage.log(
                    "warn",
                    &format!("asset fetch failed for {}: {err}", parsed.assets[index].url),
                )?;
            }
        }
    }
    Ok(())
}

fn enqueue_discovered_links(
    storage: &Storage,
    scope: &CrawlScope,
    parsed: &ParsedPage,
    depth: usize,
) -> Result<()> {
    for link in &parsed.links {
        let Ok(url) = Url::parse(&link.url) else {
            continue;
        };
        if scope.allows(&url, depth) {
            storage.enqueue_url(url.as_str(), depth, Some(&parsed.source_url))?;
        }
    }
    Ok(())
}

async fn enqueue_sitemap_urls(
    storage: &Storage,
    fetcher: &HttpFetcher,
    scope: &CrawlScope,
    seeds: &[Url],
    max_pages: usize,
) -> Result<()> {
    let mut sitemap_urls = Vec::new();
    for seed in seeds {
        match fetcher.robots_sitemaps(seed).await {
            Ok(urls) => sitemap_urls.extend(urls),
            Err(err) => storage.log("warn", &format!("robots sitemap discovery failed: {err}"))?,
        }
        if let Ok(default_sitemap) = seed.join("/sitemap.xml") {
            sitemap_urls.push(default_sitemap);
        }
    }
    sitemap_urls.sort();
    sitemap_urls.dedup();

    let mut visited = HashSet::new();
    let mut enqueued = 0usize;
    for sitemap_url in sitemap_urls {
        enqueue_sitemap_url(
            storage,
            fetcher,
            scope,
            sitemap_url,
            &mut visited,
            &mut enqueued,
            max_pages,
            0,
        )
        .await?;
        if enqueued >= max_pages {
            break;
        }
    }
    if enqueued > 0 {
        storage.log("info", &format!("queued {enqueued} URL(s) from sitemaps"))?;
    }
    Ok(())
}

#[allow(clippy::too_many_arguments)]
async fn enqueue_sitemap_url(
    storage: &Storage,
    fetcher: &HttpFetcher,
    scope: &CrawlScope,
    sitemap_url: Url,
    visited: &mut HashSet<String>,
    enqueued: &mut usize,
    max_pages: usize,
    sitemap_depth: usize,
) -> Result<()> {
    const MAX_SITEMAP_DEPTH: usize = 3;
    if sitemap_depth > MAX_SITEMAP_DEPTH
        || *enqueued >= max_pages
        || !visited.insert(sitemap_url.to_string())
    {
        return Ok(());
    }

    let result = match fetcher.fetch(sitemap_url.clone()).await {
        Ok(result) => result,
        Err(err) => {
            storage.log(
                "warn",
                &format!("sitemap fetch failed for {sitemap_url}: {err}"),
            )?;
            return Ok(());
        }
    };
    if result.status >= 400 {
        storage.log(
            "warn",
            &format!(
                "sitemap fetch returned HTTP {} for {sitemap_url}",
                result.status
            ),
        )?;
        return Ok(());
    }

    let body = sitemap_body(
        &result.bytes,
        result.mime_type.as_deref(),
        sitemap_url.path(),
    )?;
    for loc in sitemap_locs(&result.final_url, &body) {
        if *enqueued >= max_pages {
            break;
        }
        if is_sitemap_like(&loc) && sitemap_depth < MAX_SITEMAP_DEPTH {
            Box::pin(enqueue_sitemap_url(
                storage,
                fetcher,
                scope,
                loc,
                visited,
                enqueued,
                max_pages,
                sitemap_depth + 1,
            ))
            .await?;
        } else if scope.allows(&loc, 1) && storage.enqueue_url(loc.as_str(), 1, Some("sitemap"))? {
            *enqueued += 1;
        }
    }
    Ok(())
}

fn sitemap_body(bytes: &[u8], mime_type: Option<&str>, path: &str) -> Result<String> {
    if path.ends_with(".gz")
        || mime_type == Some("application/x-gzip")
        || mime_type == Some("application/gzip")
    {
        let mut decoder = flate2::read::GzDecoder::new(bytes);
        let mut body = String::new();
        std::io::Read::read_to_string(&mut decoder, &mut body)?;
        Ok(body)
    } else {
        Ok(bytes_to_string(bytes))
    }
}

fn sitemap_locs(base: &Url, body: &str) -> Vec<Url> {
    let regex = Regex::new(r"(?is)<loc>\s*([^<]+)\s*</loc>").expect("static sitemap regex");
    let mut urls = regex
        .captures_iter(body)
        .filter_map(|capture| capture.get(1).map(|value| value.as_str()))
        .map(|value| html_escape::decode_html_entities(value).trim().to_string())
        .filter_map(|value| Url::parse(&value).or_else(|_| base.join(&value)).ok())
        .collect::<Vec<_>>();
    urls.sort();
    urls.dedup();
    urls
}

fn is_sitemap_like(url: &Url) -> bool {
    let path = url.path().to_ascii_lowercase();
    path.ends_with(".xml") || path.ends_with(".xml.gz") || path.contains("sitemap")
}

fn add_downloadable_link_assets(parsed: &mut ParsedPage) {
    let mut existing = parsed
        .assets
        .iter()
        .map(|asset| asset.url.clone())
        .collect::<HashSet<_>>();
    for link in &parsed.links {
        if !is_downloadable_asset(&link.url) || !existing.insert(link.url.clone()) {
            continue;
        }
        let asset_id = format!("asset-{}", stable_id(&link.url));
        parsed.assets.push(AssetRef {
            asset_id,
            url: link.url.clone(),
            local_path: None,
            alt_text: Some(link.text.clone()).filter(|text| !text.trim().is_empty()),
            caption: None,
            dimensions: None,
            hash: None,
            appears_in: vec!["download-link".to_string()],
            ocr_text: None,
            ocr_status: None,
        });
    }
}

fn is_downloadable_asset(url: &str) -> bool {
    let lower = url.to_ascii_lowercase();
    [
        ".zip",
        ".tar",
        ".gz",
        ".pdf",
        ".cs",
        ".rs",
        ".py",
        ".js",
        ".mjs",
        ".jsx",
        ".ts",
        ".tsx",
        ".css",
        ".scss",
        ".sass",
        ".less",
        ".html",
        ".htm",
        ".md",
        ".txt",
        ".csv",
        ".json",
        ".jsonl",
        ".xml",
        ".svg",
        ".ipynb",
        ".wasm",
        ".shader",
        ".unitypackage",
    ]
    .iter()
    .any(|extension| {
        lower
            .split('?')
            .next()
            .unwrap_or(&lower)
            .ends_with(extension)
    })
}

fn append_chunks(
    config: &Config,
    layout: &ArchiveLayout,
    write_budget: &mut ArchiveWriteBudget,
    parsed: &ParsedPage,
) -> Result<()> {
    let mut payload = String::new();
    for chunk in parsed.chunks() {
        payload.push_str(&serde_json::to_string(&chunk)?);
        payload.push('\n');
    }
    reserve_archive_growth(
        config,
        write_budget,
        payload.len() as u64,
        &layout.chunks_jsonl(),
    )?;
    let mut file = OpenOptions::new()
        .create(true)
        .append(true)
        .open(layout.chunks_jsonl())?;
    file.write_all(payload.as_bytes())?;
    Ok(())
}

fn write_archive_file(
    config: &Config,
    _layout: &ArchiveLayout,
    write_budget: &mut ArchiveWriteBudget,
    path: &Path,
    bytes: &[u8],
) -> Result<()> {
    let existing = fs::metadata(path)
        .map(|metadata| metadata.len())
        .unwrap_or(0);
    let growth = (bytes.len() as u64).saturating_sub(existing);
    reserve_archive_growth(config, write_budget, growth, path)?;
    fs::write(path, bytes)?;
    Ok(())
}

fn write_archive_file_streamed<F>(
    config: &Config,
    layout: &ArchiveLayout,
    write_budget: &mut ArchiveWriteBudget,
    path: &Path,
    write: F,
) -> Result<()>
where
    F: FnOnce(&mut BufWriter<File>) -> Result<()>,
{
    fs::create_dir_all(layout.exports_dir())?;
    let temp_path = layout.exports_dir().join(format!(
        ".{}-{}.tmp",
        stable_id(&path.display().to_string()),
        std::process::id()
    ));
    let _ = fs::remove_file(&temp_path);

    let write_result = (|| -> Result<u64> {
        let mut writer = BufWriter::new(File::create(&temp_path)?);
        write(&mut writer)?;
        writer.flush()?;
        Ok(fs::metadata(&temp_path)?.len())
    })();

    let final_len = match write_result {
        Ok(final_len) => final_len,
        Err(err) => {
            let _ = fs::remove_file(&temp_path);
            return Err(err);
        }
    };
    let existing = fs::metadata(path)
        .map(|metadata| metadata.len())
        .unwrap_or(0);
    let growth = final_len.saturating_sub(existing);
    if let Err(err) = reserve_archive_growth(config, write_budget, growth, path) {
        let _ = fs::remove_file(&temp_path);
        return Err(err);
    }
    if let Some(parent) = path.parent() {
        fs::create_dir_all(parent)?;
    }
    fs::rename(&temp_path, path)?;
    Ok(())
}

fn reserve_archive_growth(
    config: &Config,
    write_budget: &mut ArchiveWriteBudget,
    growth: u64,
    path: &Path,
) -> Result<()> {
    if config.max_total_archive_size_bytes == 0 {
        return Ok(());
    }
    write_budget.reserve(growth, path)
}

fn relative_to_base(layout: &ArchiveLayout, path: &Path) -> String {
    path.strip_prefix(layout.base())
        .unwrap_or(path)
        .to_string_lossy()
        .replace('\\', "/")
}

fn extension_for(url: &Url, mime_type: Option<&str>) -> String {
    if let Some(extension) = Path::new(url.path())
        .extension()
        .and_then(|ext| ext.to_str())
    {
        let clean = extension
            .chars()
            .filter(|ch| ch.is_ascii_alphanumeric())
            .collect::<String>();
        if !clean.is_empty() && clean.len() <= 8 {
            return clean;
        }
    }
    match mime_type.unwrap_or_default() {
        "image/png" => "png",
        "image/jpeg" => "jpg",
        "image/gif" => "gif",
        "image/webp" => "webp",
        "image/svg+xml" => "svg",
        "application/pdf" => "pdf",
        "application/zip" => "zip",
        "text/css" => "css",
        "application/javascript" | "text/javascript" => "js",
        "application/json" => "json",
        "text/plain" => "txt",
        _ => "bin",
    }
    .to_string()
}

fn image_dimensions(bytes: &[u8], mime_type: Option<&str>) -> Option<ImageDimensions> {
    if !mime_type
        .map(|mime| mime.starts_with("image/"))
        .unwrap_or(false)
    {
        return None;
    }
    imagesize::blob_size(bytes)
        .ok()
        .map(|size| ImageDimensions {
            width: size.width as u32,
            height: size.height as u32,
        })
}

fn is_source_text_asset(url: &Url, mime_type: Option<&str>) -> bool {
    if mime_type
        .map(|mime| {
            mime.starts_with("text/")
                || matches!(
                    mime,
                    "application/json"
                        | "application/javascript"
                        | "application/x-javascript"
                        | "application/xml"
                        | "application/toml"
                        | "application/yaml"
                )
        })
        .unwrap_or(false)
    {
        return true;
    }
    source_extension(url).is_some()
}

fn language_for_source_asset(url: &Url, mime_type: Option<&str>) -> Option<String> {
    source_extension(url)
        .map(|extension| match extension.as_str() {
            "cs" => "csharp".to_string(),
            "rs" => "rust".to_string(),
            "py" => "python".to_string(),
            "js" => "javascript".to_string(),
            "mjs" => "javascript".to_string(),
            "jsx" => "javascript".to_string(),
            "ts" => "typescript".to_string(),
            "tsx" => "typescript".to_string(),
            "yml" => "yaml".to_string(),
            "md" => "markdown".to_string(),
            other => other.to_string(),
        })
        .or_else(|| {
            mime_type.and_then(|mime| match mime {
                "application/json" => Some("json".to_string()),
                "application/javascript" | "application/x-javascript" | "text/javascript" => {
                    Some("javascript".to_string())
                }
                "application/xml" | "text/xml" => Some("xml".to_string()),
                "text/css" => Some("css".to_string()),
                _ => None,
            })
        })
}

fn source_extension(url: &Url) -> Option<String> {
    Path::new(url.path())
        .extension()
        .and_then(|extension| extension.to_str())
        .map(|extension| extension.to_ascii_lowercase())
        .and_then(|extension| {
            matches!(
                extension.as_str(),
                "cs" | "rs"
                    | "py"
                    | "js"
                    | "mjs"
                    | "jsx"
                    | "ts"
                    | "tsx"
                    | "css"
                    | "scss"
                    | "sass"
                    | "less"
                    | "html"
                    | "htm"
                    | "md"
                    | "markdown"
                    | "txt"
                    | "csv"
                    | "json"
                    | "jsonl"
                    | "toml"
                    | "yaml"
                    | "yml"
                    | "xml"
                    | "svg"
                    | "sh"
                    | "bash"
                    | "ipynb"
                    | "wasm"
                    | "shader"
                    | "cginc"
                    | "hlsl"
                    | "glsl"
            )
            .then_some(extension)
        })
}

fn seed_scope(seed: &Url) -> SeedScope {
    let host = seed
        .host_str()
        .unwrap_or_default()
        .trim_start_matches("www.")
        .to_string();
    let path = seed.path();
    let path_prefix = if path.is_empty() || path == "/" {
        "/".to_string()
    } else if path.ends_with('/') {
        path.to_string()
    } else {
        path.rsplit_once('/')
            .map(|(prefix, _)| format!("{prefix}/"))
            .unwrap_or_else(|| "/".to_string())
    };
    SeedScope { host, path_prefix }
}

fn globset_from_patterns(patterns: &[String]) -> Result<GlobSet> {
    let mut builder = GlobSetBuilder::new();
    for pattern in patterns {
        builder.add(Glob::new(pattern)?);
    }
    Ok(builder.build()?)
}

pub fn normalize_url(base: &Url, raw: &str) -> Option<Url> {
    normalize_link(base, raw)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn scope_respects_domain_and_prefix() {
        let config = Config::default();
        let seed = Url::parse("https://catlikecoding.com/unity/tutorials/").unwrap();
        let scope = CrawlScope::new(&[seed], true, 2, &config).unwrap();
        assert!(scope.allows(
            &Url::parse("https://catlikecoding.com/unity/tutorials/mesh").unwrap(),
            1
        ));
        assert!(!scope.allows(&Url::parse("https://catlikecoding.com/other/").unwrap(), 1));
        assert!(!scope.allows(
            &Url::parse("https://example.com/unity/tutorials/").unwrap(),
            1
        ));
    }

    #[test]
    fn scope_respects_depth() {
        let config = Config::default();
        let seed = Url::parse("https://example.com/").unwrap();
        let scope = CrawlScope::new(&[seed], true, 1, &config).unwrap();
        assert!(scope.allows(&Url::parse("https://example.com/a").unwrap(), 1));
        assert!(!scope.allows(&Url::parse("https://example.com/b").unwrap(), 2));
    }

    #[test]
    fn captures_image_dimensions_from_bytes() {
        let png_1x1 = [
            137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82, 0, 0, 0, 1, 0, 0, 0, 1,
            8, 6, 0, 0, 0, 31, 21, 196, 137, 0, 0, 0, 13, 73, 68, 65, 84, 120, 156, 99, 248, 15, 4,
            0, 9, 251, 3, 253, 167, 161, 176, 219, 0, 0, 0, 0, 73, 69, 78, 68, 174, 66, 96, 130,
        ];
        let dimensions = image_dimensions(&png_1x1, Some("image/png")).unwrap();
        assert_eq!(dimensions.width, 1);
        assert_eq!(dimensions.height, 1);
    }

    #[test]
    fn progress_throttle_treats_fetch_processing_flips_as_hot() {
        assert!(is_hot_progress_transition(
            Some(ArchiveProgressPhase::Fetching),
            ArchiveProgressPhase::Processing
        ));
        assert!(is_hot_progress_transition(
            Some(ArchiveProgressPhase::Processing),
            ArchiveProgressPhase::Fetching
        ));
        assert!(!is_hot_progress_transition(
            Some(ArchiveProgressPhase::Discovering),
            ArchiveProgressPhase::Fetching
        ));
        assert!(!is_hot_progress_transition(
            Some(ArchiveProgressPhase::Fetching),
            ArchiveProgressPhase::Completed
        ));
    }

    #[test]
    fn extracts_urls_from_sitemap_xml() {
        let base = Url::parse("https://example.test/sitemap.xml").unwrap();
        let urls = sitemap_locs(
            &base,
            r#"<urlset><url><loc>https://example.test/a</loc></url><url><loc>/b</loc></url></urlset>"#,
        );
        assert_eq!(urls.len(), 2);
        assert!(urls
            .iter()
            .any(|url| url.as_str() == "https://example.test/a"));
        assert!(urls
            .iter()
            .any(|url| url.as_str() == "https://example.test/b"));
    }

    #[test]
    fn reads_gzip_sitemap_body() {
        use flate2::{write::GzEncoder, Compression};
        use std::io::Write;

        let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
        encoder.write_all(b"<urlset></urlset>").unwrap();
        let bytes = encoder.finish().unwrap();
        let body = sitemap_body(&bytes, Some("application/gzip"), "/sitemap.xml.gz").unwrap();
        assert_eq!(body, "<urlset></urlset>");
    }
}