siteforge 0.1.19

use std::fs;
use std::io::{BufRead, BufReader, BufWriter, Read, Write};
use std::path::{Component, Path, PathBuf};

use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use std::time::UNIX_EPOCH;
use url::Url;
use walkdir::WalkDir;

use crate::config::{ArchiveOutputProfile, Config};
use crate::errors::{Result, SiteforgeError};

const THESA_FORMAT_VERSION: &str = "1";
const THESA_METADATA_DIR: &str = ".thesa";

#[derive(Debug, Clone, Serialize, Deserialize)]
/// Top-level `manifest.json` stored in every siteforge archive.
pub struct ArchiveManifest {
    /// Archive ID and directory name.
    pub archive_id: String,
    /// Creation timestamp.
    pub created_at: DateTime<Utc>,
    /// Last manifest update timestamp.
    pub updated_at: DateTime<Utc>,
    /// Seed URLs that started the archive.
    pub seeds: Vec<String>,
    /// Crawl scope snapshot.
    pub scope: ArchiveScopeSummary,
    /// Crawl statistics snapshot.
    pub stats: CrawlStats,
    /// Configuration snapshot used by the archive.
    pub config: ArchiveConfigSnapshot,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
/// Scope values captured in an archive manifest.
pub struct ArchiveScopeSummary {
    /// Whether discovered links were followed.
    pub full_site: bool,
    /// Whether same-domain/path-prefix scoping was enforced.
    pub same_domain: bool,
    /// Maximum crawl depth used by the run.
    pub max_depth: usize,
    /// Include URL glob patterns used by the run.
    pub include_url_patterns: Vec<String>,
    /// Exclude URL glob patterns used by the run.
    pub exclude_url_patterns: Vec<String>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
/// Configuration values captured in an archive manifest for reproducibility.
pub struct ArchiveConfigSnapshot {
    /// Artifact output profile used by the run.
    #[serde(default)]
    pub archive_profile: ArchiveOutputProfile,
    /// User-Agent used by network/browser operations.
    pub user_agent: String,
    /// Concurrent page fetch count.
    pub concurrency: usize,
    /// Politeness delay in milliseconds.
    pub delay_ms: u64,
    /// Per-request timeout in seconds.
    pub timeout_secs: u64,
    /// Retry count.
    pub retry_count: usize,
    /// Maximum page count.
    pub max_pages: usize,
    /// Maximum page body size in bytes.
    #[serde(default = "default_max_page_size_bytes")]
    pub max_page_size_bytes: u64,
    /// Maximum asset body size in bytes.
    pub max_asset_size_bytes: u64,
    /// Maximum total archive size in bytes.
    #[serde(default)]
    pub max_total_archive_size_bytes: u64,
    /// Whether OCR was enabled.
    pub ocr_enabled: bool,
    /// Whether browser rendering was enabled.
    #[serde(default)]
    pub render_js: bool,
    /// Whether browser rendering was auto-limited to thin SPA-like pages.
    #[serde(default)]
    pub render_js_auto: bool,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
/// Result metadata returned after writing a thesa-compatible sidecar.
pub struct ThesaSidecarSummary {
    /// Path to `.thesa/manifest.json`.
    pub manifest_path: PathBuf,
    /// Path to `.thesa/checksums.blake3`.
    pub checksums_path: PathBuf,
    /// Number of files covered by the sidecar.
    pub file_count: usize,
    /// Total bytes covered by the sidecar.
    pub total_bytes: u64,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
/// Siteforge checksum manifest stored as `checksums.json`.
pub struct ChecksumManifest {
    /// Generation timestamp.
    pub generated_at: DateTime<Utc>,
    /// File checksum entries.
    pub files: Vec<ChecksumEntry>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
/// BLAKE3 checksum entry for an archive file.
pub struct ChecksumEntry {
    /// Path relative to the archive root.
    pub path: String,
    /// File size in bytes.
    pub bytes: u64,
    /// Lowercase hexadecimal BLAKE3 digest.
    pub blake3: String,
}

#[derive(Debug, Default, Clone, Serialize, Deserialize)]
/// Crawl and extraction counters.
pub struct CrawlStats {
    /// URLs discovered through seeds, sitemaps, or links.
    pub pages_discovered: usize,
    /// Page fetches attempted successfully enough to produce a response.
    pub pages_fetched: usize,
    /// HTML pages parsed and written.
    pub pages_parsed: usize,
    /// Pages skipped by content type, robots, or policy.
    pub pages_skipped: usize,
    /// Pages that failed permanently.
    pub pages_failed: usize,
    /// Asset URLs discovered while parsing pages.
    pub assets_discovered: usize,
    /// Assets downloaded successfully.
    pub assets_downloaded: usize,
    /// Assets that failed to download.
    pub assets_failed: usize,
    /// OCR attempts made for candidate assets.
    pub ocr_attempted: usize,
    /// OCR attempts that produced usable text.
    pub ocr_succeeded: usize,
    /// OCR attempts skipped because the backend was unavailable.
    pub ocr_unavailable: usize,
}

#[derive(Debug, Clone)]
/// Path helper for a single archive directory.
pub struct ArchiveLayout {
    /// Archive root containing archive directories.
    pub root: PathBuf,
    /// Archive ID and directory name.
    pub archive_id: String,
}

impl ArchiveLayout {
    /// Construct a layout from an archive root and ID.
    pub fn new(root: PathBuf, archive_id: impl Into<String>) -> Self {
        Self {
            root,
            archive_id: archive_id.into(),
        }
    }

    /// Archive base directory.
    pub fn base(&self) -> PathBuf {
        self.root.join(&self.archive_id)
    }

    /// Path to `manifest.json`.
    pub fn manifest(&self) -> PathBuf {
        self.base().join("manifest.json")
    }

    /// Path to compact crawl metadata SQLite database.
    pub fn database(&self) -> PathBuf {
        self.base().join("crawl.db")
    }

    /// Path to siteforge `checksums.json`.
    pub fn checksums(&self) -> PathBuf {
        self.base().join("checksums.json")
    }

    /// Path to `.thesa/` sidecar metadata directory.
    pub fn thesa_dir(&self) -> PathBuf {
        self.base().join(".thesa")
    }

    /// Path to `.thesa/manifest.json`.
    pub fn thesa_manifest(&self) -> PathBuf {
        self.thesa_dir().join("manifest.json")
    }

    /// Path to `.thesa/checksums.blake3`.
    pub fn thesa_checksums(&self) -> PathBuf {
        self.thesa_dir().join("checksums.blake3")
    }

    /// Directory for raw fetched HTML pages.
    pub fn raw_pages_dir(&self) -> PathBuf {
        self.base().join("raw").join("pages")
    }

    /// Directory for browser-rendered DOM captures.
    pub fn rendered_pages_dir(&self) -> PathBuf {
        self.base().join("raw").join("rendered")
    }

    /// Directory for downloaded raw assets.
    pub fn raw_assets_dir(&self) -> PathBuf {
        self.base().join("raw").join("assets")
    }

    /// Directory for YAML-frontmatter Markdown.
    pub fn readable_markdown_dir(&self) -> PathBuf {
        self.base().join("readable").join("markdown")
    }

    /// Directory for no-frontmatter Markdown optimized for agents.
    pub fn readable_basic_markdown_dir(&self) -> PathBuf {
        self.base().join("readable").join("basic_markdown")
    }

    /// Readable artifact directory.
    pub fn readable_dir(&self) -> PathBuf {
        self.base().join("readable")
    }

    /// Path to top-level agent guidance.
    pub fn agents_md(&self) -> PathBuf {
        self.base().join("AGENTS.md")
    }

    /// Path to machine-readable agent index.
    pub fn agent_index_json(&self) -> PathBuf {
        self.base().join("agent-index.json")
    }

    /// Directory for structured page JSON.
    pub fn readable_json_dir(&self) -> PathBuf {
        self.base().join("readable").join("json")
    }

    /// Directory for plain text page extracts.
    pub fn readable_text_dir(&self) -> PathBuf {
        self.base().join("readable").join("text")
    }

    /// Directory for semantic chunk artifacts.
    pub fn chunks_dir(&self) -> PathBuf {
        self.base().join("chunks")
    }

    /// Path to newline-delimited semantic chunks.
    pub fn chunks_jsonl(&self) -> PathBuf {
        self.chunks_dir().join("chunks.jsonl")
    }

    /// Directory for generated AI context packs.
    pub fn packs_dir(&self) -> PathBuf {
        self.base().join("packs")
    }

    /// Directory for persisted bounded crawl logs.
    pub fn logs_dir(&self) -> PathBuf {
        self.base().join("logs")
    }

    /// Directory for derived export outputs.
    pub fn exports_dir(&self) -> PathBuf {
        self.base().join("exports")
    }

    /// Ensure the archive directory tree exists.
    pub fn ensure(&self) -> Result<()> {
        for path in [
            self.raw_pages_dir(),
            self.rendered_pages_dir(),
            self.raw_assets_dir(),
            self.readable_markdown_dir(),
            self.readable_basic_markdown_dir(),
            self.readable_json_dir(),
            self.readable_text_dir(),
            self.chunks_dir(),
            self.packs_dir(),
            self.logs_dir(),
            self.exports_dir(),
        ] {
            fs::create_dir_all(path)?;
        }
        Ok(())
    }
}

pub fn new_manifest(
    archive_id: String,
    seeds: &[Url],
    full_site: bool,
    same_domain: bool,
    max_depth: usize,
    config: &Config,
) -> ArchiveManifest {
    let now = Utc::now();
    ArchiveManifest {
        archive_id,
        created_at: now,
        updated_at: now,
        seeds: seeds.iter().map(ToString::to_string).collect(),
        scope: ArchiveScopeSummary {
            full_site,
            same_domain,
            max_depth,
            include_url_patterns: config.include_url_patterns.clone(),
            exclude_url_patterns: config.exclude_url_patterns.clone(),
        },
        stats: CrawlStats::default(),
        config: ArchiveConfigSnapshot {
            archive_profile: config.archive_profile,
            user_agent: config.user_agent.clone(),
            concurrency: config.default_concurrency,
            delay_ms: config.default_delay_ms,
            timeout_secs: config.timeout_secs,
            retry_count: config.retry_count,
            max_pages: config.max_pages,
            max_page_size_bytes: config.max_page_size_bytes,
            max_asset_size_bytes: config.max_asset_size_bytes,
            max_total_archive_size_bytes: config.max_total_archive_size_bytes,
            ocr_enabled: config.ocr_enabled,
            render_js: config.render_js,
            render_js_auto: config.render_js_auto,
        },
    }
}

fn default_max_page_size_bytes() -> u64 {
    Config::default().max_page_size_bytes
}

pub fn write_manifest(layout: &ArchiveLayout, manifest: &ArchiveManifest) -> Result<()> {
    let mut manifest = manifest.clone();
    manifest.updated_at = Utc::now();
    fs::write(layout.manifest(), serde_json::to_string_pretty(&manifest)?)?;
    Ok(())
}

pub fn read_manifest(path: &Path) -> Result<ArchiveManifest> {
    let raw = fs::read_to_string(path)?;
    Ok(serde_json::from_str(&raw)?)
}

pub fn load_archive_manifest(config: &Config, archive_id: &str) -> Result<ArchiveManifest> {
    let layout = ArchiveLayout::new(config.resolved_archive_root()?, archive_id.to_string());
    let path = layout.manifest();
    if !path.exists() {
        return Err(SiteforgeError::ArchiveNotFound(archive_id.to_string()));
    }
    read_manifest(&path)
}

pub fn list_archives(config: &Config) -> Result<Vec<ArchiveManifest>> {
    let root = config.resolved_archive_root()?;
    if !root.exists() {
        return Ok(Vec::new());
    }

    let mut archives = Vec::new();
    for entry in fs::read_dir(root)? {
        let entry = entry?;
        let manifest_path = entry.path().join("manifest.json");
        if manifest_path.exists() {
            match read_manifest(&manifest_path) {
                Ok(manifest) => archives.push(manifest),
                Err(err) => {
                    tracing::warn!(path = %manifest_path.display(), error = %err, "skipping unreadable archive manifest")
                }
            }
        }
    }
    archives.sort_by_key(|archive| std::cmp::Reverse(archive.updated_at));
    Ok(archives)
}

pub fn archive_id_from_seed(seed: &Url) -> String {
    let host = seed.host_str().unwrap_or("archive");
    let mut slug = host
        .trim_start_matches("www.")
        .chars()
        .map(|ch| if ch.is_ascii_alphanumeric() { ch } else { '-' })
        .collect::<String>();
    while slug.contains("--") {
        slug = slug.replace("--", "-");
    }
    let timestamp = Utc::now().format("%Y%m%d%H%M%S");
    format!("{}-{}", slug.trim_matches('-'), timestamp)
}

pub fn validate_archive_id(archive_id: &str) -> Result<()> {
    let valid = !archive_id.is_empty()
        && archive_id.len() <= 128
        && archive_id != "."
        && archive_id != ".."
        && archive_id
            .chars()
            .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_' | '.'));
    if valid {
        Ok(())
    } else {
        Err(SiteforgeError::InvalidArchiveId(
            "archive IDs must be 1-128 ASCII letters, numbers, '.', '_' or '-'".to_string(),
        ))
    }
}

pub fn stable_id(input: &str) -> String {
    blake3::hash(input.as_bytes()).to_hex()[..16].to_string()
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn validates_archive_ids() {
        assert!(validate_archive_id("catlikecoding").is_ok());
        assert!(validate_archive_id("catlikecoding_2026.06").is_ok());
        assert!(validate_archive_id("../bad").is_err());
        assert!(validate_archive_id("").is_err());
    }
}

pub fn archive_size_bytes(layout: &ArchiveLayout) -> Result<u64> {
    if !layout.base().exists() {
        return Ok(0);
    }
    let mut total = 0u64;
    for entry in WalkDir::new(layout.base())
        .into_iter()
        .filter_map(std::result::Result::ok)
    {
        if entry.file_type().is_file() && !is_transient_archive_file(layout, entry.path()) {
            total = total.saturating_add(fs::metadata(entry.path())?.len());
        }
    }
    Ok(total)
}

pub fn ensure_archive_size_can_grow(
    config: &Config,
    layout: &ArchiveLayout,
    additional_bytes: u64,
    path: &Path,
) -> Result<()> {
    let limit = config.max_total_archive_size_bytes;
    if limit == 0 {
        return Ok(());
    }
    let current = archive_size_bytes(layout)?;
    let projected = current.saturating_add(additional_bytes);
    if projected > limit {
        return Err(SiteforgeError::SizeLimitExceeded {
            path: path.display().to_string(),
            size: projected,
            limit,
        });
    }
    Ok(())
}

pub struct ArchiveWriteBudget {
    limit: u64,
    current: u64,
}

impl ArchiveWriteBudget {
    pub fn new(config: &Config, layout: &ArchiveLayout) -> Result<Self> {
        Ok(Self {
            limit: config.max_total_archive_size_bytes,
            current: archive_size_bytes(layout)?,
        })
    }

    pub fn reserve(&mut self, additional_bytes: u64, path: &Path) -> Result<()> {
        if self.limit == 0 {
            self.current = self.current.saturating_add(additional_bytes);
            return Ok(());
        }
        let projected = self.current.saturating_add(additional_bytes);
        if projected > self.limit {
            return Err(SiteforgeError::SizeLimitExceeded {
                path: path.display().to_string(),
                size: projected,
                limit: self.limit,
            });
        }
        self.current = projected;
        Ok(())
    }
}

pub fn write_checksums(layout: &ArchiveLayout) -> Result<ChecksumManifest> {
    let manifest = generate_checksums(layout)?;
    fs::write(layout.checksums(), serde_json::to_string_pretty(&manifest)?)?;
    Ok(manifest)
}

pub fn write_checksums_streamed(layout: &ArchiveLayout) -> Result<()> {
    fs::create_dir_all(layout.exports_dir())?;
    let temp_path = layout.exports_dir().join(".checksums.json.tmp");
    let _ = fs::remove_file(&temp_path);
    let write_result = (|| -> Result<()> {
        let mut writer = BufWriter::new(fs::File::create(&temp_path)?);
        writer.write_all(b"{\n  \"generated_at\": ")?;
        serde_json::to_writer(&mut writer, &Utc::now())?;
        writer.write_all(b",\n  \"files\": [")?;
        let mut first = true;
        if layout.base().exists() {
            for entry in WalkDir::new(layout.base())
                .sort_by_file_name()
                .into_iter()
                .filter_map(std::result::Result::ok)
            {
                if !entry.file_type().is_file() || is_transient_archive_file(layout, entry.path()) {
                    continue;
                }
                let checksum = checksum_entry(layout, entry.path())?;
                if !first {
                    writer.write_all(b",")?;
                }
                writer.write_all(b"\n    ")?;
                serde_json::to_writer(&mut writer, &checksum)?;
                first = false;
            }
        }
        writer.write_all(b"\n  ]\n}\n")?;
        writer.flush()?;
        Ok(())
    })();

    if let Err(err) = write_result {
        let _ = fs::remove_file(&temp_path);
        return Err(err);
    }
    fs::rename(temp_path, layout.checksums())?;
    Ok(())
}

pub fn write_thesa_sidecar(
    layout: &ArchiveLayout,
    manifest: &ArchiveManifest,
    complete: bool,
) -> Result<ThesaSidecarSummary> {
    fs::create_dir_all(layout.thesa_dir())?;
    let entries_path = layout.thesa_dir().join("manifest.entries.tmp");
    let manifest_tmp_path = layout.thesa_dir().join("manifest.json.tmp");
    let checksums_tmp_path = layout.thesa_dir().join("checksums.blake3.tmp");
    let _ = fs::remove_file(&entries_path);
    let _ = fs::remove_file(&manifest_tmp_path);
    let _ = fs::remove_file(&checksums_tmp_path);

    let write_result = (|| -> Result<(usize, u64)> {
        let (file_count, total_bytes) =
            write_thesa_file_entries(layout, &entries_path, &checksums_tmp_path)?;
        write_thesa_manifest(
            &manifest_tmp_path,
            &entries_path,
            manifest,
            complete,
            file_count,
            total_bytes,
        )?;
        Ok((file_count, total_bytes))
    })();

    let (file_count, total_bytes) = match write_result {
        Ok(summary) => summary,
        Err(err) => {
            let _ = fs::remove_file(&entries_path);
            let _ = fs::remove_file(&manifest_tmp_path);
            let _ = fs::remove_file(&checksums_tmp_path);
            return Err(err);
        }
    };

    fs::rename(&manifest_tmp_path, layout.thesa_manifest())?;
    fs::rename(&checksums_tmp_path, layout.thesa_checksums())?;
    let _ = fs::remove_file(&entries_path);

    Ok(ThesaSidecarSummary {
        manifest_path: layout.thesa_manifest(),
        checksums_path: layout.thesa_checksums(),
        file_count,
        total_bytes,
    })
}

#[derive(Serialize)]
struct ThesaManifestTool<'a> {
    name: &'a str,
    version: &'a str,
}

#[derive(Serialize)]
struct ThesaManifestSource<'a> {
    platform: &'a str,
    target: &'a str,
}

#[derive(Serialize)]
struct ThesaManifestArchive {
    created_at_unix: u64,
    complete: bool,
    file_count: usize,
    total_bytes: u64,
    checksum_algorithm: &'static str,
}

#[derive(Serialize, Deserialize)]
struct ThesaManifestFile {
    path: String,
    size: u64,
    #[serde(default)]
    modified_unix_nanos: Option<u64>,
    blake3: String,
}

fn write_thesa_file_entries(
    layout: &ArchiveLayout,
    entries_path: &Path,
    checksums_path: &Path,
) -> Result<(usize, u64)> {
    let mut entries = BufWriter::new(fs::File::create(entries_path)?);
    let mut checksums = BufWriter::new(fs::File::create(checksums_path)?);
    let mut file_count = 0usize;
    let mut total_bytes = 0u64;

    if layout.base().exists() {
        for entry in WalkDir::new(layout.base())
            .sort_by_file_name()
            .into_iter()
            .filter_map(std::result::Result::ok)
        {
            if !entry.file_type().is_file() || is_transient_archive_file(layout, entry.path()) {
                continue;
            }
            let metadata = fs::metadata(entry.path())?;
            let path = archive_relative_path(layout, entry.path());
            validate_thesa_manifest_path(&path)?;
            let (size, blake3) = hash_file(entry.path())?;
            let file = ThesaManifestFile {
                path,
                size,
                modified_unix_nanos: metadata_modified_unix_nanos(&metadata),
                blake3,
            };

            serde_json::to_writer(&mut entries, &file)?;
            entries.write_all(b"\n")?;
            checksums.write_all(file.blake3.as_bytes())?;
            checksums.write_all(b"  ")?;
            checksums.write_all(file.path.as_bytes())?;
            checksums.write_all(b"\n")?;
            file_count = file_count.saturating_add(1);
            total_bytes = total_bytes.saturating_add(size);
        }
    }

    entries.flush()?;
    checksums.flush()?;
    Ok((file_count, total_bytes))
}

fn write_thesa_manifest(
    output_path: &Path,
    entries_path: &Path,
    manifest: &ArchiveManifest,
    complete: bool,
    file_count: usize,
    total_bytes: u64,
) -> Result<()> {
    let mut writer = BufWriter::new(fs::File::create(output_path)?);
    writer.write_all(b"{\"format_version\":")?;
    serde_json::to_writer(&mut writer, THESA_FORMAT_VERSION)?;
    writer.write_all(b",\"tool\":")?;
    serde_json::to_writer(
        &mut writer,
        &ThesaManifestTool {
            name: "siteforge",
            version: env!("CARGO_PKG_VERSION"),
        },
    )?;
    writer.write_all(b",\"source\":")?;
    let target = thesa_source_target(manifest);
    serde_json::to_writer(
        &mut writer,
        &ThesaManifestSource {
            platform: "siteforge",
            target: &target,
        },
    )?;
    writer.write_all(b",\"archive\":")?;
    serde_json::to_writer(
        &mut writer,
        &ThesaManifestArchive {
            created_at_unix: manifest_created_at_unix(manifest),
            complete,
            file_count,
            total_bytes,
            checksum_algorithm: "blake3",
        },
    )?;
    writer.write_all(b",\"files\":[")?;

    let file = fs::File::open(entries_path)?;
    let reader = BufReader::new(file);
    let mut first = true;
    for line in reader.lines() {
        let line = line?;
        if line.trim().is_empty() {
            continue;
        }
        if !first {
            writer.write_all(b",")?;
        }
        writer.write_all(line.as_bytes())?;
        first = false;
    }
    writer.write_all(b"]}")?;
    writer.flush()?;
    Ok(())
}

fn thesa_source_target(manifest: &ArchiveManifest) -> String {
    if manifest.seeds.is_empty() {
        manifest.archive_id.clone()
    } else {
        manifest.seeds.join(", ")
    }
}

fn manifest_created_at_unix(manifest: &ArchiveManifest) -> u64 {
    u64::try_from(manifest.created_at.timestamp()).unwrap_or_default()
}

fn metadata_modified_unix_nanos(metadata: &fs::Metadata) -> Option<u64> {
    metadata
        .modified()
        .ok()
        .and_then(|time| time.duration_since(UNIX_EPOCH).ok())
        .and_then(|duration| u64::try_from(duration.as_nanos()).ok())
}

fn validate_thesa_manifest_path(path: &str) -> Result<()> {
    if path.is_empty() {
        return Err(SiteforgeError::message("thesa manifest path is empty"));
    }
    if path.contains('\\') {
        return Err(SiteforgeError::message(format!(
            "thesa manifest path contains a backslash: {path}"
        )));
    }

    let path_ref = Path::new(path);
    if path_ref.is_absolute() {
        return Err(SiteforgeError::message(format!(
            "thesa manifest path is absolute: {path}"
        )));
    }

    let mut has_component = false;
    for (index, component) in path_ref.components().enumerate() {
        match component {
            Component::Normal(value) => {
                if index == 0 && value == std::ffi::OsStr::new(THESA_METADATA_DIR) {
                    return Err(SiteforgeError::message(format!(
                        "thesa manifest path targets metadata: {path}"
                    )));
                }
                has_component = true;
            }
            Component::CurDir
            | Component::ParentDir
            | Component::RootDir
            | Component::Prefix(_) => {
                return Err(SiteforgeError::message(format!(
                    "unsafe thesa manifest path: {path}"
                )));
            }
        }
    }
    if !has_component {
        return Err(SiteforgeError::message(
            "thesa manifest path has no component",
        ));
    }
    Ok(())
}

pub fn generate_checksums(layout: &ArchiveLayout) -> Result<ChecksumManifest> {
    let mut files = Vec::new();
    if !layout.base().exists() {
        return Ok(ChecksumManifest {
            generated_at: Utc::now(),
            files,
        });
    }

    for entry in WalkDir::new(layout.base())
        .into_iter()
        .filter_map(std::result::Result::ok)
    {
        if !entry.file_type().is_file() || is_transient_archive_file(layout, entry.path()) {
            continue;
        }
        let relative = archive_relative_path(layout, entry.path());
        let (bytes, blake3) = hash_file(entry.path())?;
        files.push(ChecksumEntry {
            path: relative,
            bytes,
            blake3,
        });
    }
    files.sort_by(|a, b| a.path.cmp(&b.path));
    Ok(ChecksumManifest {
        generated_at: Utc::now(),
        files,
    })
}

pub fn verify_archive(config: &Config, archive_id: &str) -> Result<Vec<String>> {
    let layout = ArchiveLayout::new(config.resolved_archive_root()?, archive_id.to_string());
    if !layout.base().exists() {
        return Err(SiteforgeError::ArchiveNotFound(archive_id.to_string()));
    }

    let mut problems = Vec::new();
    for required in [
        layout.manifest(),
        layout.database(),
        layout.raw_pages_dir(),
        layout.readable_markdown_dir(),
        layout.readable_json_dir(),
        layout.chunks_jsonl(),
        layout.checksums(),
    ] {
        if !required.exists() {
            problems.push(format!("missing {}", required.display()));
        }
    }

    if layout.checksums().exists() && !verify_checksums_streamed(&layout, &mut problems)? {
        let raw = fs::read_to_string(layout.checksums())?;
        let checksums: ChecksumManifest = serde_json::from_str(&raw)?;
        for entry in checksums.files {
            verify_checksum_entry(&layout, entry, &mut problems)?;
        }
    }

    Ok(problems)
}

fn verify_checksums_streamed(layout: &ArchiveLayout, problems: &mut Vec<String>) -> Result<bool> {
    let file = fs::File::open(layout.checksums())?;
    let reader = BufReader::new(file);
    let mut in_files = false;
    let mut saw_files = false;
    for line in reader.lines() {
        let line = line?;
        let trimmed = line.trim();
        if !in_files {
            let Some((_, after_start)) = trimmed.split_once("\"files\"") else {
                continue;
            };
            let Some((_, after_bracket)) = after_start.split_once('[') else {
                return Ok(false);
            };
            let after_bracket = after_bracket.trim();
            if after_bracket.is_empty() {
                in_files = true;
                saw_files = true;
                continue;
            }
            if after_bracket.starts_with(']') {
                return Ok(true);
            }
            return Ok(false);
        }

        if trimmed.starts_with(']') {
            return Ok(true);
        }
        if trimmed.is_empty() {
            continue;
        }
        let json = trimmed.strip_suffix(',').unwrap_or(trimmed);
        let Ok(entry) = serde_json::from_str::<ChecksumEntry>(json) else {
            return Ok(false);
        };
        verify_checksum_entry(layout, entry, problems)?;
    }
    Ok(saw_files)
}

fn verify_checksum_entry(
    layout: &ArchiveLayout,
    entry: ChecksumEntry,
    problems: &mut Vec<String>,
) -> Result<()> {
    let path = layout.base().join(&entry.path);
    if !path.exists() {
        problems.push(format!("checksum entry missing file {}", entry.path));
        return Ok(());
    }
    let (bytes, blake3) = hash_file(&path)?;
    if bytes != entry.bytes {
        problems.push(format!(
            "size mismatch {}: expected {} bytes, got {} bytes",
            entry.path, entry.bytes, bytes
        ));
    }
    if blake3 != entry.blake3 {
        problems.push(format!("hash mismatch {}", entry.path));
    }
    Ok(())
}

fn archive_relative_path(layout: &ArchiveLayout, path: &Path) -> String {
    path.strip_prefix(layout.base())
        .unwrap_or(path)
        .to_string_lossy()
        .replace('\\', "/")
}

fn is_transient_archive_file(layout: &ArchiveLayout, path: &Path) -> bool {
    if path == layout.checksums() {
        return true;
    }
    let relative = archive_relative_path(layout, path);
    relative.starts_with("exports/")
        || relative.starts_with(".thesa/")
        || relative.ends_with(".db-wal")
        || relative.ends_with(".db-shm")
}

fn hash_file(path: &Path) -> Result<(u64, String)> {
    let mut file = fs::File::open(path)?;
    let mut hasher = blake3::Hasher::new();
    let mut bytes = 0u64;
    let mut buffer = [0u8; 16 * 1024];
    loop {
        let read = file.read(&mut buffer)?;
        if read == 0 {
            break;
        }
        bytes = bytes.saturating_add(read as u64);
        hasher.update(&buffer[..read]);
    }
    Ok((bytes, hasher.finalize().to_hex().to_string()))
}

fn checksum_entry(layout: &ArchiveLayout, path: &Path) -> Result<ChecksumEntry> {
    let relative = archive_relative_path(layout, path);
    let (bytes, blake3) = hash_file(path)?;
    Ok(ChecksumEntry {
        path: relative,
        bytes,
        blake3,
    })
}