use std::fs;
use std::io::{BufRead, BufReader, BufWriter, Read, Write};
use std::path::{Path, PathBuf};
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use url::Url;
use walkdir::WalkDir;
use crate::config::Config;
use crate::errors::{Result, SiteforgeError};
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ArchiveManifest {
pub archive_id: String,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
pub seeds: Vec<String>,
pub scope: ArchiveScopeSummary,
pub stats: CrawlStats,
pub config: ArchiveConfigSnapshot,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ArchiveScopeSummary {
pub full_site: bool,
pub same_domain: bool,
pub max_depth: usize,
pub include_url_patterns: Vec<String>,
pub exclude_url_patterns: Vec<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ArchiveConfigSnapshot {
pub user_agent: String,
pub concurrency: usize,
pub delay_ms: u64,
pub timeout_secs: u64,
pub retry_count: usize,
pub max_pages: usize,
pub max_asset_size_bytes: u64,
#[serde(default)]
pub max_total_archive_size_bytes: u64,
pub ocr_enabled: bool,
#[serde(default)]
pub render_js: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ChecksumManifest {
pub generated_at: DateTime<Utc>,
pub files: Vec<ChecksumEntry>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ChecksumEntry {
pub path: String,
pub bytes: u64,
pub blake3: String,
}
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
pub struct CrawlStats {
pub pages_discovered: usize,
pub pages_fetched: usize,
pub pages_parsed: usize,
pub pages_skipped: usize,
pub pages_failed: usize,
pub assets_discovered: usize,
pub assets_downloaded: usize,
pub assets_failed: usize,
pub ocr_attempted: usize,
pub ocr_succeeded: usize,
pub ocr_unavailable: usize,
}
#[derive(Debug, Clone)]
pub struct ArchiveLayout {
pub root: PathBuf,
pub archive_id: String,
}
impl ArchiveLayout {
pub fn new(root: PathBuf, archive_id: impl Into<String>) -> Self {
Self {
root,
archive_id: archive_id.into(),
}
}
pub fn base(&self) -> PathBuf {
self.root.join(&self.archive_id)
}
pub fn manifest(&self) -> PathBuf {
self.base().join("manifest.json")
}
pub fn database(&self) -> PathBuf {
self.base().join("crawl.db")
}
pub fn checksums(&self) -> PathBuf {
self.base().join("checksums.json")
}
pub fn raw_pages_dir(&self) -> PathBuf {
self.base().join("raw").join("pages")
}
pub fn rendered_pages_dir(&self) -> PathBuf {
self.base().join("raw").join("rendered")
}
pub fn raw_assets_dir(&self) -> PathBuf {
self.base().join("raw").join("assets")
}
pub fn readable_markdown_dir(&self) -> PathBuf {
self.base().join("readable").join("markdown")
}
pub fn readable_basic_markdown_dir(&self) -> PathBuf {
self.base().join("readable").join("basic_markdown")
}
pub fn readable_dir(&self) -> PathBuf {
self.base().join("readable")
}
pub fn agents_md(&self) -> PathBuf {
self.base().join("AGENTS.md")
}
pub fn agent_index_json(&self) -> PathBuf {
self.base().join("agent-index.json")
}
pub fn readable_json_dir(&self) -> PathBuf {
self.base().join("readable").join("json")
}
pub fn readable_text_dir(&self) -> PathBuf {
self.base().join("readable").join("text")
}
pub fn chunks_dir(&self) -> PathBuf {
self.base().join("chunks")
}
pub fn chunks_jsonl(&self) -> PathBuf {
self.chunks_dir().join("chunks.jsonl")
}
pub fn packs_dir(&self) -> PathBuf {
self.base().join("packs")
}
pub fn logs_dir(&self) -> PathBuf {
self.base().join("logs")
}
pub fn exports_dir(&self) -> PathBuf {
self.base().join("exports")
}
pub fn ensure(&self) -> Result<()> {
for path in [
self.raw_pages_dir(),
self.rendered_pages_dir(),
self.raw_assets_dir(),
self.readable_markdown_dir(),
self.readable_basic_markdown_dir(),
self.readable_json_dir(),
self.readable_text_dir(),
self.chunks_dir(),
self.packs_dir(),
self.logs_dir(),
self.exports_dir(),
] {
fs::create_dir_all(path)?;
}
Ok(())
}
}
pub fn new_manifest(
archive_id: String,
seeds: &[Url],
full_site: bool,
same_domain: bool,
max_depth: usize,
config: &Config,
) -> ArchiveManifest {
let now = Utc::now();
ArchiveManifest {
archive_id,
created_at: now,
updated_at: now,
seeds: seeds.iter().map(ToString::to_string).collect(),
scope: ArchiveScopeSummary {
full_site,
same_domain,
max_depth,
include_url_patterns: config.include_url_patterns.clone(),
exclude_url_patterns: config.exclude_url_patterns.clone(),
},
stats: CrawlStats::default(),
config: ArchiveConfigSnapshot {
user_agent: config.user_agent.clone(),
concurrency: config.default_concurrency,
delay_ms: config.default_delay_ms,
timeout_secs: config.timeout_secs,
retry_count: config.retry_count,
max_pages: config.max_pages,
max_asset_size_bytes: config.max_asset_size_bytes,
max_total_archive_size_bytes: config.max_total_archive_size_bytes,
ocr_enabled: config.ocr_enabled,
render_js: config.render_js,
},
}
}
pub fn write_manifest(layout: &ArchiveLayout, manifest: &ArchiveManifest) -> Result<()> {
let mut manifest = manifest.clone();
manifest.updated_at = Utc::now();
fs::write(layout.manifest(), serde_json::to_string_pretty(&manifest)?)?;
Ok(())
}
pub fn read_manifest(path: &Path) -> Result<ArchiveManifest> {
let raw = fs::read_to_string(path)?;
Ok(serde_json::from_str(&raw)?)
}
pub fn load_archive_manifest(config: &Config, archive_id: &str) -> Result<ArchiveManifest> {
let layout = ArchiveLayout::new(config.resolved_archive_root()?, archive_id.to_string());
let path = layout.manifest();
if !path.exists() {
return Err(SiteforgeError::ArchiveNotFound(archive_id.to_string()));
}
read_manifest(&path)
}
pub fn list_archives(config: &Config) -> Result<Vec<ArchiveManifest>> {
let root = config.resolved_archive_root()?;
if !root.exists() {
return Ok(Vec::new());
}
let mut archives = Vec::new();
for entry in fs::read_dir(root)? {
let entry = entry?;
let manifest_path = entry.path().join("manifest.json");
if manifest_path.exists() {
match read_manifest(&manifest_path) {
Ok(manifest) => archives.push(manifest),
Err(err) => {
tracing::warn!(path = %manifest_path.display(), error = %err, "skipping unreadable archive manifest")
}
}
}
}
archives.sort_by_key(|archive| std::cmp::Reverse(archive.updated_at));
Ok(archives)
}
pub fn archive_id_from_seed(seed: &Url) -> String {
let host = seed.host_str().unwrap_or("archive");
let mut slug = host
.trim_start_matches("www.")
.chars()
.map(|ch| if ch.is_ascii_alphanumeric() { ch } else { '-' })
.collect::<String>();
while slug.contains("--") {
slug = slug.replace("--", "-");
}
let timestamp = Utc::now().format("%Y%m%d%H%M%S");
format!("{}-{}", slug.trim_matches('-'), timestamp)
}
pub fn validate_archive_id(archive_id: &str) -> Result<()> {
let valid = !archive_id.is_empty()
&& archive_id.len() <= 128
&& archive_id != "."
&& archive_id != ".."
&& archive_id
.chars()
.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_' | '.'));
if valid {
Ok(())
} else {
Err(SiteforgeError::InvalidArchiveId(
"archive IDs must be 1-128 ASCII letters, numbers, '.', '_' or '-'".to_string(),
))
}
}
pub fn stable_id(input: &str) -> String {
blake3::hash(input.as_bytes()).to_hex()[..16].to_string()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn validates_archive_ids() {
assert!(validate_archive_id("catlikecoding").is_ok());
assert!(validate_archive_id("catlikecoding_2026.06").is_ok());
assert!(validate_archive_id("../bad").is_err());
assert!(validate_archive_id("").is_err());
}
}
pub fn archive_size_bytes(layout: &ArchiveLayout) -> Result<u64> {
if !layout.base().exists() {
return Ok(0);
}
let mut total = 0u64;
for entry in WalkDir::new(layout.base())
.into_iter()
.filter_map(std::result::Result::ok)
{
if entry.file_type().is_file() && !is_transient_archive_file(layout, entry.path()) {
total = total.saturating_add(fs::metadata(entry.path())?.len());
}
}
Ok(total)
}
pub fn ensure_archive_size_can_grow(
config: &Config,
layout: &ArchiveLayout,
additional_bytes: u64,
path: &Path,
) -> Result<()> {
let limit = config.max_total_archive_size_bytes;
if limit == 0 {
return Ok(());
}
let current = archive_size_bytes(layout)?;
let projected = current.saturating_add(additional_bytes);
if projected > limit {
return Err(SiteforgeError::SizeLimitExceeded {
path: path.display().to_string(),
size: projected,
limit,
});
}
Ok(())
}
pub fn write_checksums(layout: &ArchiveLayout) -> Result<ChecksumManifest> {
let manifest = generate_checksums(layout)?;
fs::write(layout.checksums(), serde_json::to_string_pretty(&manifest)?)?;
Ok(manifest)
}
pub fn write_checksums_streamed(layout: &ArchiveLayout) -> Result<()> {
fs::create_dir_all(layout.exports_dir())?;
let temp_path = layout.exports_dir().join(".checksums.json.tmp");
let _ = fs::remove_file(&temp_path);
let write_result = (|| -> Result<()> {
let mut writer = BufWriter::new(fs::File::create(&temp_path)?);
writer.write_all(b"{\n \"generated_at\": ")?;
serde_json::to_writer(&mut writer, &Utc::now())?;
writer.write_all(b",\n \"files\": [")?;
let mut first = true;
if layout.base().exists() {
for entry in WalkDir::new(layout.base())
.sort_by_file_name()
.into_iter()
.filter_map(std::result::Result::ok)
{
if !entry.file_type().is_file() || is_transient_archive_file(layout, entry.path()) {
continue;
}
let checksum = checksum_entry(layout, entry.path())?;
if !first {
writer.write_all(b",")?;
}
writer.write_all(b"\n ")?;
serde_json::to_writer(&mut writer, &checksum)?;
first = false;
}
}
writer.write_all(b"\n ]\n}\n")?;
writer.flush()?;
Ok(())
})();
if let Err(err) = write_result {
let _ = fs::remove_file(&temp_path);
return Err(err);
}
fs::rename(temp_path, layout.checksums())?;
Ok(())
}
pub fn generate_checksums(layout: &ArchiveLayout) -> Result<ChecksumManifest> {
let mut files = Vec::new();
if !layout.base().exists() {
return Ok(ChecksumManifest {
generated_at: Utc::now(),
files,
});
}
for entry in WalkDir::new(layout.base())
.into_iter()
.filter_map(std::result::Result::ok)
{
if !entry.file_type().is_file() || is_transient_archive_file(layout, entry.path()) {
continue;
}
let relative = archive_relative_path(layout, entry.path());
let (bytes, blake3) = hash_file(entry.path())?;
files.push(ChecksumEntry {
path: relative,
bytes,
blake3,
});
}
files.sort_by(|a, b| a.path.cmp(&b.path));
Ok(ChecksumManifest {
generated_at: Utc::now(),
files,
})
}
pub fn verify_archive(config: &Config, archive_id: &str) -> Result<Vec<String>> {
let layout = ArchiveLayout::new(config.resolved_archive_root()?, archive_id.to_string());
if !layout.base().exists() {
return Err(SiteforgeError::ArchiveNotFound(archive_id.to_string()));
}
let mut problems = Vec::new();
for required in [
layout.manifest(),
layout.database(),
layout.raw_pages_dir(),
layout.readable_markdown_dir(),
layout.readable_json_dir(),
layout.chunks_jsonl(),
layout.checksums(),
] {
if !required.exists() {
problems.push(format!("missing {}", required.display()));
}
}
if layout.checksums().exists() && !verify_checksums_streamed(&layout, &mut problems)? {
let raw = fs::read_to_string(layout.checksums())?;
let checksums: ChecksumManifest = serde_json::from_str(&raw)?;
for entry in checksums.files {
verify_checksum_entry(&layout, entry, &mut problems)?;
}
}
Ok(problems)
}
fn verify_checksums_streamed(layout: &ArchiveLayout, problems: &mut Vec<String>) -> Result<bool> {
let file = fs::File::open(layout.checksums())?;
let reader = BufReader::new(file);
let mut in_files = false;
let mut saw_files = false;
for line in reader.lines() {
let line = line?;
let trimmed = line.trim();
if !in_files {
let Some((_, after_start)) = trimmed.split_once("\"files\"") else {
continue;
};
let Some((_, after_bracket)) = after_start.split_once('[') else {
return Ok(false);
};
let after_bracket = after_bracket.trim();
if after_bracket.is_empty() {
in_files = true;
saw_files = true;
continue;
}
if after_bracket.starts_with(']') {
return Ok(true);
}
return Ok(false);
}
if trimmed.starts_with(']') {
return Ok(true);
}
if trimmed.is_empty() {
continue;
}
let json = trimmed.strip_suffix(',').unwrap_or(trimmed);
let Ok(entry) = serde_json::from_str::<ChecksumEntry>(json) else {
return Ok(false);
};
verify_checksum_entry(layout, entry, problems)?;
}
Ok(saw_files)
}
fn verify_checksum_entry(
layout: &ArchiveLayout,
entry: ChecksumEntry,
problems: &mut Vec<String>,
) -> Result<()> {
let path = layout.base().join(&entry.path);
if !path.exists() {
problems.push(format!("checksum entry missing file {}", entry.path));
return Ok(());
}
let (bytes, blake3) = hash_file(&path)?;
if bytes != entry.bytes {
problems.push(format!(
"size mismatch {}: expected {} bytes, got {} bytes",
entry.path, entry.bytes, bytes
));
}
if blake3 != entry.blake3 {
problems.push(format!("hash mismatch {}", entry.path));
}
Ok(())
}
fn archive_relative_path(layout: &ArchiveLayout, path: &Path) -> String {
path.strip_prefix(layout.base())
.unwrap_or(path)
.to_string_lossy()
.replace('\\', "/")
}
fn is_transient_archive_file(layout: &ArchiveLayout, path: &Path) -> bool {
if path == layout.checksums() {
return true;
}
let relative = archive_relative_path(layout, path);
relative.starts_with("exports/")
|| relative.ends_with(".db-wal")
|| relative.ends_with(".db-shm")
}
fn hash_file(path: &Path) -> Result<(u64, String)> {
let mut file = fs::File::open(path)?;
let mut hasher = blake3::Hasher::new();
let mut bytes = 0u64;
let mut buffer = [0u8; 16 * 1024];
loop {
let read = file.read(&mut buffer)?;
if read == 0 {
break;
}
bytes = bytes.saturating_add(read as u64);
hasher.update(&buffer[..read]);
}
Ok((bytes, hasher.finalize().to_hex().to_string()))
}
fn checksum_entry(layout: &ArchiveLayout, path: &Path) -> Result<ChecksumEntry> {
let relative = archive_relative_path(layout, path);
let (bytes, blake3) = hash_file(path)?;
Ok(ChecksumEntry {
path: relative,
bytes,
blake3,
})
}