use std::collections::HashMap;
use std::fs;
use std::io::{Read, Write};
use std::net::{SocketAddr, TcpListener, TcpStream};
use std::path::Path;
use std::process::Command as ProcessCommand;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::Arc;
use std::thread;
use std::time::Duration;
use flate2::read::GzDecoder;
use siteforge::archive::{
ensure_archive_size_can_grow, new_manifest, verify_archive, write_checksums,
write_checksums_streamed, write_manifest, write_thesa_sidecar, ArchiveLayout,
};
use siteforge::config::{ArchiveOutputProfile, Config};
use siteforge::crawler::{run_archive, ArchiveRunOptions};
use siteforge::export::{create_ai_packs, export_archive, ExportFormat};
use siteforge::ocr::{OcrBackend, TesseractOcr};
use siteforge::parse::{normalize_link, parse_html, BlockType};
use siteforge::search::search_archives;
use siteforge::storage::Storage;
use siteforge::{ArchiveSummary, CrawlStats, Siteforge};
use tempfile::tempdir;
use url::Url;
#[test]
fn html_to_markdown_extracts_main_content() {
let html = include_str!("fixtures/normal_article.html");
let url = Url::parse("https://example.test/articles/normal").unwrap();
let page = parse_html("archive", "page", &url, html, "hash", 0);
let markdown = page.to_markdown().unwrap();
assert!(markdown.contains("title: Normal Article"));
assert!(markdown.contains("# Normal Article"));
assert!(markdown.contains("mesh deformation"));
assert!(!markdown.contains("Global navigation"));
assert!(markdown.contains("Archived from: https://example.test/articles/normal"));
}
#[test]
fn binary_basic_help_and_list_work() {
let binary = env!("CARGO_BIN_EXE_siteforge");
let help = ProcessCommand::new(binary).arg("--help").output().unwrap();
assert!(help.status.success());
let help_text = String::from_utf8_lossy(&help.stdout);
assert!(help_text.contains("archive"));
assert!(help_text.contains("tui"));
let archive_help = ProcessCommand::new(binary)
.args(["archive", "--help"])
.output()
.unwrap();
assert!(archive_help.status.success());
let archive_help_text = String::from_utf8_lossy(&archive_help.stdout);
assert!(archive_help_text.contains("--concurrency"));
assert!(archive_help_text.contains("--render-js"));
assert!(archive_help_text.contains("--render-js-auto"));
assert!(archive_help_text.contains("--archive-profile"));
let temp = tempdir().unwrap();
let config_path = temp.path().join("config.yaml");
fs::write(
&config_path,
format!("archive_root: {}\n", temp.path().join("archives").display()),
)
.unwrap();
let list = ProcessCommand::new(binary)
.args(["--config", config_path.to_str().unwrap(), "list"])
.output()
.unwrap();
assert!(list.status.success());
assert!(String::from_utf8_lossy(&list.stdout).contains("No archives found"));
}
#[test]
fn structured_json_schema_contains_expected_fields() {
let html = include_str!("fixtures/code_tutorial.html");
let url = Url::parse("https://example.test/tutorials/mesh").unwrap();
let page = parse_html("archive", "page", &url, html, "hash", 1);
let json = serde_json::to_value(page.to_page_json()).unwrap();
assert_eq!(json["archive_id"], "archive");
assert_eq!(json["page_id"], "page");
assert_eq!(json["title"], "Code Tutorial");
assert!(json["blocks"]
.as_array()
.unwrap()
.iter()
.any(|block| block["type"] == "code"));
assert!(json["code_blocks"].as_array().unwrap().len() == 1);
}
#[test]
fn metadata_json_ld_and_basic_markdown_are_agent_readable() {
let html = r#"
<html lang="en">
<head>
<title>Agent Page</title>
<meta name="description" content="Agent readable description">
<meta property="og:title" content="Open Graph Title">
<link rel="alternate" type="application/rss+xml" href="/feed.xml" title="Feed">
<script type="application/ld+json">{"@context":"https://schema.org","@type":"Article","headline":"Agent Page"}</script>
</head>
<body><main><h1>Agent Page</h1><p>Useful content.</p><a href="/docs">Docs</a></main></body>
</html>
"#;
let url = Url::parse("https://example.test/agent/page").unwrap();
let page = parse_html("archive", "page", &url, html, "hash", 0);
let json = serde_json::to_value(page.to_page_json()).unwrap();
let basic = page.to_basic_markdown();
assert_eq!(
page.metadata.description.as_deref(),
Some("Agent readable description")
);
assert_eq!(page.metadata.json_ld.len(), 1);
assert_eq!(json["metadata"]["json_ld"][0]["parsed"]["@type"], "Article");
assert!(basic.contains("Description: Agent readable description"));
assert!(basic.contains("## Page Data"));
assert!(basic.contains("### JSON-LD Structured Data"));
assert!(basic.contains("[Docs](https://example.test/docs)"));
}
#[test]
fn code_blocks_are_preserved_and_not_split_into_chunks() {
let html = include_str!("fixtures/code_tutorial.html");
let url = Url::parse("https://example.test/tutorials/mesh").unwrap();
let page = parse_html("archive", "page", &url, html, "hash", 0);
let code = page
.blocks
.iter()
.find(|block| block.block_type == BlockType::Code)
.unwrap();
assert!(code.text.contains(" void Update()"));
assert_eq!(code.language.as_deref(), Some("csharp"));
let code_chunks = page
.chunks()
.into_iter()
.filter(|chunk| chunk.content_type == "code")
.collect::<Vec<_>>();
assert_eq!(code_chunks.len(), 1);
assert!(code_chunks[0].text.contains("Debug.Log"));
}
#[test]
fn asset_path_rewriting_updates_markdown() {
let html = include_str!("fixtures/page_with_images.html");
let url = Url::parse("https://example.test/tutorial/").unwrap();
let mut page = parse_html("archive", "page", &url, html, "hash", 0);
let asset_id = page.assets[0].asset_id.clone();
page.rewrite_asset_local_paths(&HashMap::from([(
asset_id,
"../../raw/assets/code-screenshot.png".to_string(),
)]));
let markdown = page.to_markdown().unwrap();
assert!(markdown.contains("../../raw/assets/code-screenshot.png"));
assert!(markdown.contains("Code shown in an image"));
}
#[test]
fn rendered_lazy_and_srcset_images_are_cataloged() {
let html = r#"
<article>
<h1>Rendered Assets</h1>
<picture>
<source srcset="images/diagram-small.webp 1x, images/diagram-large.webp 2x">
<img data-src="images/lazy-code.png" srcset="images/code-1x.png 1x, images/code-2x.png 2x" alt="rendered code">
</picture>
</article>
"#;
let url = Url::parse("https://example.test/tutorial/").unwrap();
let page = parse_html("archive", "page", &url, html, "hash", 0);
let asset_urls = page
.assets
.iter()
.map(|asset| asset.url.as_str())
.collect::<Vec<_>>();
assert!(asset_urls.contains(&"https://example.test/tutorial/images/lazy-code.png"));
assert!(asset_urls.contains(&"https://example.test/tutorial/images/code-1x.png"));
assert!(asset_urls.contains(&"https://example.test/tutorial/images/code-2x.png"));
assert!(asset_urls.contains(&"https://example.test/tutorial/images/diagram-small.webp"));
assert!(asset_urls.contains(&"https://example.test/tutorial/images/diagram-large.webp"));
}
#[test]
fn external_page_resources_are_cataloged_as_assets() {
let html = r#"
<html>
<head>
<script src="/static/app.mjs" type="module"></script>
<link rel="stylesheet" href="/static/site.css">
<link rel="manifest" href="/app.webmanifest">
<link rel="alternate" type="application/rss+xml" href="/feed.xml">
</head>
<body>
<article><h1>Resources</h1><p>Page with external resources.</p></article>
<video src="/media/demo.mp4"><track src="/media/captions.vtt"></video>
<iframe src="/embed/widget.html"></iframe>
<object data="/downloads/example.wasm"></object>
</body>
</html>
"#;
let url = Url::parse("https://example.test/tutorial/").unwrap();
let page = parse_html("archive", "page", &url, html, "hash", 0);
let asset_urls = page
.assets
.iter()
.map(|asset| asset.url.as_str())
.collect::<Vec<_>>();
for expected in [
"https://example.test/static/app.mjs",
"https://example.test/static/site.css",
"https://example.test/app.webmanifest",
"https://example.test/feed.xml",
"https://example.test/media/demo.mp4",
"https://example.test/media/captions.vtt",
"https://example.test/embed/widget.html",
"https://example.test/downloads/example.wasm",
] {
assert!(asset_urls.contains(&expected), "missing {expected}");
}
}
#[test]
fn url_normalization_resolves_relative_links() {
let base = Url::parse("https://example.test/unity/tutorials/advanced/").unwrap();
let url = normalize_link(&base, "../basics/?b=2&a=1#intro").unwrap();
assert_eq!(
url.as_str(),
"https://example.test/unity/tutorials/basics/?a=1&b=2"
);
}
#[test]
fn search_index_returns_page_matches() {
let temp = tempdir().unwrap();
let layout = ArchiveLayout::new(temp.path().to_path_buf(), "archive");
layout.ensure().unwrap();
let storage = Storage::open(&layout.database()).unwrap();
let html = include_str!("fixtures/code_tutorial.html");
let url = Url::parse("https://example.test/tutorials/mesh").unwrap();
let page = parse_html("archive", "page", &url, html, "hash", 0);
storage
.upsert_page(
&page,
url.as_str(),
200,
Some("text/html"),
Some("raw/pages/page.html"),
Some("readable/markdown/page.md"),
Some("readable/json/page.json"),
Some("readable/text/page.txt"),
None,
None,
)
.unwrap();
let results = storage.search("mesh deformation", 10).unwrap();
assert!(!results.is_empty());
assert_eq!(results[0].page_id, "page");
}
#[test]
fn compact_archives_search_readable_files_without_fts_bloat() {
let temp = tempdir().unwrap();
let config = test_config(temp.path());
let layout = ArchiveLayout::new(temp.path().to_path_buf(), "archive");
layout.ensure().unwrap();
let seed = Url::parse("https://example.test/tutorials/mesh").unwrap();
let manifest = new_manifest(
"archive".to_string(),
std::slice::from_ref(&seed),
false,
true,
0,
&config,
);
write_manifest(&layout, &manifest).unwrap();
let storage = Storage::open_compact(&layout.database()).unwrap();
let html = include_str!("fixtures/code_tutorial.html");
let page = parse_html("archive", "page", &seed, html, "hash", 0);
fs::write(
layout.readable_text_dir().join("page.txt"),
page.to_plain_text(),
)
.unwrap();
fs::write(
layout.readable_markdown_dir().join("page.md"),
page.to_markdown().unwrap(),
)
.unwrap();
storage
.upsert_page(
&page,
seed.as_str(),
200,
Some("text/html"),
Some("raw/pages/page.html"),
Some("readable/markdown/page.md"),
Some("readable/json/page.json"),
Some("readable/text/page.txt"),
None,
None,
)
.unwrap();
let results = search_archives(&config, Some("archive"), "Debug.Log", 5).unwrap();
assert_eq!(results.len(), 1);
assert!(results[0]
.local_file_path
.as_deref()
.unwrap()
.ends_with("page.txt"));
}
#[test]
fn siteforge_facade_resolves_archive_layouts() {
let temp = tempdir().unwrap();
let config = test_config(temp.path());
let siteforge = Siteforge::new(config);
let layout = siteforge.archive_layout("docs").unwrap();
assert_eq!(layout.base(), temp.path().join("docs"));
}
#[test]
fn archive_run_options_helpers_use_config_defaults() {
let config = Config {
default_concurrency: 0,
default_delay_ms: 75,
max_depth: 3,
max_pages: 42,
..Config::default()
};
let options = ArchiveRunOptions::full_site_url("https://example.test/docs/", &config)
.unwrap()
.with_archive_id("docs")
.with_concurrency(0)
.with_delay_ms(5)
.with_header("Authorization", "Bearer token")
.with_cookie("session=1");
assert_eq!(options.seeds[0].as_str(), "https://example.test/docs/");
assert!(options.full_site);
assert!(options.same_domain);
assert_eq!(options.max_depth, 3);
assert_eq!(options.max_pages, 42);
assert_eq!(options.concurrency, 1);
assert_eq!(options.delay_ms, 5);
assert_eq!(options.archive_id.as_deref(), Some("docs"));
assert_eq!(options.cookie.as_deref(), Some("session=1"));
assert_eq!(
options.headers[0],
("Authorization".into(), "Bearer token".into())
);
let page_options = ArchiveRunOptions::single_page_url("https://example.test/page", &config)
.unwrap()
.allowing_cross_domain();
assert!(!page_options.full_site);
assert!(!page_options.same_domain);
assert_eq!(page_options.max_depth, 0);
assert_eq!(page_options.max_pages, 1);
}
#[test]
fn archive_summary_exposes_common_paths() {
let temp = tempdir().unwrap();
let summary = ArchiveSummary {
archive_id: "docs".to_string(),
archive_path: temp.path().join("docs"),
stats: CrawlStats::default(),
};
assert_eq!(
summary.manifest_path(),
temp.path().join("docs/manifest.json")
);
assert_eq!(
summary.checksums_path(),
temp.path().join("docs/checksums.json")
);
assert_eq!(summary.agents_path(), temp.path().join("docs/AGENTS.md"));
assert_eq!(
summary.agent_index_path(),
temp.path().join("docs/agent-index.json")
);
assert_eq!(
summary.thesa_manifest_path(),
temp.path().join("docs/.thesa/manifest.json")
);
assert_eq!(
summary.thesa_checksums_path(),
temp.path().join("docs/.thesa/checksums.blake3")
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn archive_uses_configured_concurrency_for_many_seed_urls() {
let server = ConcurrentTestServer::start();
let temp = tempdir().unwrap();
let config = Config {
archive_root: temp.path().to_path_buf(),
default_delay_ms: 0,
retry_count: 0,
max_pages: 3,
..Config::default()
};
let seeds = ["/a", "/b", "/c"]
.into_iter()
.map(|path| Url::parse(&server.url(path)).unwrap())
.collect::<Vec<_>>();
let options = ArchiveRunOptions::new(seeds, &config)
.with_max_pages(3)
.with_concurrency(3)
.with_delay_ms(0)
.with_archive_id("concurrent");
let summary = run_archive(config, options).await.unwrap();
assert_eq!(summary.stats.pages_parsed, 3);
let layout = ArchiveLayout::new(temp.path().to_path_buf(), "concurrent");
assert!(layout.agents_md().exists());
assert!(layout.agent_index_json().exists());
let agent_index: serde_json::Value =
serde_json::from_str(&fs::read_to_string(layout.agent_index_json()).unwrap()).unwrap();
assert_eq!(agent_index["pages"].as_array().unwrap().len(), 3);
assert_eq!(agent_index["paths"]["chunks_jsonl"], "chunks/chunks.jsonl");
assert!(layout.thesa_manifest().exists());
assert!(layout.thesa_checksums().exists());
let readme = fs::read_to_string(layout.readable_dir().join("README.md")).unwrap();
assert!(readme.contains("## Pages"));
assert!(readme.contains("Source: http://"));
assert!(fs::read_dir(layout.readable_basic_markdown_dir())
.unwrap()
.next()
.is_some());
assert!(
server.max_active() > 1,
"expected overlapping page fetches with concurrency=3"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn lean_archive_profile_omits_raw_assets_and_duplicate_text() {
let server = ConcurrentTestServer::start();
let temp = tempdir().unwrap();
let config = Config {
archive_root: temp.path().to_path_buf(),
archive_profile: ArchiveOutputProfile::Lean,
default_delay_ms: 0,
retry_count: 0,
..Config::default()
};
let seed = Url::parse(&server.url("/lean")).unwrap();
let options = ArchiveRunOptions::single_page(seed, &config)
.with_delay_ms(0)
.with_archive_id("lean");
let summary = run_archive(config, options).await.unwrap();
assert_eq!(summary.stats.pages_parsed, 1);
let layout = ArchiveLayout::new(temp.path().to_path_buf(), "lean");
assert!(fs::read_dir(layout.readable_basic_markdown_dir())
.unwrap()
.next()
.is_some());
assert!(fs::read_dir(layout.readable_json_dir())
.unwrap()
.next()
.is_some());
assert!(fs::read_dir(layout.raw_pages_dir())
.unwrap()
.next()
.is_none());
assert!(fs::read_dir(layout.raw_assets_dir())
.unwrap()
.next()
.is_none());
assert!(fs::read_dir(layout.readable_text_dir())
.unwrap()
.next()
.is_none());
assert!(fs::read_dir(layout.readable_markdown_dir())
.unwrap()
.next()
.is_none());
}
#[test]
fn resume_resets_in_progress_frontier() {
let temp = tempdir().unwrap();
let layout = ArchiveLayout::new(temp.path().to_path_buf(), "archive");
layout.ensure().unwrap();
let storage = Storage::open(&layout.database()).unwrap();
storage
.enqueue_url("https://example.test/a", 0, None)
.unwrap();
storage
.mark_frontier("https://example.test/a", "in_progress", None)
.unwrap();
storage.reset_in_progress_frontier().unwrap();
let pending = storage.pending_frontier(10).unwrap();
assert_eq!(pending.len(), 1);
assert_eq!(pending[0].url, "https://example.test/a");
}
#[test]
fn ai_pack_generation_writes_context_bundle() {
let temp = tempdir().unwrap();
let config = test_config(temp.path());
let layout = ArchiveLayout::new(temp.path().to_path_buf(), "archive");
layout.ensure().unwrap();
let storage = Storage::open(&layout.database()).unwrap();
let html = include_str!("fixtures/code_tutorial.html");
let url = Url::parse("https://example.test/tutorials/mesh").unwrap();
let page = parse_html("archive", "page", &url, html, "hash", 0);
let markdown_path = layout.readable_markdown_dir().join("page.md");
fs::write(&markdown_path, page.to_markdown().unwrap()).unwrap();
storage
.upsert_page(
&page,
url.as_str(),
200,
Some("text/html"),
Some("raw/pages/page.html"),
Some("readable/markdown/page.md"),
Some("readable/json/page.json"),
Some("readable/text/page.txt"),
None,
None,
)
.unwrap();
let packs = create_ai_packs(&config, "archive", Some(1_000)).unwrap();
assert_eq!(packs.len(), 1);
let pack = fs::read_to_string(&packs[0]).unwrap();
assert!(pack.contains("Table of Contents"));
assert!(pack.contains("Debug.Log"));
}
#[test]
fn oversized_pages_pack_by_semantic_chunks() {
let temp = tempdir().unwrap();
let config = test_config(temp.path());
let layout = ArchiveLayout::new(temp.path().to_path_buf(), "archive");
layout.ensure().unwrap();
let storage = Storage::open(&layout.database()).unwrap();
let repeated_a = "alpha ".repeat(3_000);
let repeated_b = "beta ".repeat(3_000);
let html = format!(
"<article><h1>Large</h1><p>{}</p><pre><code class=\"language-rust\">fn main() {{\n println!(\"large\");\n}}</code></pre><p>{}</p></article>",
repeated_a, repeated_b
);
let url = Url::parse("https://example.test/large").unwrap();
let page = parse_html("archive", "page", &url, &html, "hash", 0);
fs::write(
layout.readable_markdown_dir().join("page.md"),
page.to_markdown().unwrap(),
)
.unwrap();
let mut chunks_jsonl = String::new();
for chunk in page.chunks() {
chunks_jsonl.push_str(&serde_json::to_string(&chunk).unwrap());
chunks_jsonl.push('\n');
}
fs::write(layout.chunks_jsonl(), chunks_jsonl).unwrap();
storage
.upsert_page(
&page,
url.as_str(),
200,
Some("text/html"),
Some("raw/pages/page.html"),
Some("readable/markdown/page.md"),
Some("readable/json/page.json"),
Some("readable/text/page.txt"),
None,
None,
)
.unwrap();
let packs = create_ai_packs(&config, "archive", Some(1_000)).unwrap();
assert!(packs.len() > 1);
let combined = packs
.iter()
.map(|path| fs::read_to_string(path).unwrap())
.collect::<Vec<_>>()
.join("\n");
assert!(combined.contains("```rust\nfn main()"));
}
#[test]
fn zip_and_tar_exports_create_bundle_files() {
let temp = tempdir().unwrap();
let config = test_config(temp.path());
let layout = ArchiveLayout::new(temp.path().to_path_buf(), "archive");
layout.ensure().unwrap();
fs::write(layout.manifest(), "{}").unwrap();
fs::write(layout.chunks_jsonl(), "{}").unwrap();
fs::write(layout.agents_md(), "# Agents").unwrap();
fs::write(layout.agent_index_json(), "{}").unwrap();
fs::write(layout.raw_pages_dir().join("page.html"), "<html></html>").unwrap();
fs::write(
layout.readable_basic_markdown_dir().join("page.md"),
"# Basic",
)
.unwrap();
fs::write(layout.readable_markdown_dir().join("page.md"), "# Page").unwrap();
fs::create_dir_all(layout.exports_dir().join("old")).unwrap();
fs::write(layout.exports_dir().join("old/nested.tar"), "old").unwrap();
let zip_dir = export_archive(&config, "archive", ExportFormat::Zip, None).unwrap();
let tar_dir = export_archive(&config, "archive", ExportFormat::Tar, None).unwrap();
let basic_dir = export_archive(&config, "archive", ExportFormat::BasicMarkdown, None).unwrap();
let agent_dir = export_archive(&config, "archive", ExportFormat::AgentTar, None).unwrap();
assert!(zip_dir.join("archive.zip").exists());
assert!(tar_dir.join("archive.tar").exists());
assert!(basic_dir.join("page.md").exists());
let agent_tar = agent_dir.join("archive-agent.tar.gz");
assert!(agent_tar.exists());
let decoder = GzDecoder::new(std::fs::File::open(agent_tar).unwrap());
let mut archive = tar::Archive::new(decoder);
let names = archive
.entries()
.unwrap()
.map(|entry| entry.unwrap().path().unwrap().to_string_lossy().to_string())
.collect::<Vec<_>>();
assert!(names.contains(&"AGENTS.md".to_string()));
assert!(names.contains(&"agent-index.json".to_string()));
assert!(names.contains(&"readable/basic_markdown/page.md".to_string()));
assert!(names.contains(&"raw/pages/page.html".to_string()));
assert!(!names.iter().any(|name| name.starts_with("exports/")));
}
#[test]
fn warc_export_writes_all_stored_pages() {
let temp = tempdir().unwrap();
let config = test_config(temp.path());
let layout = ArchiveLayout::new(temp.path().to_path_buf(), "archive");
layout.ensure().unwrap();
let storage = Storage::open(&layout.database()).unwrap();
let html = "<article><h1>Page</h1><p>body</p></article>";
for index in 0..3 {
let url = Url::parse(&format!("https://example.test/{index}")).unwrap();
let page = parse_html("archive", &format!("page-{index}"), &url, html, "hash", 0);
storage
.upsert_page(
&page,
url.as_str(),
200,
Some("text/html"),
Some(&format!("raw/pages/page-{index}.html")),
Some(&format!("readable/markdown/page-{index}.md")),
Some(&format!("readable/json/page-{index}.json")),
Some(&format!("readable/text/page-{index}.txt")),
None,
None,
)
.unwrap();
}
let export_dir = export_archive(&config, "archive", ExportFormat::Warc, None).unwrap();
let warc = fs::read_to_string(export_dir.join("archive.warc.jsonl")).unwrap();
assert_eq!(warc.lines().count(), 3);
assert!(warc.contains("https://example.test/0"));
assert!(warc.contains("raw/pages/page-2.html"));
}
#[test]
fn ocr_unavailable_is_non_fatal_and_clear() {
let backend = TesseractOcr::new("siteforge-missing-ocr-binary {input} stdout");
let err = backend.is_available().unwrap_err().to_string();
assert!(err.contains("OCR unavailable"));
}
#[test]
fn malformed_html_still_extracts_recoverable_content() {
let html = include_str!("fixtures/malformed.html");
let url = Url::parse("https://example.test/broken").unwrap();
let page = parse_html("archive", "page", &url, html, "hash", 0);
let text = page.to_plain_text();
assert!(text.contains("Broken"));
assert!(text.contains("still parse"));
}
#[test]
fn checksum_manifest_verifies_and_detects_tampering() {
let temp = tempdir().unwrap();
let config = test_config(temp.path());
let layout = ArchiveLayout::new(temp.path().to_path_buf(), "archive");
layout.ensure().unwrap();
fs::write(layout.manifest(), "{}").unwrap();
fs::write(layout.chunks_jsonl(), "").unwrap();
let storage = Storage::open(&layout.database()).unwrap();
storage.checkpoint().unwrap();
let raw_page = layout.raw_pages_dir().join("page.html");
fs::write(&raw_page, "<html>ok</html>").unwrap();
write_checksums(&layout).unwrap();
assert!(verify_archive(&config, "archive").unwrap().is_empty());
fs::write(&raw_page, "<html>tampered</html>").unwrap();
let problems = verify_archive(&config, "archive").unwrap();
assert!(problems
.iter()
.any(|problem| problem.contains("hash mismatch")));
}
#[test]
fn streamed_checksum_manifest_verifies_and_detects_tampering() {
let temp = tempdir().unwrap();
let config = test_config(temp.path());
let layout = ArchiveLayout::new(temp.path().to_path_buf(), "archive");
layout.ensure().unwrap();
fs::write(layout.manifest(), "{}").unwrap();
fs::write(layout.chunks_jsonl(), "").unwrap();
let storage = Storage::open(&layout.database()).unwrap();
storage.checkpoint().unwrap();
let raw_page = layout.raw_pages_dir().join("page.html");
fs::write(&raw_page, "<html>ok</html>").unwrap();
write_checksums_streamed(&layout).unwrap();
let checksums = fs::read_to_string(layout.checksums()).unwrap();
assert!(checksums
.lines()
.any(|line| line.trim().starts_with("{\"path\":")));
assert!(verify_archive(&config, "archive").unwrap().is_empty());
fs::write(&raw_page, "<html>tampered</html>").unwrap();
let problems = verify_archive(&config, "archive").unwrap();
assert!(problems
.iter()
.any(|problem| problem.contains("hash mismatch")));
}
#[test]
fn thesa_sidecar_matches_schema_and_skips_transient_files() {
let temp = tempdir().unwrap();
let config = test_config(temp.path());
let layout = ArchiveLayout::new(temp.path().to_path_buf(), "archive");
layout.ensure().unwrap();
let seed = Url::parse("https://example.test/").unwrap();
let manifest = new_manifest("archive".to_string(), &[seed], false, true, 0, &config);
write_manifest(&layout, &manifest).unwrap();
fs::write(layout.raw_pages_dir().join("page.html"), "<html>ok</html>").unwrap();
fs::create_dir_all(layout.exports_dir().join("old")).unwrap();
fs::write(layout.exports_dir().join("old/export.zip"), "derived").unwrap();
fs::write(layout.base().join("crawl.db-wal"), "transient").unwrap();
fs::create_dir_all(layout.thesa_dir()).unwrap();
fs::write(layout.thesa_dir().join("old"), "ignore").unwrap();
let summary = write_thesa_sidecar(&layout, &manifest, true).unwrap();
assert_eq!(summary.manifest_path, layout.thesa_manifest());
assert_eq!(summary.checksums_path, layout.thesa_checksums());
let manifest_text = fs::read_to_string(layout.thesa_manifest()).unwrap();
assert!(!manifest_text.contains('\n'));
let parsed: serde_json::Value = serde_json::from_str(&manifest_text).unwrap();
assert_eq!(parsed["format_version"], "1");
assert_eq!(parsed["tool"]["name"], "siteforge");
assert_eq!(parsed["source"]["platform"], "siteforge");
assert_eq!(parsed["archive"]["complete"], true);
let paths = parsed["files"]
.as_array()
.unwrap()
.iter()
.map(|file| file["path"].as_str().unwrap().to_string())
.collect::<Vec<_>>();
assert!(paths.contains(&"manifest.json".to_string()));
assert!(paths.contains(&"raw/pages/page.html".to_string()));
assert!(!paths.iter().any(|path| path.starts_with(".thesa/")));
assert!(!paths.iter().any(|path| path.starts_with("exports/")));
assert!(!paths.iter().any(|path| path.ends_with(".db-wal")));
let checksums = fs::read_to_string(layout.thesa_checksums()).unwrap();
assert_eq!(checksums.lines().count(), summary.file_count);
assert!(checksums.lines().all(|line| line.contains(" ")));
}
#[test]
fn total_archive_size_limit_is_enforced() {
let temp = tempdir().unwrap();
let config = Config {
archive_root: temp.path().to_path_buf(),
max_total_archive_size_bytes: 5,
..Config::default()
};
let layout = ArchiveLayout::new(temp.path().to_path_buf(), "archive");
layout.ensure().unwrap();
fs::write(layout.raw_pages_dir().join("a.html"), "1234").unwrap();
let err =
ensure_archive_size_can_grow(&config, &layout, 2, &layout.raw_pages_dir().join("b.html"))
.unwrap_err()
.to_string();
assert!(err.contains("size limit exceeded"));
}
#[test]
fn downloaded_source_asset_becomes_code_block_and_chunk() {
let html = include_str!("fixtures/relative_links.html");
let url = Url::parse("https://example.test/unity/tutorials/advanced/").unwrap();
let mut page = parse_html("archive", "page", &url, html, "hash", 0);
page.add_source_asset_code(
"asset-source",
"https://example.test/downloads/example.rs",
"fn main() {\n println!(\"asset\");\n}".to_string(),
Some("rust".to_string()),
);
let markdown = page.to_markdown().unwrap();
assert!(markdown.contains("Downloaded source asset"));
assert!(markdown.contains("rust\nfn main()"));
let chunks = page.chunks();
assert!(chunks.iter().any(|chunk| {
chunk.content_type == "code" && chunk.asset_refs == vec!["asset-source".to_string()]
}));
}
fn test_config(archive_root: &Path) -> Config {
Config {
archive_root: archive_root.to_path_buf(),
..Config::default()
}
}
struct ConcurrentTestServer {
addr: SocketAddr,
max_active: Arc<AtomicUsize>,
shutdown: Arc<AtomicBool>,
}
impl ConcurrentTestServer {
fn start() -> Self {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
listener.set_nonblocking(true).unwrap();
let addr = listener.local_addr().unwrap();
let active = Arc::new(AtomicUsize::new(0));
let max_active = Arc::new(AtomicUsize::new(0));
let shutdown = Arc::new(AtomicBool::new(false));
let active_for_thread = active.clone();
let max_for_thread = max_active.clone();
let shutdown_for_thread = shutdown.clone();
thread::spawn(move || {
while !shutdown_for_thread.load(Ordering::SeqCst) {
match listener.accept() {
Ok((stream, _)) => {
let active = active_for_thread.clone();
let max_active = max_for_thread.clone();
thread::spawn(move || handle_test_request(stream, active, max_active));
}
Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => {
thread::sleep(Duration::from_millis(5));
}
Err(_) => break,
}
}
});
Self {
addr,
max_active,
shutdown,
}
}
fn url(&self, path: &str) -> String {
format!("http://{}{}", self.addr, path)
}
fn max_active(&self) -> usize {
self.max_active.load(Ordering::SeqCst)
}
}
impl Drop for ConcurrentTestServer {
fn drop(&mut self) {
self.shutdown.store(true, Ordering::SeqCst);
let _ = TcpStream::connect(self.addr);
}
}
fn handle_test_request(
mut stream: TcpStream,
active: Arc<AtomicUsize>,
max_active: Arc<AtomicUsize>,
) {
let mut buffer = [0u8; 1024];
let read = stream.read(&mut buffer).unwrap_or(0);
let request = String::from_utf8_lossy(&buffer[..read]);
let path = request
.lines()
.next()
.and_then(|line| line.split_whitespace().nth(1))
.unwrap_or("/");
if path == "/robots.txt" {
let _ = stream
.write_all(b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n");
return;
}
let current = active.fetch_add(1, Ordering::SeqCst) + 1;
update_max(&max_active, current);
thread::sleep(Duration::from_millis(200));
active.fetch_sub(1, Ordering::SeqCst);
let body = format!(
"<!doctype html><html><head><title>{path}</title></head><body><article><h1>{path}</h1><p>concurrency page {path}</p></article></body></html>"
);
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
body.len(), body
);
let _ = stream.write_all(response.as_bytes());
}
fn update_max(max_active: &AtomicUsize, current: usize) {
let mut observed = max_active.load(Ordering::SeqCst);
while current > observed {
match max_active.compare_exchange(observed, current, Ordering::SeqCst, Ordering::SeqCst) {
Ok(_) => break,
Err(next) => observed = next,
}
}
}