use std::fs;
use std::path::{Path, PathBuf};
use siteforge::{
ArchiveLayout as SiteforgeArchiveLayout, ArchiveOutputProfile,
ArchiveRunOptions as SiteforgeArchiveRunOptions, Config as SiteforgeConfig,
CrawlStats as SiteforgeCrawlStats, Siteforge,
};
use crate::cli::{
Cli, SiteArchiveArgs, SiteforgeArchiveIdArgs, SiteforgeDiagnosticsArgs, SiteforgeExportArgs,
SiteforgePackArgs, SiteforgeRootArgs, SiteforgeSearchArgs,
};
use crate::constants::THESA_METADATA_DIR;
use crate::error::{Result, ThesaError};
use crate::manifest::{ArchiveManifest, read_archive_manifest_sidecar, write_archive_manifest};
use crate::terminal::TerminalSession;
use crate::types::{SiteArchiveProfile, SiteTuiOptions};
fn siteforge_config_from_root(root: &SiteforgeRootArgs) -> SiteforgeConfig {
SiteforgeConfig {
archive_root: root.archive_root.clone(),
..SiteforgeConfig::default()
}
}
fn siteforge_err(context: &str, err: impl std::fmt::Display) -> ThesaError {
ThesaError::Message(format!("siteforge {context} failed: {err}"))
}
pub(crate) fn run_siteforge_list(args: &SiteforgeRootArgs) -> Result<()> {
let config = siteforge_config_from_root(args);
let root = config
.resolved_archive_root()
.map_err(|err| siteforge_err("list", err))?;
let archives = Siteforge::new(config)
.list_archives()
.map_err(|err| siteforge_err("list", err))?;
if archives.is_empty() {
println!("No Siteforge archives found at {}", root.display());
return Ok(());
}
for archive in archives {
println!(
"{} updated={} pages={}/{} failed={} seeds={}",
archive.archive_id,
archive.updated_at,
archive.stats.pages_parsed,
archive.stats.pages_discovered,
archive.stats.pages_failed,
archive.seeds.join(", ")
);
}
Ok(())
}
pub(crate) fn run_siteforge_inspect(args: &SiteforgeArchiveIdArgs) -> Result<()> {
let config = siteforge_config_from_root(&args.root);
let manifest = siteforge::archive::load_archive_manifest(&config, &args.archive_id)
.map_err(|err| siteforge_err("inspect", err))?;
let layout = SiteforgeArchiveLayout::new(
config
.resolved_archive_root()
.map_err(|err| siteforge_err("inspect", err))?,
args.archive_id.clone(),
);
println!("archive_id: {}", manifest.archive_id);
println!("created_at: {}", manifest.created_at);
println!("updated_at: {}", manifest.updated_at);
println!("path: {}", layout.base().display());
println!("seeds: {}", manifest.seeds.join(", "));
println!("full_site: {}", manifest.scope.full_site);
println!("same_domain: {}", manifest.scope.same_domain);
println!("max_depth: {}", manifest.scope.max_depth);
println!("archive_profile: {:?}", manifest.config.archive_profile);
println!("render_js: {}", manifest.config.render_js);
println!("render_js_auto: {}", manifest.config.render_js_auto);
println!("pages_discovered: {}", manifest.stats.pages_discovered);
println!("pages_parsed: {}", manifest.stats.pages_parsed);
println!("pages_failed: {}", manifest.stats.pages_failed);
println!("assets_downloaded: {}", manifest.stats.assets_downloaded);
if layout.database().exists() {
let storage = siteforge::storage::Storage::open(&layout.database())
.map_err(|err| siteforge_err("inspect", err))?;
println!(
"pages_indexed: {}",
storage
.page_count()
.map_err(|err| siteforge_err("inspect", err))?
);
println!(
"assets_indexed: {}",
storage
.asset_count()
.map_err(|err| siteforge_err("inspect", err))?
);
println!(
"asset_bytes: {}",
storage
.total_asset_size_bytes()
.map_err(|err| siteforge_err("inspect", err))?
);
let recent_pages = storage
.recent_pages_limited(5)
.map_err(|err| siteforge_err("inspect", err))?;
if !recent_pages.is_empty() {
println!("recent_pages:");
for page in recent_pages {
println!(" - [{}] {} ({})", page.status, page.title, page.source_url);
}
}
}
Ok(())
}
pub(crate) fn run_siteforge_search(args: &SiteforgeSearchArgs) -> Result<()> {
let config = siteforge_config_from_root(&args.root);
let results = Siteforge::new(config)
.search(args.archive_id.as_deref(), &args.query, args.limit)
.map_err(|err| siteforge_err("search", err))?;
if results.is_empty() {
println!("No matches.");
return Ok(());
}
for result in results {
println!("{} | {}", result.archive_id, result.title);
println!(" {}", result.source_url);
if let Some(heading) = result.matched_heading.as_deref() {
println!(" heading: {heading}");
}
println!(" snippet: {}", result.snippet.replace('\n', " "));
if let Some(path) = result.local_file_path.as_deref() {
println!(" local: {path}");
}
}
Ok(())
}
pub(crate) fn run_siteforge_export(args: &SiteforgeExportArgs, dry_run: bool) -> Result<()> {
if dry_run {
println!("Dry run: siteforge would export {}", args.archive_id);
println!("Archive root: {}", args.root.archive_root.display());
println!("Format: {}", args.format.as_str());
if let Some(output) = &args.output {
println!("Output: {}", output.display());
}
return Ok(());
}
let config = siteforge_config_from_root(&args.root);
let path = Siteforge::new(config)
.export(&args.archive_id, args.format, args.output.as_deref())
.map_err(|err| siteforge_err("export", err))?;
println!("Exported: {}", path.display());
Ok(())
}
pub(crate) fn run_siteforge_pack(args: &SiteforgePackArgs, dry_run: bool) -> Result<()> {
if dry_run {
println!(
"Dry run: siteforge would create AI packs for {}",
args.archive_id
);
println!("Archive root: {}", args.root.archive_root.display());
if let Some(max_tokens) = args.max_tokens {
println!("Max tokens: {max_tokens}");
}
return Ok(());
}
let config = siteforge_config_from_root(&args.root);
let packs = Siteforge::new(config)
.create_ai_packs(&args.archive_id, args.max_tokens)
.map_err(|err| siteforge_err("pack", err))?;
for pack in packs {
println!("Pack: {}", pack.display());
}
Ok(())
}
pub(crate) fn run_siteforge_verify(args: &SiteforgeArchiveIdArgs) -> Result<()> {
let config = siteforge_config_from_root(&args.root);
let problems = Siteforge::new(config)
.verify(&args.archive_id)
.map_err(|err| siteforge_err("verify", err))?;
if problems.is_empty() {
println!("Siteforge archive verified: {}", args.archive_id);
} else {
println!(
"Siteforge archive verification found {} problem(s):",
problems.len()
);
for problem in problems {
println!("- {problem}");
}
}
Ok(())
}
pub(crate) fn run_siteforge_resume(args: &SiteforgeArchiveIdArgs, dry_run: bool) -> Result<()> {
if dry_run {
println!("Dry run: siteforge would resume {}", args.archive_id);
println!("Archive root: {}", args.root.archive_root.display());
return Ok(());
}
let config = siteforge_config_from_root(&args.root);
let runtime = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.map_err(|err| ThesaError::Message(format!("failed to start async runtime: {err}")))?;
let summary = runtime
.block_on(Siteforge::new(config).resume(&args.archive_id))
.map_err(|err| siteforge_err("resume", err))?;
println!("Resumed archive: {}", summary.archive_id);
println!("Archive: {}", summary.archive_path.display());
println!(
"Pages: fetched={} parsed={} failed={}",
summary.stats.pages_fetched, summary.stats.pages_parsed, summary.stats.pages_failed
);
Ok(())
}
pub(crate) fn run_siteforge_diagnostics(args: &SiteforgeDiagnosticsArgs) -> Result<()> {
let config = siteforge_config_from_root(&args.root);
let root = config
.resolved_archive_root()
.map_err(|err| siteforge_err("diagnostics", err))?;
println!("archive_root: {}", root.display());
println!("default_concurrency: {}", config.default_concurrency);
println!("archive_profile: {:?}", config.archive_profile);
println!("render_js: {}", config.render_js);
println!("render_js_auto: {}", config.render_js_auto);
println!("max_pages: {}", config.max_pages);
println!("max_page_size_bytes: {}", config.max_page_size_bytes);
println!("max_asset_size_bytes: {}", config.max_asset_size_bytes);
println!(
"max_total_archive_size_bytes: {}",
config.max_total_archive_size_bytes
);
if let Some(archive_id) = &args.archive_id {
let manifest = siteforge::archive::load_archive_manifest(&config, archive_id)
.map_err(|err| siteforge_err("diagnostics", err))?;
let layout = SiteforgeArchiveLayout::new(root, archive_id.clone());
println!("archive_id: {}", manifest.archive_id);
println!("archive_path: {}", layout.base().display());
println!(
"archive_size_bytes: {}",
siteforge::archive::archive_size_bytes(&layout)
.map_err(|err| siteforge_err("diagnostics", err))?
);
if layout.database().exists() {
let storage = siteforge::storage::Storage::open(&layout.database())
.map_err(|err| siteforge_err("diagnostics", err))?;
println!(
"jobs: {}",
storage
.job_count()
.map_err(|err| siteforge_err("diagnostics", err))?
);
for job in storage
.job_snapshots_limited(20)
.map_err(|err| siteforge_err("diagnostics", err))?
{
println!(
"- job={} status={} active_url={}",
job.job_id,
job.status,
job.active_url.as_deref().unwrap_or("none")
);
}
}
return Ok(());
}
let archives = Siteforge::new(config)
.list_archives()
.map_err(|err| siteforge_err("diagnostics", err))?;
println!("archives: {}", archives.len());
for archive in archives.into_iter().take(50) {
println!(
"- {} updated={} parsed={} failed={}",
archive.archive_id,
archive.updated_at,
archive.stats.pages_parsed,
archive.stats.pages_failed
);
}
Ok(())
}
pub(crate) fn print_site_archive_dry_run(args: &SiteArchiveArgs) -> Result<()> {
let seeds = load_site_seed_urls(args.url.as_deref(), args.input.as_deref())?;
let headers = parse_site_headers(&args.headers)?;
println!(
"Dry run: siteforge would archive {}",
site_seed_display(&seeds)
);
println!("Output root: {}", args.output.display());
if let Some(input) = &args.input {
println!("Input seeds: {}", input.display());
}
println!(
"Mode: {}",
if args.full_site {
"same-domain full-site crawl"
} else {
"single page"
}
);
println!(
"Cross-domain: {}",
if args.allow_cross_domain {
"allowed"
} else {
"blocked"
}
);
println!(
"Max depth: {}",
args.max_depth
.map(|value| value.to_string())
.unwrap_or_else(|| "config default".to_string())
);
println!(
"Max pages: {}",
args.max_pages
.map(|value| value.to_string())
.unwrap_or_else(|| "config default".to_string())
);
println!(
"Concurrency: {}",
args.concurrency
.map(|value| value.to_string())
.unwrap_or_else(|| "config default".to_string())
);
println!(
"Delay: {}",
args.delay_ms
.map(|value| format!("{value} ms"))
.unwrap_or_else(|| "config default".to_string())
);
if !args.include_url_patterns.is_empty() {
println!("Include globs: {}", args.include_url_patterns.join(", "));
}
if !args.exclude_url_patterns.is_empty() {
println!("Exclude globs: {}", args.exclude_url_patterns.join(", "));
}
if let Some(archive_profile) = args.archive_profile {
println!(
"Archive profile: {}",
site_archive_profile_label(archive_profile.as_output_profile())
);
}
if let Some(max_page_size_bytes) = args.max_page_size_bytes {
println!("Max page size: {max_page_size_bytes} bytes");
}
if let Some(max_asset_size_bytes) = args.max_asset_size_bytes {
println!("Max asset size: {max_asset_size_bytes} bytes");
}
if let Some(max_total_archive_size_bytes) = args.max_total_archive_size_bytes {
println!("Max total archive size: {max_total_archive_size_bytes} bytes");
}
println!(
"Render JS: {}",
site_render_mode_label(args.render_js, args.render_js_auto)
);
if let Some(browser_command) = &args.browser_command {
println!("Browser command: {browser_command}");
}
if let Some(browser_profile) = &args.browser_profile {
println!("Browser profile: {}", browser_profile.display());
}
if let Some(render_wait_ms) = args.render_wait_ms {
println!("Render wait: {render_wait_ms} ms");
}
if let Some(user_agent) = &args.user_agent {
println!("User-Agent: {user_agent}");
}
if let Some(timeout_secs) = args.timeout_secs {
println!("Timeout: {timeout_secs} seconds");
}
if let Some(retry_count) = args.retry_count {
println!("Retries: {retry_count}");
}
println!(
"OCR: {}",
if args.ocr_enabled {
"enabled"
} else {
"config default"
}
);
if !headers.is_empty() {
let names = headers
.iter()
.map(|(name, _)| name.as_str())
.collect::<Vec<_>>()
.join(", ");
println!("Headers: {} configured ({names})", headers.len());
}
if args.cookie.is_some() {
println!("Cookie: configured (redacted)");
}
if let Some(archive_id) = &args.archive_id {
println!("Archive ID: {archive_id}");
}
Ok(())
}
pub(crate) fn load_site_seed_urls(
primary: Option<&str>,
input: Option<&Path>,
) -> Result<Vec<reqwest::Url>> {
let mut seeds = Vec::new();
if let Some(primary) = primary {
seeds.push(parse_site_url(primary)?);
}
if let Some(input) = input {
let raw = fs::read_to_string(input)?;
for (line_index, line) in raw.lines().enumerate() {
let line = line.trim();
if line.is_empty() || line.starts_with('#') {
continue;
}
let seed = parse_site_url(line).map_err(|err| {
ThesaError::Message(format!(
"invalid site URL in {} at line {}: {err}",
input.display(),
line_index + 1
))
})?;
seeds.push(seed);
}
}
if seeds.is_empty() {
return Err(ThesaError::Message(
"siteforge archive requires a URL or --input seed file".to_string(),
));
}
Ok(seeds)
}
fn site_seed_display(seeds: &[reqwest::Url]) -> String {
match seeds {
[] => "no seeds".to_string(),
[seed] => seed.as_str().to_string(),
[seed, rest @ ..] => format!("{} (+{} more seeds)", seed.as_str(), rest.len()),
}
}
fn site_source_target(seeds: &[reqwest::Url]) -> String {
match seeds {
[] => "siteforge".to_string(),
[seed] => seed.as_str().to_string(),
[seed, rest @ ..] => format!("{} (+{} seeds)", seed.as_str(), rest.len()),
}
}
pub(crate) fn parse_site_headers(values: &[String]) -> Result<Vec<(String, String)>> {
values
.iter()
.map(|value| {
let (name, header_value) = value.split_once(':').ok_or_else(|| {
ThesaError::Message(format!("invalid header {value:?}; expected 'Name: Value'"))
})?;
let name = name.trim();
let header_value = header_value.trim();
if name.is_empty() || header_value.is_empty() {
return Err(ThesaError::Message(format!(
"invalid header {value:?}; name and value are required"
)));
}
Ok((name.to_string(), header_value.to_string()))
})
.collect()
}
pub(crate) fn run_sites_mode(
args: &mut Cli,
terminal_session: &mut Option<TerminalSession>,
tui_options: Option<SiteTuiOptions>,
) -> Result<()> {
let url = args
.target
.as_deref()
.unwrap_or_default()
.trim()
.to_string();
parse_site_url(&url)?;
*terminal_session = None;
let output = if args.output == Path::new("./archives") {
PathBuf::from("./archives/sites")
} else {
args.output.clone()
};
let full_site = tui_options.map(|options| options.full_site).unwrap_or(true);
let max_depth = tui_options.map(|options| options.max_depth);
let max_pages = tui_options.map(|options| options.max_pages);
let archive_profile = tui_options.map(|options| options.archive_profile);
let render_js = tui_options
.map(|options| options.render_js)
.unwrap_or(false);
let render_js_auto = tui_options
.map(|options| options.render_js_auto)
.unwrap_or(false);
if args.dry_run {
println!("Dry run: siteforge would archive {url}");
println!("Output root: {}", output.display());
println!(
"Mode: {}",
if full_site {
"same-domain full-site crawl"
} else {
"single page"
}
);
if let Some(max_depth) = max_depth {
println!("Max depth: {max_depth}");
}
if let Some(max_pages) = max_pages {
println!("Max pages: {max_pages}");
}
if let Some(archive_profile) = archive_profile {
println!(
"Archive profile: {}",
site_archive_profile_label(archive_profile)
);
}
println!(
"Render JS: {}",
site_render_mode_label(render_js, render_js_auto)
);
return Ok(());
}
run_site_archive(SiteArchiveArgs {
url: Some(url),
input: None,
output,
full_site,
allow_cross_domain: false,
max_depth,
max_pages,
concurrency: Some(args.concurrency),
delay_ms: None,
archive_id: None,
include_url_patterns: Vec::new(),
exclude_url_patterns: Vec::new(),
archive_profile: archive_profile.map(SiteArchiveProfile::from),
max_page_size_bytes: None,
max_asset_size_bytes: None,
max_total_archive_size_bytes: None,
render_js,
render_js_auto,
browser_command: None,
browser_profile: None,
render_wait_ms: None,
headers: Vec::new(),
cookie: None,
user_agent: None,
timeout_secs: None,
retry_count: None,
ocr_enabled: false,
dry_run: false,
})
}
pub(crate) fn run_site_archive(args: SiteArchiveArgs) -> Result<()> {
let mut config = SiteforgeConfig {
archive_root: args.output.clone(),
..SiteforgeConfig::default()
};
if let Some(concurrency) = args.concurrency {
config.default_concurrency = concurrency.max(1);
}
if let Some(delay_ms) = args.delay_ms {
config.default_delay_ms = delay_ms;
}
if let Some(max_depth) = args.max_depth {
config.max_depth = max_depth;
}
if let Some(max_pages) = args.max_pages {
config.max_pages = max_pages;
}
if let Some(archive_profile) = args.archive_profile {
config.archive_profile = archive_profile.as_output_profile();
}
if let Some(max_page_size_bytes) = args.max_page_size_bytes {
config.max_page_size_bytes = max_page_size_bytes;
}
if let Some(max_asset_size_bytes) = args.max_asset_size_bytes {
config.max_asset_size_bytes = max_asset_size_bytes;
}
if let Some(max_total_archive_size_bytes) = args.max_total_archive_size_bytes {
config.max_total_archive_size_bytes = max_total_archive_size_bytes;
}
if args.render_js {
config.render_js = true;
config.render_js_auto = false;
}
if args.render_js_auto {
config.render_js = true;
config.render_js_auto = true;
}
if let Some(browser_command) = &args.browser_command {
config.browser_command = Some(browser_command.clone());
}
if let Some(browser_profile) = &args.browser_profile {
config.browser_profile = Some(browser_profile.clone());
}
if let Some(render_wait_ms) = args.render_wait_ms {
config.render_wait_ms = render_wait_ms;
}
if let Some(user_agent) = &args.user_agent {
config.user_agent = user_agent.clone();
}
if let Some(timeout_secs) = args.timeout_secs {
config.timeout_secs = timeout_secs;
}
if let Some(retry_count) = args.retry_count {
config.retry_count = retry_count;
}
if args.ocr_enabled {
config.ocr_enabled = true;
}
if !args.include_url_patterns.is_empty() {
config.include_url_patterns = args.include_url_patterns.clone();
}
if !args.exclude_url_patterns.is_empty() {
config
.exclude_url_patterns
.extend(args.exclude_url_patterns.clone());
}
let seeds = load_site_seed_urls(args.url.as_deref(), args.input.as_deref())?;
let source_target = site_source_target(&seeds);
let seed_display = site_seed_display(&seeds);
let headers = parse_site_headers(&args.headers)?;
let options = SiteforgeArchiveRunOptions {
seeds,
full_site: args.full_site,
same_domain: !args.allow_cross_domain,
max_depth: args.max_depth.unwrap_or(config.max_depth),
max_pages: args.max_pages.unwrap_or(config.max_pages),
concurrency: args
.concurrency
.unwrap_or(config.default_concurrency)
.max(1),
delay_ms: args.delay_ms.unwrap_or(config.default_delay_ms),
headers,
cookie: args.cookie.clone(),
archive_id: args.archive_id.clone(),
};
println!("Archiving website with siteforge: {seed_display}");
let runtime = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.map_err(|err| ThesaError::Message(format!("failed to start async runtime: {err}")))?;
let siteforge = Siteforge::new(config);
let summary = runtime
.block_on(siteforge.archive(options))
.map_err(|err| {
ThesaError::Message(siteforge_error_message(&err.to_string(), &args.output))
})?;
let manifest = read_archive_manifest_sidecar(&summary.archive_path).or_else(|_| {
write_archive_manifest(&summary.archive_path, "siteforge", &source_target, true)
})?;
print_siteforge_summary(
&summary.archive_id,
&summary.archive_path,
&manifest,
&summary.stats,
);
Ok(())
}
fn siteforge_error_message(message: &str, output_root: &Path) -> String {
if message.contains("already exists") {
format!(
"siteforge archive failed: {message}\n output root: {}\n suggestion: pass --archive-id with a new id, remove the existing archive, or use siteforge resume directly",
output_root.display()
)
} else {
format!("siteforge archive failed: {message}")
}
}
fn print_siteforge_summary(
archive_id: &str,
archive_path: &Path,
manifest: &ArchiveManifest,
stats: &SiteforgeCrawlStats,
) {
let layout = SiteforgeArchiveLayout::new(
archive_path
.parent()
.map(Path::to_path_buf)
.unwrap_or_else(|| archive_path.to_path_buf()),
archive_id.to_string(),
);
println!("Archive ID: {archive_id}");
println!("Archive: {}", archive_path.display());
println!("Open/read first:");
print_existing_path(" AGENTS.md", &layout.agents_md());
print_existing_path(" Agent index", &layout.agent_index_json());
print_existing_path(" Readable markdown", &layout.readable_markdown_dir());
print_existing_path(" Readable text", &layout.readable_text_dir());
print_existing_path(" Chunks JSONL", &layout.chunks_jsonl());
println!("Machine/index data:");
print_existing_path(" Siteforge manifest", &layout.manifest());
print_existing_path(" Siteforge checksums", &layout.checksums());
print_existing_path(" Crawl database", &layout.database());
print_existing_path(" Raw pages", &layout.raw_pages_dir());
print_existing_path(" Raw assets", &layout.raw_assets_dir());
println!("Pages parsed: {}", stats.pages_parsed);
println!("Pages failed: {}", stats.pages_failed);
println!("Assets downloaded: {}", stats.assets_downloaded);
println!(
"Manifest: {}/manifest.json ({} files, {} bytes)",
THESA_METADATA_DIR, manifest.archive.file_count, manifest.archive.total_bytes
);
}
fn print_existing_path(label: &str, path: &Path) {
if path.exists() {
println!("{label}: {}", path.display());
} else {
println!("{label}: {} (not written)", path.display());
}
}
pub(crate) fn site_render_mode_label(render_js: bool, render_js_auto: bool) -> &'static str {
match (render_js, render_js_auto) {
(_, true) => "auto",
(true, false) => "all",
(false, false) => "off",
}
}
pub(crate) fn site_archive_profile_label(profile: ArchiveOutputProfile) -> &'static str {
match profile {
ArchiveOutputProfile::Full => "full",
ArchiveOutputProfile::Agent => "agent",
ArchiveOutputProfile::Lean => "lean",
}
}
pub(crate) fn parse_site_url(input: &str) -> Result<reqwest::Url> {
let seed = input.trim();
let parsed = reqwest::Url::parse(seed)
.map_err(|err| ThesaError::Message(format!("invalid site URL '{seed}': {err}")))?;
if !matches!(parsed.scheme(), "http" | "https") {
return Err(ThesaError::Message(format!(
"unsupported site URL scheme '{}': expected http or https",
parsed.scheme()
)));
}
Ok(parsed)
}