use std::fs;
use std::io::{self, IsTerminal, Write};
use std::path::{Path, PathBuf};
use aisling::{Loader, LoaderConfig, LoaderKind, LoaderProgress};
use siteforge::{
ArchiveLayout as SiteforgeArchiveLayout, ArchiveOutputProfile, ArchiveProgress,
ArchiveProgressPhase, ArchiveRunOptions as SiteforgeArchiveRunOptions,
ArchiveSummary as SiteforgeArchiveSummary, Config as SiteforgeConfig,
CrawlStats as SiteforgeCrawlStats, Siteforge,
};
use crate::cli::{
Cli, SiteArchiveArgs, SiteforgeArchiveIdArgs, SiteforgeDiagnosticsArgs, SiteforgeExportArgs,
SiteforgePackArgs, SiteforgeRootArgs, SiteforgeSearchArgs,
};
use crate::constants::THESA_METADATA_DIR;
use crate::error::{Result, ThesaError};
use crate::manifest::{ArchiveManifest, read_archive_manifest_sidecar, write_archive_manifest};
use crate::terminal::TerminalSession;
use crate::types::{SiteArchiveProfile, SiteTuiOptions};
fn siteforge_config_from_root(root: &SiteforgeRootArgs) -> SiteforgeConfig {
SiteforgeConfig {
archive_root: root.archive_root.clone(),
..SiteforgeConfig::default()
}
}
fn siteforge_err(context: &str, err: impl std::fmt::Display) -> ThesaError {
ThesaError::Message(format!("siteforge {context} failed: {err}"))
}
pub(crate) fn run_siteforge_list(args: &SiteforgeRootArgs) -> Result<()> {
let config = siteforge_config_from_root(args);
let root = config
.resolved_archive_root()
.map_err(|err| siteforge_err("list", err))?;
let archives = Siteforge::new(config)
.list_archives()
.map_err(|err| siteforge_err("list", err))?;
if archives.is_empty() {
println!("No Siteforge archives found at {}", root.display());
return Ok(());
}
for archive in archives {
println!(
"{} updated={} pages={}/{} failed={} seeds={}",
archive.archive_id,
archive.updated_at,
archive.stats.pages_parsed,
archive.stats.pages_discovered,
archive.stats.pages_failed,
archive.seeds.join(", ")
);
}
Ok(())
}
pub(crate) fn run_siteforge_inspect(args: &SiteforgeArchiveIdArgs) -> Result<()> {
let config = siteforge_config_from_root(&args.root);
let manifest = siteforge::archive::load_archive_manifest(&config, &args.archive_id)
.map_err(|err| siteforge_err("inspect", err))?;
let layout = SiteforgeArchiveLayout::new(
config
.resolved_archive_root()
.map_err(|err| siteforge_err("inspect", err))?,
args.archive_id.clone(),
);
println!("archive_id: {}", manifest.archive_id);
println!("created_at: {}", manifest.created_at);
println!("updated_at: {}", manifest.updated_at);
println!("path: {}", layout.base().display());
println!("seeds: {}", manifest.seeds.join(", "));
println!("full_site: {}", manifest.scope.full_site);
println!("same_domain: {}", manifest.scope.same_domain);
println!("max_depth: {}", manifest.scope.max_depth);
println!("archive_profile: {:?}", manifest.config.archive_profile);
println!("render_js: {}", manifest.config.render_js);
println!("render_js_auto: {}", manifest.config.render_js_auto);
println!("pages_discovered: {}", manifest.stats.pages_discovered);
println!("pages_parsed: {}", manifest.stats.pages_parsed);
println!("pages_failed: {}", manifest.stats.pages_failed);
println!("assets_downloaded: {}", manifest.stats.assets_downloaded);
if layout.database().exists() {
let storage = siteforge::storage::Storage::open(&layout.database())
.map_err(|err| siteforge_err("inspect", err))?;
println!(
"pages_indexed: {}",
storage
.page_count()
.map_err(|err| siteforge_err("inspect", err))?
);
println!(
"assets_indexed: {}",
storage
.asset_count()
.map_err(|err| siteforge_err("inspect", err))?
);
println!(
"asset_bytes: {}",
storage
.total_asset_size_bytes()
.map_err(|err| siteforge_err("inspect", err))?
);
let recent_pages = storage
.recent_pages_limited(5)
.map_err(|err| siteforge_err("inspect", err))?;
if !recent_pages.is_empty() {
println!("recent_pages:");
for page in recent_pages {
println!(" - [{}] {} ({})", page.status, page.title, page.source_url);
}
}
}
Ok(())
}
pub(crate) fn run_siteforge_search(args: &SiteforgeSearchArgs) -> Result<()> {
let config = siteforge_config_from_root(&args.root);
let results = Siteforge::new(config)
.search(args.archive_id.as_deref(), &args.query, args.limit)
.map_err(|err| siteforge_err("search", err))?;
if results.is_empty() {
println!("No matches.");
return Ok(());
}
for result in results {
println!("{} | {}", result.archive_id, result.title);
println!(" {}", result.source_url);
if let Some(heading) = result.matched_heading.as_deref() {
println!(" heading: {heading}");
}
println!(" snippet: {}", result.snippet.replace('\n', " "));
if let Some(path) = result.local_file_path.as_deref() {
println!(" local: {path}");
}
}
Ok(())
}
pub(crate) fn run_siteforge_export(args: &SiteforgeExportArgs, dry_run: bool) -> Result<()> {
if dry_run {
println!("Dry run: siteforge would export {}", args.archive_id);
println!("Archive root: {}", args.root.archive_root.display());
println!("Format: {}", args.format.as_str());
if let Some(output) = &args.output {
println!("Output: {}", output.display());
}
return Ok(());
}
let config = siteforge_config_from_root(&args.root);
let path = Siteforge::new(config)
.export(&args.archive_id, args.format, args.output.as_deref())
.map_err(|err| siteforge_err("export", err))?;
println!("Exported: {}", path.display());
Ok(())
}
pub(crate) fn run_siteforge_pack(args: &SiteforgePackArgs, dry_run: bool) -> Result<()> {
if dry_run {
println!(
"Dry run: siteforge would create AI packs for {}",
args.archive_id
);
println!("Archive root: {}", args.root.archive_root.display());
if let Some(max_tokens) = args.max_tokens {
println!("Max tokens: {max_tokens}");
}
return Ok(());
}
let config = siteforge_config_from_root(&args.root);
let packs = Siteforge::new(config)
.create_ai_packs(&args.archive_id, args.max_tokens)
.map_err(|err| siteforge_err("pack", err))?;
for pack in packs {
println!("Pack: {}", pack.display());
}
Ok(())
}
pub(crate) fn run_siteforge_verify(args: &SiteforgeArchiveIdArgs) -> Result<()> {
let config = siteforge_config_from_root(&args.root);
let problems = Siteforge::new(config)
.verify(&args.archive_id)
.map_err(|err| siteforge_err("verify", err))?;
if problems.is_empty() {
println!("Siteforge archive verified: {}", args.archive_id);
} else {
println!(
"Siteforge archive verification found {} problem(s):",
problems.len()
);
for problem in problems {
println!("- {problem}");
}
}
Ok(())
}
pub(crate) fn run_siteforge_resume(args: &SiteforgeArchiveIdArgs, dry_run: bool) -> Result<()> {
if dry_run {
println!("Dry run: siteforge would resume {}", args.archive_id);
println!("Archive root: {}", args.root.archive_root.display());
return Ok(());
}
let config = siteforge_config_from_root(&args.root);
let runtime = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.map_err(|err| ThesaError::Message(format!("failed to start async runtime: {err}")))?;
let summary = runtime
.block_on(Siteforge::new(config).resume(&args.archive_id))
.map_err(|err| siteforge_err("resume", err))?;
println!("Resumed archive: {}", summary.archive_id);
println!("Archive: {}", summary.archive_path.display());
println!(
"Pages: fetched={} parsed={} failed={}",
summary.stats.pages_fetched, summary.stats.pages_parsed, summary.stats.pages_failed
);
Ok(())
}
pub(crate) fn run_siteforge_diagnostics(args: &SiteforgeDiagnosticsArgs) -> Result<()> {
let config = siteforge_config_from_root(&args.root);
let root = config
.resolved_archive_root()
.map_err(|err| siteforge_err("diagnostics", err))?;
println!("archive_root: {}", root.display());
println!("default_concurrency: {}", config.default_concurrency);
println!("archive_profile: {:?}", config.archive_profile);
println!("render_js: {}", config.render_js);
println!("render_js_auto: {}", config.render_js_auto);
println!("max_pages: {}", config.max_pages);
println!("max_page_size_bytes: {}", config.max_page_size_bytes);
println!("max_asset_size_bytes: {}", config.max_asset_size_bytes);
println!(
"max_total_archive_size_bytes: {}",
config.max_total_archive_size_bytes
);
if let Some(archive_id) = &args.archive_id {
let manifest = siteforge::archive::load_archive_manifest(&config, archive_id)
.map_err(|err| siteforge_err("diagnostics", err))?;
let layout = SiteforgeArchiveLayout::new(root, archive_id.clone());
println!("archive_id: {}", manifest.archive_id);
println!("archive_path: {}", layout.base().display());
println!(
"archive_size_bytes: {}",
siteforge::archive::archive_size_bytes(&layout)
.map_err(|err| siteforge_err("diagnostics", err))?
);
if layout.database().exists() {
let storage = siteforge::storage::Storage::open(&layout.database())
.map_err(|err| siteforge_err("diagnostics", err))?;
println!(
"jobs: {}",
storage
.job_count()
.map_err(|err| siteforge_err("diagnostics", err))?
);
for job in storage
.job_snapshots_limited(20)
.map_err(|err| siteforge_err("diagnostics", err))?
{
println!(
"- job={} status={} active_url={}",
job.job_id,
job.status,
job.active_url.as_deref().unwrap_or("none")
);
}
}
return Ok(());
}
let archives = Siteforge::new(config)
.list_archives()
.map_err(|err| siteforge_err("diagnostics", err))?;
println!("archives: {}", archives.len());
for archive in archives.into_iter().take(50) {
println!(
"- {} updated={} parsed={} failed={}",
archive.archive_id,
archive.updated_at,
archive.stats.pages_parsed,
archive.stats.pages_failed
);
}
Ok(())
}
pub(crate) fn print_site_archive_dry_run(args: &SiteArchiveArgs) -> Result<()> {
let seeds = load_site_seed_urls(args.url.as_deref(), args.input.as_deref())?;
let headers = parse_site_headers(&args.headers)?;
println!(
"Dry run: siteforge would archive {}",
site_seed_display(&seeds)
);
println!("Output root: {}", args.output.display());
if let Some(input) = &args.input {
println!("Input seeds: {}", input.display());
}
println!(
"Mode: {}",
if args.full_site {
"same-domain full-site crawl"
} else {
"single page"
}
);
println!(
"Cross-domain: {}",
if args.allow_cross_domain {
"allowed"
} else {
"blocked"
}
);
println!(
"Max depth: {}",
args.max_depth
.map(|value| value.to_string())
.unwrap_or_else(|| "config default".to_string())
);
println!(
"Max pages: {}",
args.max_pages
.map(|value| value.to_string())
.unwrap_or_else(|| "config default".to_string())
);
println!(
"Concurrency: {}",
args.concurrency
.map(|value| value.to_string())
.unwrap_or_else(|| "config default".to_string())
);
println!(
"Delay: {}",
args.delay_ms
.map(|value| format!("{value} ms"))
.unwrap_or_else(|| "config default".to_string())
);
if !args.include_url_patterns.is_empty() {
println!("Include globs: {}", args.include_url_patterns.join(", "));
}
if !args.exclude_url_patterns.is_empty() {
println!("Exclude globs: {}", args.exclude_url_patterns.join(", "));
}
if let Some(archive_profile) = args.archive_profile {
println!(
"Archive profile: {}",
site_archive_profile_label(archive_profile.as_output_profile())
);
}
if let Some(max_page_size_bytes) = args.max_page_size_bytes {
println!("Max page size: {max_page_size_bytes} bytes");
}
if let Some(max_asset_size_bytes) = args.max_asset_size_bytes {
println!("Max asset size: {max_asset_size_bytes} bytes");
}
if let Some(max_total_archive_size_bytes) = args.max_total_archive_size_bytes {
println!("Max total archive size: {max_total_archive_size_bytes} bytes");
}
println!(
"Render JS: {}",
site_render_mode_label(args.render_js, args.render_js_auto)
);
if let Some(browser_command) = &args.browser_command {
println!("Browser command: {browser_command}");
}
if let Some(browser_profile) = &args.browser_profile {
println!("Browser profile: {}", browser_profile.display());
}
if let Some(render_wait_ms) = args.render_wait_ms {
println!("Render wait: {render_wait_ms} ms");
}
if let Some(user_agent) = &args.user_agent {
println!("User-Agent: {user_agent}");
}
if let Some(timeout_secs) = args.timeout_secs {
println!("Timeout: {timeout_secs} seconds");
}
if let Some(retry_count) = args.retry_count {
println!("Retries: {retry_count}");
}
println!(
"OCR: {}",
if args.ocr_enabled {
"enabled"
} else {
"config default"
}
);
if !headers.is_empty() {
let names = headers
.iter()
.map(|(name, _)| name.as_str())
.collect::<Vec<_>>()
.join(", ");
println!("Headers: {} configured ({names})", headers.len());
}
if args.cookie.is_some() {
println!("Cookie: configured (redacted)");
}
if let Some(archive_id) = &args.archive_id {
println!("Archive ID: {archive_id}");
}
Ok(())
}
pub(crate) fn load_site_seed_urls(
primary: Option<&str>,
input: Option<&Path>,
) -> Result<Vec<reqwest::Url>> {
let mut seeds = Vec::new();
if let Some(primary) = primary {
seeds.push(parse_site_url(primary)?);
}
if let Some(input) = input {
let raw = fs::read_to_string(input)?;
for (line_index, line) in raw.lines().enumerate() {
let line = line.trim();
if line.is_empty() || line.starts_with('#') {
continue;
}
let seed = parse_site_url(line).map_err(|err| {
ThesaError::Message(format!(
"invalid site URL in {} at line {}: {err}",
input.display(),
line_index + 1
))
})?;
seeds.push(seed);
}
}
if seeds.is_empty() {
return Err(ThesaError::Message(
"siteforge archive requires a URL or --input seed file".to_string(),
));
}
Ok(seeds)
}
fn site_seed_display(seeds: &[reqwest::Url]) -> String {
match seeds {
[] => "no seeds".to_string(),
[seed] => seed.as_str().to_string(),
[seed, rest @ ..] => format!("{} (+{} more seeds)", seed.as_str(), rest.len()),
}
}
fn site_source_target(seeds: &[reqwest::Url]) -> String {
match seeds {
[] => "siteforge".to_string(),
[seed] => seed.as_str().to_string(),
[seed, rest @ ..] => format!("{} (+{} seeds)", seed.as_str(), rest.len()),
}
}
pub(crate) fn parse_site_headers(values: &[String]) -> Result<Vec<(String, String)>> {
values
.iter()
.map(|value| {
let (name, header_value) = value.split_once(':').ok_or_else(|| {
ThesaError::Message(format!("invalid header {value:?}; expected 'Name: Value'"))
})?;
let name = name.trim();
let header_value = header_value.trim();
if name.is_empty() || header_value.is_empty() {
return Err(ThesaError::Message(format!(
"invalid header {value:?}; name and value are required"
)));
}
Ok((name.to_string(), header_value.to_string()))
})
.collect()
}
pub(crate) fn run_sites_mode(
args: &mut Cli,
terminal_session: &mut Option<TerminalSession>,
tui_options: Option<SiteTuiOptions>,
) -> Result<()> {
let url = args
.target
.as_deref()
.unwrap_or_default()
.trim()
.to_string();
parse_site_url(&url)?;
*terminal_session = None;
let output = if args.output == Path::new("./archives") {
PathBuf::from("./archives/sites")
} else {
args.output.clone()
};
let full_site = tui_options.map(|options| options.full_site).unwrap_or(true);
let max_depth = tui_options.map(|options| options.max_depth);
let max_pages = tui_options.map(|options| options.max_pages);
let archive_profile = tui_options.map(|options| options.archive_profile);
let render_js = tui_options
.map(|options| options.render_js)
.unwrap_or(false);
let render_js_auto = tui_options
.map(|options| options.render_js_auto)
.unwrap_or(false);
if args.dry_run {
println!("Dry run: siteforge would archive {url}");
println!("Output root: {}", output.display());
println!(
"Mode: {}",
if full_site {
"same-domain full-site crawl"
} else {
"single page"
}
);
if let Some(max_depth) = max_depth {
println!("Max depth: {max_depth}");
}
if let Some(max_pages) = max_pages {
println!("Max pages: {max_pages}");
}
if let Some(archive_profile) = archive_profile {
println!(
"Archive profile: {}",
site_archive_profile_label(archive_profile)
);
}
println!(
"Render JS: {}",
site_render_mode_label(render_js, render_js_auto)
);
return Ok(());
}
run_site_archive(SiteArchiveArgs {
url: Some(url),
input: None,
output,
full_site,
allow_cross_domain: false,
max_depth,
max_pages,
concurrency: Some(args.concurrency),
delay_ms: None,
archive_id: None,
include_url_patterns: Vec::new(),
exclude_url_patterns: Vec::new(),
archive_profile: archive_profile.map(SiteArchiveProfile::from),
max_page_size_bytes: None,
max_asset_size_bytes: None,
max_total_archive_size_bytes: None,
render_js,
render_js_auto,
browser_command: None,
browser_profile: None,
render_wait_ms: None,
headers: Vec::new(),
cookie: None,
user_agent: None,
timeout_secs: None,
retry_count: None,
ocr_enabled: false,
dry_run: false,
})
}
pub(crate) fn run_site_archive(args: SiteArchiveArgs) -> Result<()> {
let mut config = SiteforgeConfig {
archive_root: args.output.clone(),
..SiteforgeConfig::default()
};
if let Some(concurrency) = args.concurrency {
config.default_concurrency = concurrency.max(1);
}
if let Some(delay_ms) = args.delay_ms {
config.default_delay_ms = delay_ms;
}
if let Some(max_depth) = args.max_depth {
config.max_depth = max_depth;
}
if let Some(max_pages) = args.max_pages {
config.max_pages = max_pages;
}
if let Some(archive_profile) = args.archive_profile {
config.archive_profile = archive_profile.as_output_profile();
}
if let Some(max_page_size_bytes) = args.max_page_size_bytes {
config.max_page_size_bytes = max_page_size_bytes;
}
if let Some(max_asset_size_bytes) = args.max_asset_size_bytes {
config.max_asset_size_bytes = max_asset_size_bytes;
}
if let Some(max_total_archive_size_bytes) = args.max_total_archive_size_bytes {
config.max_total_archive_size_bytes = max_total_archive_size_bytes;
}
if args.render_js {
config.render_js = true;
config.render_js_auto = false;
}
if args.render_js_auto {
config.render_js = true;
config.render_js_auto = true;
}
if let Some(browser_command) = &args.browser_command {
config.browser_command = Some(browser_command.clone());
}
if let Some(browser_profile) = &args.browser_profile {
config.browser_profile = Some(browser_profile.clone());
}
if let Some(render_wait_ms) = args.render_wait_ms {
config.render_wait_ms = render_wait_ms;
}
if let Some(user_agent) = &args.user_agent {
config.user_agent = user_agent.clone();
}
if let Some(timeout_secs) = args.timeout_secs {
config.timeout_secs = timeout_secs;
}
if let Some(retry_count) = args.retry_count {
config.retry_count = retry_count;
}
if args.ocr_enabled {
config.ocr_enabled = true;
}
if !args.include_url_patterns.is_empty() {
config.include_url_patterns = args.include_url_patterns.clone();
}
if !args.exclude_url_patterns.is_empty() {
config
.exclude_url_patterns
.extend(args.exclude_url_patterns.clone());
}
let seeds = load_site_seed_urls(args.url.as_deref(), args.input.as_deref())?;
let source_target = site_source_target(&seeds);
let seed_display = site_seed_display(&seeds);
let headers = parse_site_headers(&args.headers)?;
let options = SiteforgeArchiveRunOptions {
seeds,
full_site: args.full_site,
same_domain: !args.allow_cross_domain,
max_depth: args.max_depth.unwrap_or(config.max_depth),
max_pages: args.max_pages.unwrap_or(config.max_pages),
concurrency: args
.concurrency
.unwrap_or(config.default_concurrency)
.max(1),
delay_ms: args.delay_ms.unwrap_or(config.default_delay_ms),
headers,
cookie: args.cookie.clone(),
archive_id: args.archive_id.clone(),
};
println!("Archiving website with siteforge: {seed_display}");
let runtime = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.map_err(|err| ThesaError::Message(format!("failed to start async runtime: {err}")))?;
let siteforge = Siteforge::new(config);
let summary =
run_siteforge_archive_with_output(&runtime, siteforge, options).map_err(|err| {
ThesaError::Message(siteforge_error_message(&err.to_string(), &args.output))
})?;
let manifest = read_archive_manifest_sidecar(&summary.archive_path).or_else(|_| {
write_archive_manifest(&summary.archive_path, "siteforge", &source_target, true)
})?;
print_siteforge_summary(
&summary.archive_id,
&summary.archive_path,
&manifest,
&summary.stats,
);
Ok(())
}
fn run_siteforge_archive_with_output(
runtime: &tokio::runtime::Runtime,
siteforge: Siteforge,
options: SiteforgeArchiveRunOptions,
) -> siteforge::Result<SiteforgeArchiveSummary> {
if !io::stderr().is_terminal() {
return runtime.block_on(siteforge.archive(options));
}
runtime.block_on(async move {
let (progress_tx, progress_rx) = tokio::sync::mpsc::channel(256);
let renderer = tokio::spawn(render_siteforge_progress(progress_rx));
let result = siteforge.archive_with_progress(options, progress_tx).await;
let _ = renderer.await;
result
})
}
async fn render_siteforge_progress(mut rx: tokio::sync::mpsc::Receiver<ArchiveProgress>) {
let settings = SiteforgeProgressSettings::detect();
let mut latest = None;
let mut interval = tokio::time::interval(std::time::Duration::from_millis(120));
let mut stderr = io::stderr();
let loader =
siteforge_progress_loader(settings.use_color, siteforge_loader_width(settings.width));
let mut tick = 0usize;
let mut wrote = false;
let _ = write!(stderr, "\x1b[?25l");
loop {
let should_render = tokio::select! {
progress = rx.recv() => {
let Some(progress) = progress else {
break;
};
latest = Some(progress);
true
}
_ = interval.tick() => true,
};
if should_render {
if let Some(progress) = latest.as_ref() {
write_siteforge_progress(&mut stderr, &loader, tick, progress, settings);
tick = tick.wrapping_add(1);
wrote = true;
}
}
}
if let Some(progress) = latest.as_ref() {
write_siteforge_progress(&mut stderr, &loader, tick, progress, settings);
wrote = true;
}
if wrote {
let _ = writeln!(stderr);
}
let _ = write!(stderr, "\x1b[?25h");
let _ = stderr.flush();
}
#[derive(Debug, Clone, Copy)]
struct SiteforgeProgressSettings {
use_color: bool,
width: usize,
}
impl SiteforgeProgressSettings {
fn detect() -> Self {
Self {
use_color: siteforge_color_enabled(),
width: terminal_width(),
}
}
}
fn siteforge_progress_loader(use_color: bool, width: usize) -> Loader {
let config = LoaderConfig::default()
.with_width(width)
.with_label("siteforge")
.with_unit("pages")
.with_fraction(true);
let config = if use_color {
config
} else {
config.without_color()
};
Loader::with_config(LoaderKind::Tqdm, config)
}
fn write_siteforge_progress(
stderr: &mut io::Stderr,
loader: &Loader,
tick: usize,
progress: &ArchiveProgress,
settings: SiteforgeProgressSettings,
) {
let (current, total) = siteforge_progress_counts(progress);
let loader_line = if settings.use_color {
loader
.frame(tick, LoaderProgress::from_counts(current, total))
.to_ansi_string()
} else {
loader.line(tick, LoaderProgress::from_counts(current, total))
};
let line = siteforge_progress_line(&loader_line, progress, settings);
let _ = write!(
stderr,
"\r\x1b[2K{}",
truncate_to_visible_width(&line, settings.width.saturating_sub(1))
);
let _ = stderr.flush();
}
fn siteforge_progress_line(
loader_line: &str,
progress: &ArchiveProgress,
settings: SiteforgeProgressSettings,
) -> String {
let phase = colorize_progress(
progress.phase.as_str(),
siteforge_phase_color(progress.phase),
settings.use_color,
);
let archive = truncate_middle(&progress.archive_id, 18);
let stats = if settings.width >= 104 {
format!(
"a={} p={} f={} s={} assets={}/{}",
archive,
progress.stats.pages_parsed,
progress.stats.pages_failed,
progress.stats.pages_skipped,
progress.stats.assets_downloaded,
progress.stats.assets_discovered,
)
} else {
format!(
"p={} f={} a={}/{}",
progress.stats.pages_parsed,
progress.stats.pages_failed,
progress.stats.assets_downloaded,
progress.stats.assets_discovered,
)
};
let hint = (settings.width >= 110)
.then(|| colorize_progress("Ctrl+C saves resume", "2", settings.use_color));
let active_budget = settings.width.saturating_sub(
visible_width(loader_line)
.saturating_add(progress.phase.as_str().len())
.saturating_add(stats.len())
.saturating_add(hint.as_deref().map(visible_width).unwrap_or(0))
.saturating_add(20),
);
let active = progress
.active_url
.as_deref()
.filter(|_| active_budget >= 16)
.map(|url| format!(" | {}", truncate_middle(url, active_budget.min(70))))
.unwrap_or_default();
let mut line = format!(
"{} | {} | {}{}",
loader_line.trim_end(),
phase,
stats,
active
);
if let Some(hint) = hint {
line.push_str(" | ");
line.push_str(&hint);
}
line
}
fn siteforge_progress_counts(progress: &ArchiveProgress) -> (u64, u64) {
let processed = progress
.stats
.pages_parsed
.saturating_add(progress.stats.pages_failed)
.saturating_add(progress.stats.pages_skipped);
let discovered = progress.stats.pages_discovered.max(processed).max(1);
let capped_total = if progress.max_pages == 0 {
discovered
} else {
discovered.min(progress.max_pages.max(1))
};
if matches!(
progress.phase,
ArchiveProgressPhase::Completed | ArchiveProgressPhase::Interrupted
) {
let final_total = processed.max(1);
(final_total as u64, final_total as u64)
} else {
(processed.min(capped_total) as u64, capped_total as u64)
}
}
fn siteforge_loader_width(terminal_width: usize) -> usize {
match terminal_width {
0..=79 => 28,
80..=119 => 34,
_ => 46,
}
}
fn terminal_width() -> usize {
std::env::var("COLUMNS")
.ok()
.and_then(|value| value.parse::<usize>().ok())
.filter(|value| *value >= 40)
.unwrap_or(120)
}
fn siteforge_color_enabled() -> bool {
std::env::var_os("NO_COLOR").is_none()
&& std::env::var("TERM")
.map(|term| term != "dumb")
.unwrap_or(true)
}
fn siteforge_phase_color(phase: ArchiveProgressPhase) -> &'static str {
match phase {
ArchiveProgressPhase::Completed => "32;1",
ArchiveProgressPhase::Interrupted => "33;1",
ArchiveProgressPhase::Processing => "36;1",
ArchiveProgressPhase::Fetching => "34;1",
ArchiveProgressPhase::Discovering => "35;1",
ArchiveProgressPhase::Starting => "37;1",
}
}
fn colorize_progress(text: &str, code: &str, enabled: bool) -> String {
if enabled {
format!("\x1b[{code}m{text}\x1b[0m")
} else {
text.to_string()
}
}
fn truncate_middle(input: &str, max_chars: usize) -> String {
let chars = input.chars().collect::<Vec<_>>();
if chars.len() <= max_chars {
return input.to_string();
}
let keep = max_chars.saturating_sub(3);
let left = keep / 2;
let right = keep.saturating_sub(left);
let mut output = chars.iter().take(left).collect::<String>();
output.push_str("...");
output.push_str(
&chars[chars.len().saturating_sub(right)..]
.iter()
.collect::<String>(),
);
output
}
fn truncate_to_visible_width(input: &str, max_width: usize) -> String {
if visible_width(input) <= max_width {
return input.to_string();
}
let mut output = String::new();
let mut visible = 0usize;
let mut chars = input.chars().peekable();
while let Some(ch) = chars.next() {
if ch == '\x1b' {
output.push(ch);
for next in chars.by_ref() {
output.push(next);
if next == 'm' {
break;
}
}
continue;
}
if visible + 1 >= max_width.saturating_sub(1) {
output.push_str("...");
break;
}
output.push(ch);
visible += 1;
}
if input.contains("\x1b[") {
output.push_str("\x1b[0m");
}
output
}
fn visible_width(input: &str) -> usize {
let mut width = 0usize;
let mut chars = input.chars().peekable();
while let Some(ch) = chars.next() {
if ch == '\x1b' {
for next in chars.by_ref() {
if next == 'm' {
break;
}
}
continue;
}
width += 1;
}
width
}
fn siteforge_error_message(message: &str, output_root: &Path) -> String {
if message.contains("already exists") {
format!(
"siteforge archive failed: {message}\n output root: {}\n suggestion: pass --archive-id with a new id, remove the existing archive, or use siteforge resume directly",
output_root.display()
)
} else {
format!("siteforge archive failed: {message}")
}
}
fn print_siteforge_summary(
archive_id: &str,
archive_path: &Path,
manifest: &ArchiveManifest,
stats: &SiteforgeCrawlStats,
) {
let layout = SiteforgeArchiveLayout::new(
archive_path
.parent()
.map(Path::to_path_buf)
.unwrap_or_else(|| archive_path.to_path_buf()),
archive_id.to_string(),
);
println!("Archive ID: {archive_id}");
println!("Archive: {}", archive_path.display());
println!("Open/read first:");
print_existing_path(" AGENTS.md", &layout.agents_md());
print_existing_path(" Agent index", &layout.agent_index_json());
print_existing_path(" Readable markdown", &layout.readable_markdown_dir());
print_existing_path(" Readable text", &layout.readable_text_dir());
print_existing_path(" Chunks JSONL", &layout.chunks_jsonl());
println!("Machine/index data:");
print_existing_path(" Siteforge manifest", &layout.manifest());
print_existing_path(" Siteforge checksums", &layout.checksums());
print_existing_path(" Crawl database", &layout.database());
print_existing_path(" Raw pages", &layout.raw_pages_dir());
print_existing_path(" Raw assets", &layout.raw_assets_dir());
println!("Pages parsed: {}", stats.pages_parsed);
println!("Pages failed: {}", stats.pages_failed);
println!("Assets downloaded: {}", stats.assets_downloaded);
println!(
"Manifest: {}/manifest.json ({} files, {} bytes)",
THESA_METADATA_DIR, manifest.archive.file_count, manifest.archive.total_bytes
);
}
fn print_existing_path(label: &str, path: &Path) {
if path.exists() {
println!("{label}: {}", path.display());
} else {
println!("{label}: {} (not written)", path.display());
}
}
pub(crate) fn site_render_mode_label(render_js: bool, render_js_auto: bool) -> &'static str {
match (render_js, render_js_auto) {
(_, true) => "auto",
(true, false) => "all",
(false, false) => "off",
}
}
pub(crate) fn site_archive_profile_label(profile: ArchiveOutputProfile) -> &'static str {
match profile {
ArchiveOutputProfile::Full => "full",
ArchiveOutputProfile::Agent => "agent",
ArchiveOutputProfile::Lean => "lean",
}
}
pub(crate) fn parse_site_url(input: &str) -> Result<reqwest::Url> {
let seed = input.trim();
let parsed = reqwest::Url::parse(seed)
.map_err(|err| ThesaError::Message(format!("invalid site URL '{seed}': {err}")))?;
if !matches!(parsed.scheme(), "http" | "https") {
return Err(ThesaError::Message(format!(
"unsupported site URL scheme '{}': expected http or https",
parsed.scheme()
)));
}
Ok(parsed)
}