use std::fs;
use std::io::{self, IsTerminal, Write};
use std::path::{Path, PathBuf};
use std::process::Command;
use std::time::Duration;
use aisling::{Loader, LoaderConfig, LoaderKind, LoaderProgress};
use clap::{Args, Parser, Subcommand, ValueEnum};
use tokio::sync::mpsc;
use url::Url;
use crate::archive::{
archive_size_bytes, list_archives, load_archive_manifest, verify_archive, ArchiveLayout,
};
use crate::config::Config;
use crate::crawler::{
self, ArchiveProgress, ArchiveProgressPhase, ArchiveRunOptions, ArchiveSummary,
};
use crate::errors::{Result, SiteforgeError};
use crate::export::{self, ExportFormat};
use crate::search::search_archives;
use crate::storage::Storage;
const CLI_PROGRESS_CHANNEL_CAPACITY: usize = 32;
#[derive(Debug, Parser)]
#[command(
name = "siteforge",
version,
about = "Archive websites into AI-readable local knowledge archives"
)]
pub struct Cli {
#[arg(long, global = true, value_name = "PATH")]
pub config: Option<PathBuf>,
#[command(subcommand)]
pub command: Commands,
}
#[derive(Debug, Subcommand)]
pub enum Commands {
Archive(Box<ArchiveCommand>),
Tui,
List,
Inspect {
archive_id: String,
},
Search {
query: String,
#[arg(long)]
archive_id: Option<String>,
#[arg(long, default_value_t = 20)]
limit: usize,
},
Export {
archive_id: String,
#[arg(long, value_enum)]
format: ExportFormat,
#[arg(long)]
output: Option<PathBuf>,
},
Pack {
archive_id: String,
#[arg(long)]
for_ai: bool,
#[arg(long)]
max_tokens: Option<usize>,
},
Verify {
archive_id: String,
},
Resume {
job_id: String,
#[arg(long, value_enum, default_value = "auto")]
progress: CliProgressMode,
},
Diagnostics {
#[arg(long)]
archive_id: Option<String>,
},
Config {
#[arg(long)]
edit: bool,
},
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
pub enum CliProgressMode {
Auto,
Always,
Never,
}
#[derive(Debug, Args)]
pub struct ArchiveCommand {
pub url: Option<String>,
#[arg(long, value_name = "FILE")]
pub input: Option<PathBuf>,
#[arg(long, value_name = "ARCHIVE_ID")]
pub archive_id: Option<String>,
#[arg(long)]
pub full_site: bool,
#[arg(long)]
pub allow_cross_domain: bool,
#[arg(short = 'j', long, alias = "jobs")]
pub concurrency: Option<usize>,
#[arg(long)]
pub delay_ms: Option<u64>,
#[arg(long)]
pub max_depth: Option<usize>,
#[arg(long)]
pub max_pages: Option<usize>,
#[arg(long = "include", value_name = "GLOB")]
pub include_url_patterns: Vec<String>,
#[arg(long = "exclude", value_name = "GLOB")]
pub exclude_url_patterns: Vec<String>,
#[arg(long)]
pub max_asset_size_bytes: Option<u64>,
#[arg(long)]
pub max_total_archive_size_bytes: Option<u64>,
#[arg(long)]
pub render_js: bool,
#[arg(long, value_name = "COMMAND")]
pub browser_command: Option<String>,
#[arg(long, value_name = "DIR")]
pub browser_profile: Option<PathBuf>,
#[arg(long)]
pub render_wait_ms: Option<u64>,
#[arg(long = "header", value_name = "NAME: VALUE")]
pub headers: Vec<String>,
#[arg(long, value_name = "COOKIE_HEADER")]
pub cookie: Option<String>,
#[arg(long, value_enum, default_value = "auto")]
pub progress: CliProgressMode,
}
pub async fn run_archive(command: ArchiveCommand, mut config: Config) -> Result<()> {
let seeds = load_seed_urls(command.url.as_deref(), command.input.as_deref())?;
if !command.include_url_patterns.is_empty() {
config.include_url_patterns = command.include_url_patterns.clone();
}
if !command.exclude_url_patterns.is_empty() {
config
.exclude_url_patterns
.extend(command.exclude_url_patterns.clone());
}
if let Some(max_asset_size_bytes) = command.max_asset_size_bytes {
config.max_asset_size_bytes = max_asset_size_bytes;
}
if let Some(max_total_archive_size_bytes) = command.max_total_archive_size_bytes {
config.max_total_archive_size_bytes = max_total_archive_size_bytes;
}
if command.render_js {
config.render_js = true;
}
if command.browser_command.is_some() {
config.browser_command = command.browser_command.clone();
}
if command.browser_profile.is_some() {
config.browser_profile = command.browser_profile.clone();
}
if let Some(render_wait_ms) = command.render_wait_ms {
config.render_wait_ms = render_wait_ms;
}
let options = ArchiveRunOptions {
seeds,
full_site: command.full_site,
same_domain: !command.allow_cross_domain,
max_depth: command.max_depth.unwrap_or(config.max_depth),
max_pages: command.max_pages.unwrap_or(config.max_pages),
concurrency: command
.concurrency
.unwrap_or(config.default_concurrency)
.max(1),
delay_ms: command.delay_ms.unwrap_or(config.default_delay_ms),
headers: parse_headers(&command.headers)?,
cookie: command.cookie,
archive_id: command.archive_id,
};
match run_archive_with_cli_progress(config, options, command.progress).await {
Ok(summary) => print_archive_summary(&summary),
Err(SiteforgeError::ArchiveInterrupted(archive_id)) => {
print_interrupted_archive(&archive_id);
Ok(())
}
Err(err) => Err(err),
}
}
pub async fn run_list(config: Config) -> Result<()> {
let archives = list_archives(&config)?;
if archives.is_empty() {
println!(
"No archives found at {}",
config.resolved_archive_root()?.display()
);
return Ok(());
}
for archive in archives {
println!(
"{} updated={} pages={}/{} failed={} seeds={}",
archive.archive_id,
archive.updated_at,
archive.stats.pages_parsed,
archive.stats.pages_discovered,
archive.stats.pages_failed,
archive.seeds.join(", ")
);
}
Ok(())
}
pub async fn run_inspect(config: Config, archive_id: &str) -> Result<()> {
let manifest = load_archive_manifest(&config, archive_id)?;
let layout = ArchiveLayout::new(config.resolved_archive_root()?, archive_id.to_string());
let storage = Storage::open(&layout.database())?;
let stats = storage.crawl_stats()?;
let pages = storage.list_pages()?;
let assets = storage.list_assets()?;
let failed_pages = pages
.iter()
.filter(|page| page.error.is_some() || page.status >= 400)
.collect::<Vec<_>>();
let total_asset_bytes: u64 = assets.iter().filter_map(|asset| asset.size_bytes).sum();
println!("archive_id: {}", manifest.archive_id);
println!("created_at: {}", manifest.created_at);
println!("updated_at: {}", manifest.updated_at);
println!("path: {}", layout.base().display());
println!("seeds: {}", manifest.seeds.join(", "));
println!("full_site: {}", manifest.scope.full_site);
println!("same_domain: {}", manifest.scope.same_domain);
println!("max_depth: {}", manifest.scope.max_depth);
println!("pages_discovered: {}", stats.pages_discovered);
println!("pages_fetched: {}", stats.pages_fetched);
println!("pages_parsed: {}", stats.pages_parsed);
println!("pages_failed: {}", stats.pages_failed);
println!("assets_downloaded: {}", stats.assets_downloaded);
println!("assets_indexed: {}", assets.len());
println!("asset_bytes: {}", total_asset_bytes);
println!("ocr_succeeded: {}", stats.ocr_succeeded);
if !pages.is_empty() {
println!("recent_pages:");
for page in pages.iter().rev().take(5) {
println!(" - [{}] {} ({})", page.status, page.title, page.source_url);
}
}
if !failed_pages.is_empty() {
println!("failed_pages:");
for page in failed_pages.into_iter().take(10) {
println!(
" - [{}] {} error={}",
page.status,
page.source_url,
page.error.as_deref().unwrap_or("unknown")
);
}
}
Ok(())
}
pub async fn run_search(
config: Config,
archive_id: Option<&str>,
query: &str,
limit: usize,
) -> Result<()> {
let results = search_archives(&config, archive_id, query, limit)?;
if results.is_empty() {
println!("No matches.");
return Ok(());
}
for result in results {
println!("{} | {}", result.archive_id, result.title);
println!(" {}", result.source_url);
if let Some(heading) = result.matched_heading.as_deref() {
println!(" heading: {heading}");
}
println!(" snippet: {}", result.snippet.replace('\n', " "));
if let Some(path) = result.local_file_path.as_deref() {
println!(" local: {path}");
}
}
Ok(())
}
pub async fn run_export(
config: Config,
archive_id: &str,
format: ExportFormat,
output: Option<&Path>,
) -> Result<()> {
let path = export::export_archive(&config, archive_id, format, output)?;
println!("exported: {}", path.display());
Ok(())
}
pub async fn run_pack(
config: Config,
archive_id: &str,
for_ai: bool,
max_tokens: Option<usize>,
) -> Result<()> {
if !for_ai {
return Err(SiteforgeError::MissingInput("--for-ai"));
}
let packs = export::create_ai_packs(&config, archive_id, max_tokens)?;
for pack in packs {
println!("pack: {}", pack.display());
}
Ok(())
}
pub async fn run_verify(config: Config, archive_id: &str) -> Result<()> {
let problems = verify_archive(&config, archive_id)?;
if problems.is_empty() {
println!("archive verified: {archive_id}");
} else {
println!("archive verification found {} problem(s):", problems.len());
for problem in problems {
println!("- {problem}");
}
}
Ok(())
}
pub async fn run_diagnostics(config: Config, archive_id: Option<&str>) -> Result<()> {
let root = config.resolved_archive_root()?;
println!("archive_root: {}", root.display());
println!("progress_channel_capacity: {CLI_PROGRESS_CHANNEL_CAPACITY}");
println!("default_concurrency: {}", config.default_concurrency);
println!("max_pages: {}", config.max_pages);
println!("max_asset_size_bytes: {}", config.max_asset_size_bytes);
println!(
"max_total_archive_size_bytes: {}",
config.max_total_archive_size_bytes
);
match archive_id {
Some(archive_id) => print_archive_diagnostics(&config, archive_id)?,
None => {
let archives = list_archives(&config)?;
println!("archives: {}", archives.len());
for archive in archives.iter().take(50) {
let layout = ArchiveLayout::new(root.clone(), archive.archive_id.clone());
println!(
"- {} updated={} parsed={} failed={} size_bytes={}",
archive.archive_id,
archive.updated_at,
archive.stats.pages_parsed,
archive.stats.pages_failed,
archive_size_bytes(&layout).unwrap_or(0),
);
}
if archives.len() > 50 {
println!("archives_omitted: {}", archives.len() - 50);
}
}
}
Ok(())
}
fn print_archive_diagnostics(config: &Config, archive_id: &str) -> Result<()> {
let manifest = load_archive_manifest(config, archive_id)?;
let layout = ArchiveLayout::new(config.resolved_archive_root()?, archive_id.to_string());
println!("archive_id: {}", manifest.archive_id);
println!("archive_path: {}", layout.base().display());
println!("archive_size_bytes: {}", archive_size_bytes(&layout)?);
println!("database_bytes: {}", file_len(&layout.database()));
println!("checksums_bytes: {}", file_len(&layout.checksums()));
println!("config_concurrency: {}", manifest.config.concurrency);
println!("config_max_pages: {}", manifest.config.max_pages);
if layout.database().exists() {
let storage = Storage::open(&layout.database())?;
let stats = storage.crawl_stats()?;
println!("pages_discovered: {}", stats.pages_discovered);
println!("pages_fetched: {}", stats.pages_fetched);
println!("pages_parsed: {}", stats.pages_parsed);
println!("pages_failed: {}", stats.pages_failed);
println!("pages_skipped: {}", stats.pages_skipped);
println!("assets_discovered: {}", stats.assets_discovered);
println!("assets_downloaded: {}", stats.assets_downloaded);
println!("assets_failed: {}", stats.assets_failed);
let jobs = storage.job_snapshots()?;
println!("jobs: {}", jobs.len());
for job in jobs.iter().take(20) {
println!(
"- job={} status={} active_url={}",
job.job_id,
job.status,
job.active_url.as_deref().unwrap_or("none")
);
}
if jobs.len() > 20 {
println!("jobs_omitted: {}", jobs.len() - 20);
}
}
Ok(())
}
fn file_len(path: &Path) -> u64 {
fs::metadata(path)
.map(|metadata| metadata.len())
.unwrap_or(0)
}
pub async fn run_resume(config: Config, job_id: &str, progress: CliProgressMode) -> Result<()> {
match resume_archive_with_cli_progress(config, job_id, progress).await {
Ok(summary) => {
println!("resumed archive: {}", summary.archive_id);
println!("archive_path: {}", summary.archive_path.display());
println!(
"pages: fetched={} parsed={} failed={}",
summary.stats.pages_fetched, summary.stats.pages_parsed, summary.stats.pages_failed
);
Ok(())
}
Err(SiteforgeError::ArchiveInterrupted(archive_id)) => {
print_interrupted_archive(&archive_id);
Ok(())
}
Err(err) => Err(err),
}
}
fn print_archive_summary(summary: &ArchiveSummary) -> Result<()> {
println!("archive_id: {}", summary.archive_id);
println!("archive_path: {}", summary.archive_path.display());
println!(
"pages: fetched={} parsed={} failed={} skipped={}",
summary.stats.pages_fetched,
summary.stats.pages_parsed,
summary.stats.pages_failed,
summary.stats.pages_skipped
);
println!(
"assets: downloaded={} failed={}",
summary.stats.assets_downloaded, summary.stats.assets_failed
);
Ok(())
}
fn print_interrupted_archive(archive_id: &str) {
eprintln!("archive interrupted and state was saved");
eprintln!("resume: siteforge resume {archive_id}");
}
async fn run_archive_with_cli_progress(
config: Config,
options: ArchiveRunOptions,
progress: CliProgressMode,
) -> Result<ArchiveSummary> {
let Some(settings) = ProgressRenderSettings::new(progress) else {
return crawler::run_archive(config, options).await;
};
let (tx, rx) = mpsc::channel(CLI_PROGRESS_CHANNEL_CAPACITY);
let renderer = tokio::spawn(render_archive_progress(rx, settings));
let result = crawler::run_archive_with_progress(config, options, tx).await;
let _ = renderer.await;
result
}
async fn resume_archive_with_cli_progress(
config: Config,
archive_id: &str,
progress: CliProgressMode,
) -> Result<ArchiveSummary> {
let Some(settings) = ProgressRenderSettings::new(progress) else {
return crawler::resume_archive(config, archive_id).await;
};
let (tx, rx) = mpsc::channel(CLI_PROGRESS_CHANNEL_CAPACITY);
let renderer = tokio::spawn(render_archive_progress(rx, settings));
let result = crawler::resume_archive_with_progress(config, archive_id, tx).await;
let _ = renderer.await;
result
}
#[derive(Debug, Clone, Copy)]
struct ProgressRenderSettings {
in_place: bool,
use_color: bool,
width: usize,
}
impl ProgressRenderSettings {
fn new(mode: CliProgressMode) -> Option<Self> {
if mode == CliProgressMode::Never {
return None;
}
let terminal = io::stderr().is_terminal();
if mode == CliProgressMode::Auto && !terminal {
return None;
}
Some(Self {
in_place: true,
use_color: terminal && cli_color_enabled(),
width: terminal_width(),
})
}
}
async fn render_archive_progress(
mut rx: mpsc::Receiver<ArchiveProgress>,
settings: ProgressRenderSettings,
) {
let mut stderr = io::stderr();
let loader = cli_progress_loader(settings.use_color, cli_loader_width(settings.width));
let mut interval = tokio::time::interval(Duration::from_millis(120));
let mut latest: Option<ArchiveProgress> = None;
let mut tick = 0usize;
let mut wrote = false;
loop {
let should_render = tokio::select! {
progress = rx.recv() => {
let Some(progress) = progress else {
break;
};
latest = Some(progress);
true
}
_ = interval.tick(), if settings.in_place => {
true
}
};
if settings.in_place && should_render {
if let Some(progress) = latest.as_ref() {
write_progress_line(
&mut stderr,
settings.in_place,
settings.use_color,
settings.width,
&loader,
tick,
progress,
);
tick = tick.wrapping_add(1);
wrote = true;
}
}
}
if let Some(progress) = latest.as_ref() {
write_progress_line(
&mut stderr,
settings.in_place,
settings.use_color,
settings.width,
&loader,
tick,
progress,
);
wrote = true;
}
if wrote {
let _ = writeln!(stderr);
let _ = stderr.flush();
}
}
fn cli_progress_loader(use_color: bool, width: usize) -> Loader {
let config = LoaderConfig::default()
.with_width(width)
.with_label("siteforge")
.with_unit("pages")
.with_fraction(true);
let config = if use_color {
config
} else {
config.without_color()
};
Loader::with_config(LoaderKind::Tqdm, config)
}
fn write_progress_line(
stderr: &mut io::Stderr,
in_place: bool,
use_color: bool,
terminal_width: usize,
loader: &Loader,
tick: usize,
progress: &ArchiveProgress,
) {
let (current, total) = cli_progress_counts(progress);
let loader_line = if use_color {
loader
.frame(tick, LoaderProgress::from_counts(current, total))
.to_ansi_string()
} else {
loader.line(tick, LoaderProgress::from_counts(current, total))
};
let line = progress_line(&loader_line, progress, terminal_width, use_color);
if in_place {
let _ = write!(stderr, "\r\x1b[2K{}", line);
} else {
let _ = writeln!(stderr, "{}", line);
}
let _ = stderr.flush();
}
fn progress_line(
loader_line: &str,
progress: &ArchiveProgress,
terminal_width: usize,
use_color: bool,
) -> String {
let phase = colorize(
progress.phase.as_str(),
phase_color(progress.phase),
use_color,
);
let archive = truncate_middle(&progress.archive_id, 18);
let stats = if terminal_width >= 100 {
format!(
"a={} p={} f={} s={} assets={}/{}",
archive,
progress.stats.pages_parsed,
progress.stats.pages_failed,
progress.stats.pages_skipped,
progress.stats.assets_downloaded,
progress.stats.assets_discovered,
)
} else {
format!(
"p={} f={} a={}/{}",
progress.stats.pages_parsed,
progress.stats.pages_failed,
progress.stats.assets_downloaded,
progress.stats.assets_discovered,
)
};
let hint = (terminal_width >= 105).then(|| colorize("Ctrl+C saves resume", "2", use_color));
let active_budget = terminal_width.saturating_sub(
loader_line_visible_width(loader_line)
.saturating_add(progress.phase.as_str().len())
.saturating_add(stats.len())
.saturating_add(hint.as_deref().map(visible_width).unwrap_or(0))
.saturating_add(18),
);
let active = progress
.active_url
.as_deref()
.filter(|_| active_budget >= 16)
.map(|url| format!(" | {}", truncate_middle(url, active_budget.min(70))))
.unwrap_or_default();
let mut line = format!("{} | {} | {}{}", loader_line, phase, stats, active);
if let Some(hint) = hint {
line.push_str(" | ");
line.push_str(&hint);
}
line
}
fn cli_progress_counts(progress: &ArchiveProgress) -> (u64, u64) {
let processed = progress
.stats
.pages_parsed
.saturating_add(progress.stats.pages_failed)
.saturating_add(progress.stats.pages_skipped);
let discovered = progress.stats.pages_discovered.max(processed).max(1);
let capped_total = if progress.max_pages == 0 {
discovered
} else {
discovered.min(progress.max_pages.max(1))
};
if matches!(
progress.phase,
ArchiveProgressPhase::Completed | ArchiveProgressPhase::Interrupted
) {
let final_total = processed.max(1);
(final_total as u64, final_total as u64)
} else {
(processed.min(capped_total) as u64, capped_total as u64)
}
}
fn truncate_middle(input: &str, max_chars: usize) -> String {
let chars = input.chars().collect::<Vec<_>>();
if chars.len() <= max_chars {
return input.to_string();
}
let keep = max_chars.saturating_sub(3);
let left = keep / 2;
let right = keep.saturating_sub(left);
let mut output = chars.iter().take(left).collect::<String>();
output.push_str("...");
output.push_str(
&chars[chars.len().saturating_sub(right)..]
.iter()
.collect::<String>(),
);
output
}
fn cli_loader_width(terminal_width: usize) -> usize {
match terminal_width {
0..=79 => 28,
80..=119 => 34,
_ => 46,
}
}
fn terminal_width() -> usize {
std::env::var("COLUMNS")
.ok()
.and_then(|value| value.parse::<usize>().ok())
.filter(|value| *value >= 40)
.unwrap_or(120)
}
fn cli_color_enabled() -> bool {
std::env::var_os("NO_COLOR").is_none()
&& std::env::var("TERM")
.map(|term| term != "dumb")
.unwrap_or(true)
}
fn phase_color(phase: ArchiveProgressPhase) -> &'static str {
match phase {
ArchiveProgressPhase::Completed => "32;1",
ArchiveProgressPhase::Interrupted => "33;1",
ArchiveProgressPhase::Processing => "36;1",
ArchiveProgressPhase::Fetching => "34;1",
ArchiveProgressPhase::Discovering => "35;1",
ArchiveProgressPhase::Starting => "37;1",
}
}
fn colorize(text: &str, code: &str, enabled: bool) -> String {
if enabled {
format!("\x1b[{code}m{text}\x1b[0m")
} else {
text.to_string()
}
}
fn loader_line_visible_width(line: &str) -> usize {
visible_width(line.trim_end())
}
fn visible_width(input: &str) -> usize {
let mut chars = input.chars().peekable();
let mut width = 0usize;
while let Some(ch) = chars.next() {
if ch == '\x1b' && chars.peek() == Some(&'[') {
chars.next();
for next in chars.by_ref() {
if next.is_ascii_alphabetic() {
break;
}
}
} else {
width += 1;
}
}
width
}
pub async fn run_config(config: Config, edit: bool) -> Result<()> {
let path = Config::ensure_default_file()?;
if edit {
let editor = std::env::var("EDITOR").unwrap_or_else(|_| "vi".to_string());
let status = Command::new(editor).arg(&path).status()?;
if !status.success() {
return Err(SiteforgeError::message(
"editor exited with a non-zero status",
));
}
} else {
println!("config_path: {}", path.display());
println!("{}", serde_yaml::to_string(&config)?);
}
Ok(())
}
fn load_seed_urls(url: Option<&str>, input: Option<&Path>) -> Result<Vec<Url>> {
let mut seeds = Vec::new();
if let Some(url) = url {
seeds.push(Url::parse(url)?);
}
if let Some(input) = input {
let raw = fs::read_to_string(input)?;
for line in raw.lines() {
let line = line.trim();
if line.is_empty() || line.starts_with('#') {
continue;
}
seeds.push(Url::parse(line)?);
}
}
if seeds.is_empty() {
return Err(SiteforgeError::MissingInput("URL or --input"));
}
Ok(seeds)
}
fn parse_headers(values: &[String]) -> Result<Vec<(String, String)>> {
values
.iter()
.map(|value| {
let (name, header_value) = value.split_once(':').ok_or_else(|| {
SiteforgeError::message(format!("invalid header {value:?}; expected 'Name: Value'"))
})?;
let name = name.trim();
let header_value = header_value.trim();
if name.is_empty() || header_value.is_empty() {
return Err(SiteforgeError::message(format!(
"invalid header {value:?}; name and value are required"
)));
}
Ok((name.to_string(), header_value.to_string()))
})
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
use clap::CommandFactory;
#[test]
fn parses_archive_headers() {
let headers = parse_headers(&["Authorization: Bearer token".to_string()]).unwrap();
assert_eq!(
headers,
vec![("Authorization".to_string(), "Bearer token".to_string())]
);
}
#[test]
fn rejects_malformed_archive_headers() {
let err = parse_headers(&["missing-colon".to_string()])
.unwrap_err()
.to_string();
assert!(err.contains("invalid header"));
}
#[test]
fn cli_definition_is_valid() {
Cli::command().debug_assert();
}
#[test]
fn parses_basic_archive_cli_with_concurrency() {
let cli = Cli::try_parse_from([
"siteforge",
"archive",
"https://example.test/docs",
"--full-site",
"-j",
"8",
"--archive-id",
"docs",
"--render-js",
"--progress",
"never",
])
.unwrap();
match cli.command {
Commands::Archive(command) => {
assert_eq!(command.url.as_deref(), Some("https://example.test/docs"));
assert!(command.full_site);
assert_eq!(command.concurrency, Some(8));
assert_eq!(command.archive_id.as_deref(), Some("docs"));
assert!(command.render_js);
assert_eq!(command.progress, CliProgressMode::Never);
}
_ => panic!("expected archive command"),
}
}
#[test]
fn parses_basic_operational_commands() {
assert!(matches!(
Cli::try_parse_from(["siteforge", "list"]).unwrap().command,
Commands::List
));
assert!(matches!(
Cli::try_parse_from(["siteforge", "inspect", "docs"])
.unwrap()
.command,
Commands::Inspect { .. }
));
assert!(matches!(
Cli::try_parse_from(["siteforge", "search", "mesh", "--limit", "5"])
.unwrap()
.command,
Commands::Search { limit: 5, .. }
));
assert!(matches!(
Cli::try_parse_from(["siteforge", "export", "docs", "--format", "zip"])
.unwrap()
.command,
Commands::Export {
format: ExportFormat::Zip,
..
}
));
assert!(matches!(
Cli::try_parse_from(["siteforge", "pack", "docs", "--for-ai"])
.unwrap()
.command,
Commands::Pack { for_ai: true, .. }
));
assert!(matches!(
Cli::try_parse_from(["siteforge", "resume", "docs", "--progress", "always"])
.unwrap()
.command,
Commands::Resume {
progress: CliProgressMode::Always,
..
}
));
assert!(matches!(
Cli::try_parse_from(["siteforge", "diagnostics", "--archive-id", "docs"])
.unwrap()
.command,
Commands::Diagnostics { .. }
));
}
#[test]
fn cli_progress_counts_complete_at_finished_total() {
let mut progress = ArchiveProgress {
archive_id: "archive".to_string(),
phase: ArchiveProgressPhase::Processing,
active_url: None,
stats: Default::default(),
max_pages: 100,
};
progress.stats.pages_discovered = 10;
progress.stats.pages_parsed = 3;
progress.stats.pages_failed = 1;
assert_eq!(cli_progress_counts(&progress), (4, 10));
progress.phase = ArchiveProgressPhase::Completed;
assert_eq!(cli_progress_counts(&progress), (4, 4));
}
#[test]
fn truncates_long_progress_urls_in_middle() {
let url =
"https://example.test/docs/tutorials/really/long/path/with/many/segments/page.html";
let truncated = truncate_middle(url, 32);
assert!(truncated.len() <= 32);
assert!(truncated.starts_with("https://"));
assert!(truncated.ends_with("page.html"));
assert!(truncated.contains("..."));
}
#[test]
fn progress_line_compacts_for_narrow_terminals() {
let mut progress = ArchiveProgress {
archive_id: "very-long-archive-name-for-docs".to_string(),
phase: ArchiveProgressPhase::Fetching,
active_url: Some("https://example.test/a/very/long/page.html".to_string()),
stats: Default::default(),
max_pages: 10,
};
progress.stats.pages_parsed = 2;
progress.stats.pages_failed = 1;
progress.stats.assets_downloaded = 3;
progress.stats.assets_discovered = 5;
let line = progress_line("siteforge [====] 30%", &progress, 80, false);
assert!(line.contains("fetching"));
assert!(line.contains("p=2 f=1 a=3/5"));
assert!(!line.contains("Ctrl+C"));
assert!(!line.contains("very-long-archive"));
}
#[test]
fn visible_width_ignores_ansi_color_codes() {
assert_eq!(visible_width("\x1b[32;1mgreen\x1b[0m"), 5);
}
}