use std::path::PathBuf;
use clap::{Args, Parser, Subcommand};
use siteforge::ExportFormat as SiteforgeExportFormat;
use crate::types::{
AssetFormat, AssetProvider, AssetResolution, AudioKind, DatasetProvider, Mode, ModelProvider,
PaperProvider, PaperSort, RepositoryProvider, SiteArchiveProfile,
};
fn parse_asset_limit(value: &str) -> Result<usize, String> {
let limit = value
.parse::<usize>()
.map_err(|_| "asset limit must be an integer from 1 to 100".to_string())?;
if (1..=100).contains(&limit) {
Ok(limit)
} else {
Err("asset limit must be from 1 to 100".to_string())
}
}
fn parse_audio_limit(value: &str) -> Result<usize, String> {
let limit = value
.parse::<usize>()
.map_err(|_| "audio limit must be an integer from 1 to 50".to_string())?;
if (1..=50).contains(&limit) {
Ok(limit)
} else {
Err("audio limit must be from 1 to 50".to_string())
}
}
fn parse_paper_limit(value: &str) -> Result<usize, String> {
let limit = value
.parse::<usize>()
.map_err(|_| "paper limit must be an integer from 1 to 100".to_string())?;
if (1..=100).contains(&limit) {
Ok(limit)
} else {
Err("paper limit must be from 1 to 100".to_string())
}
}
#[derive(Parser, Clone)]
#[command(
name = "thesa",
version,
about = "Archive repositories, models, datasets, websites, assets, audio, scholarly papers, and technical reports",
subcommand_precedence_over_arg = true
)]
pub(crate) struct Cli {
#[arg(long, value_name = "SELECTOR", num_args = 0..=1, default_missing_value = "all")]
pub(crate) agent: Option<String>,
#[command(subcommand)]
pub(crate) command: Option<Commands>,
pub(crate) target: Option<String>,
#[arg(short = 'o', long = "output", default_value = "./archives")]
pub(crate) output: PathBuf,
#[arg(short = 't', long = "token")]
pub(crate) token: Option<String>,
#[arg(long = "bitbucket-token")]
pub(crate) bitbucket_token: Option<String>,
#[arg(long = "hf-token")]
pub(crate) hf_token: Option<String>,
#[arg(long = "civitai-token")]
pub(crate) civitai_token: Option<String>,
#[arg(
long = "asset-token",
env = "SKETCHFAB_API_TOKEN",
hide_env_values = true
)]
pub(crate) asset_token: Option<String>,
#[arg(long = "depth")]
pub(crate) depth: Option<u32>,
#[arg(short = 'c', long = "concurrency", default_value_t = 4)]
pub(crate) concurrency: usize,
#[arg(long = "skip-existing")]
pub(crate) skip_existing: bool,
#[arg(long = "filter")]
pub(crate) filter: Option<String>,
#[arg(long = "dry-run")]
pub(crate) dry_run: bool,
#[arg(short = 'T', long = "tui")]
pub(crate) tui: bool,
#[arg(
long = "mode",
value_enum,
default_value_t = Mode::Repos,
help = "Archive mode: repos, models, datasets, sites, assets, audio, or papers"
)]
pub(crate) mode: Mode,
#[arg(long = "repo-provider", value_enum, default_value_t = RepositoryProvider::Github)]
pub(crate) repo_provider: RepositoryProvider,
#[arg(long = "provider", value_enum, default_value_t = ModelProvider::Hf)]
pub(crate) provider: ModelProvider,
#[arg(long = "dataset-provider", value_enum, default_value_t = DatasetProvider::Hf)]
pub(crate) dataset_provider: DatasetProvider,
#[arg(long = "asset-provider", value_enum, default_value_t = AssetProvider::Polyhaven)]
pub(crate) asset_provider: AssetProvider,
#[arg(long = "asset-format", value_enum, default_value_t = AssetFormat::Auto)]
pub(crate) asset_format: AssetFormat,
#[arg(long = "asset-resolution", value_enum, default_value_t = AssetResolution::Auto)]
pub(crate) asset_resolution: AssetResolution,
#[arg(long = "asset-max-download-bytes", default_value_t = 1024 * 1024 * 1024)]
pub(crate) asset_max_download_bytes: u64,
#[arg(long = "asset-limit", default_value_t = 20, value_parser = parse_asset_limit)]
pub(crate) asset_limit: usize,
#[arg(long = "audio-kind", value_enum, default_value_t = AudioKind::Any)]
pub(crate) audio_kind: AudioKind,
#[arg(long = "audio-limit", default_value_t = 20, value_parser = parse_audio_limit)]
pub(crate) audio_limit: usize,
#[arg(long = "audio-max-download-bytes", default_value_t = 512 * 1024 * 1024)]
pub(crate) audio_max_download_bytes: u64,
#[arg(long = "paper-provider", value_enum, default_value_t = PaperProvider::Arxiv)]
pub(crate) paper_provider: PaperProvider,
#[arg(long = "paper-sort", value_enum)]
pub(crate) paper_sort: Option<PaperSort>,
#[arg(long = "paper-limit", default_value_t = 20, value_parser = parse_paper_limit)]
pub(crate) paper_limit: usize,
#[arg(long = "paper-max-download-bytes", default_value_t = 256 * 1024 * 1024)]
pub(crate) paper_max_download_bytes: u64,
#[arg(long = "hf-mirror")]
pub(crate) hf_mirror: bool,
}
#[derive(Debug, Clone, Subcommand)]
#[allow(clippy::large_enum_variant)]
pub(crate) enum Commands {
Archive {
#[arg(long = "dry-run")]
dry_run: bool,
#[command(subcommand)]
source: ArchiveCommands,
},
Verify {
archive: PathBuf,
#[arg(long)]
json: bool,
},
}
#[derive(Debug, Clone, Subcommand)]
pub(crate) enum ArchiveCommands {
#[command(alias = "update", alias = "pull")]
Refresh(GitRefreshArgs),
Site(SiteArchiveArgs),
Dataset(DatasetArchiveArgs),
#[command(alias = "assets", alias = "3d")]
Asset(AssetArchiveArgs),
#[command(alias = "sound", alias = "sounds", alias = "soundforge")]
Audio(AudioArchiveArgs),
#[command(alias = "papers", alias = "paperforge")]
Paper(PaperArchiveArgs),
Url(SiteArchiveArgs),
List(SiteforgeRootArgs),
Inspect(SiteforgeArchiveIdArgs),
Search(SiteforgeSearchArgs),
Export(SiteforgeExportArgs),
Pack(SiteforgePackArgs),
Verify(SiteforgeArchiveIdArgs),
Resume(SiteforgeArchiveIdArgs),
Diagnostics(SiteforgeDiagnosticsArgs),
}
#[derive(Debug, Clone, Args)]
pub(crate) struct GitRefreshArgs {
#[arg(value_name = "NAMESPACE")]
pub(crate) owner: Option<String>,
#[arg(long = "archive-root", short = 'r', default_value = "./archives")]
pub(crate) archive_root: PathBuf,
#[arg(
long = "owner",
alias = "user",
alias = "username",
alias = "workspace",
value_name = "NAMESPACE"
)]
pub(crate) owner_filter: Option<String>,
#[arg(short = 'c', long = "concurrency", default_value_t = 4)]
pub(crate) concurrency: usize,
#[arg(long = "filter")]
pub(crate) filter: Option<String>,
#[arg(long = "dry-run")]
pub(crate) dry_run: bool,
}
#[derive(Debug, Clone, Args)]
pub(crate) struct SiteforgeRootArgs {
#[arg(long = "archive-root", default_value = "./archives/sites")]
pub(crate) archive_root: PathBuf,
}
#[derive(Debug, Clone, Args)]
pub(crate) struct DatasetArchiveArgs {
pub(crate) target: String,
#[arg(long = "provider", value_enum, default_value_t = DatasetProvider::Hf)]
pub(crate) provider: DatasetProvider,
#[arg(
long = "archive-root",
short = 'r',
default_value = "./archives/datasets"
)]
pub(crate) archive_root: PathBuf,
#[arg(short = 'c', long = "concurrency", default_value_t = 4)]
pub(crate) concurrency: usize,
#[arg(long = "skip-existing")]
pub(crate) skip_existing: bool,
#[arg(long = "filter")]
pub(crate) filter: Option<String>,
#[arg(long = "hf-token")]
pub(crate) hf_token: Option<String>,
#[arg(long = "dry-run")]
pub(crate) dry_run: bool,
}
#[derive(Debug, Clone, Args)]
pub(crate) struct AssetArchiveArgs {
pub(crate) target: String,
#[arg(long = "provider", value_enum, default_value_t = AssetProvider::Polyhaven)]
pub(crate) provider: AssetProvider,
#[arg(long = "format", value_enum, default_value_t = AssetFormat::Auto)]
pub(crate) format: AssetFormat,
#[arg(long = "resolution", value_enum, default_value_t = AssetResolution::Auto)]
pub(crate) resolution: AssetResolution,
#[arg(
long = "archive-root",
short = 'r',
default_value = "./archives/assets"
)]
pub(crate) archive_root: PathBuf,
#[arg(long = "skip-existing")]
pub(crate) skip_existing: bool,
#[arg(long = "filter")]
pub(crate) filter: Option<String>,
#[arg(long = "max-download-bytes", default_value_t = 1024 * 1024 * 1024)]
pub(crate) max_download_bytes: u64,
#[arg(long, default_value_t = 20, value_parser = parse_asset_limit)]
pub(crate) limit: usize,
#[arg(long, env = "SKETCHFAB_API_TOKEN", hide_env_values = true)]
pub(crate) token: Option<String>,
#[arg(long = "dry-run")]
pub(crate) dry_run: bool,
}
#[derive(Debug, Clone, Args)]
pub(crate) struct AudioArchiveArgs {
pub(crate) target: String,
#[arg(long = "kind", value_enum, default_value_t = AudioKind::Any)]
pub(crate) kind: AudioKind,
#[arg(
long = "archive-root",
alias = "output",
short = 'r',
default_value = "./archives/audio"
)]
pub(crate) archive_root: PathBuf,
#[arg(long = "skip-existing")]
pub(crate) skip_existing: bool,
#[arg(long = "filter")]
pub(crate) filter: Option<String>,
#[arg(long = "max-download-bytes", default_value_t = 512 * 1024 * 1024)]
pub(crate) max_download_bytes: u64,
#[arg(long, default_value_t = 20, value_parser = parse_audio_limit)]
pub(crate) limit: usize,
#[arg(long = "dry-run")]
pub(crate) dry_run: bool,
}
#[derive(Debug, Clone, Args)]
pub(crate) struct PaperArchiveArgs {
pub(crate) query: String,
#[arg(long = "provider", value_enum)]
pub(crate) provider: Option<PaperProvider>,
#[arg(long = "sort", value_enum)]
pub(crate) sort: Option<PaperSort>,
#[arg(
long = "archive-root",
alias = "output",
short = 'r',
default_value = "./archives/papers"
)]
pub(crate) archive_root: PathBuf,
#[arg(long = "skip-existing")]
pub(crate) skip_existing: bool,
#[arg(long = "filter")]
pub(crate) filter: Option<String>,
#[arg(long = "max-download-bytes", default_value_t = 256 * 1024 * 1024)]
pub(crate) max_download_bytes: u64,
#[arg(long, default_value_t = 20, value_parser = parse_paper_limit)]
pub(crate) limit: usize,
#[arg(long = "dry-run")]
pub(crate) dry_run: bool,
}
#[derive(Debug, Clone, Args)]
pub(crate) struct SiteforgeArchiveIdArgs {
pub(crate) archive_id: String,
#[command(flatten)]
pub(crate) root: SiteforgeRootArgs,
}
#[derive(Debug, Clone, Args)]
pub(crate) struct SiteforgeSearchArgs {
pub(crate) query: String,
#[arg(long = "archive-id")]
pub(crate) archive_id: Option<String>,
#[arg(long = "limit", default_value_t = 20)]
pub(crate) limit: usize,
#[command(flatten)]
pub(crate) root: SiteforgeRootArgs,
}
#[derive(Debug, Clone, Args)]
pub(crate) struct SiteforgeExportArgs {
pub(crate) archive_id: String,
#[arg(long = "format", value_enum)]
pub(crate) format: SiteforgeExportFormat,
#[arg(long = "output")]
pub(crate) output: Option<PathBuf>,
#[command(flatten)]
pub(crate) root: SiteforgeRootArgs,
}
#[derive(Debug, Clone, Args)]
pub(crate) struct SiteforgePackArgs {
pub(crate) archive_id: String,
#[arg(long = "max-tokens")]
pub(crate) max_tokens: Option<usize>,
#[command(flatten)]
pub(crate) root: SiteforgeRootArgs,
}
#[derive(Debug, Clone, Args)]
pub(crate) struct SiteforgeDiagnosticsArgs {
pub(crate) archive_id: Option<String>,
#[command(flatten)]
pub(crate) root: SiteforgeRootArgs,
}
#[derive(Debug, Clone, Args)]
pub(crate) struct SiteArchiveArgs {
#[arg(value_name = "URL")]
pub(crate) url: Option<String>,
#[arg(long = "input", value_name = "FILE")]
pub(crate) input: Option<PathBuf>,
#[arg(short = 'o', long = "output", default_value = "./archives/sites")]
pub(crate) output: PathBuf,
#[arg(long = "full-site")]
pub(crate) full_site: bool,
#[arg(long = "allow-cross-domain")]
pub(crate) allow_cross_domain: bool,
#[arg(long = "max-depth")]
pub(crate) max_depth: Option<usize>,
#[arg(long = "max-pages")]
pub(crate) max_pages: Option<usize>,
#[arg(short = 'c', short_alias = 'j', long = "concurrency", alias = "jobs")]
pub(crate) concurrency: Option<usize>,
#[arg(long = "delay-ms")]
pub(crate) delay_ms: Option<u64>,
#[arg(long = "archive-id")]
pub(crate) archive_id: Option<String>,
#[arg(long = "include", value_name = "GLOB")]
pub(crate) include_url_patterns: Vec<String>,
#[arg(long = "exclude", value_name = "GLOB")]
pub(crate) exclude_url_patterns: Vec<String>,
#[arg(long = "archive-profile", value_enum)]
pub(crate) archive_profile: Option<SiteArchiveProfile>,
#[arg(long = "max-page-size-bytes")]
pub(crate) max_page_size_bytes: Option<u64>,
#[arg(long = "max-asset-size-bytes")]
pub(crate) max_asset_size_bytes: Option<u64>,
#[arg(long = "max-total-archive-size-bytes")]
pub(crate) max_total_archive_size_bytes: Option<u64>,
#[arg(long = "render-js")]
pub(crate) render_js: bool,
#[arg(long = "render-js-auto", conflicts_with = "render_js")]
pub(crate) render_js_auto: bool,
#[arg(long = "browser-command", value_name = "COMMAND")]
pub(crate) browser_command: Option<String>,
#[arg(long = "browser-profile", value_name = "DIR")]
pub(crate) browser_profile: Option<PathBuf>,
#[arg(long = "render-wait-ms")]
pub(crate) render_wait_ms: Option<u64>,
#[arg(long = "header", value_name = "NAME: VALUE")]
pub(crate) headers: Vec<String>,
#[arg(long = "cookie", value_name = "COOKIE_HEADER")]
pub(crate) cookie: Option<String>,
#[arg(long = "user-agent", value_name = "VALUE")]
pub(crate) user_agent: Option<String>,
#[arg(long = "timeout-secs")]
pub(crate) timeout_secs: Option<u64>,
#[arg(long = "retry-count")]
pub(crate) retry_count: Option<usize>,
#[arg(long = "ocr")]
pub(crate) ocr_enabled: bool,
#[arg(long = "dry-run")]
pub(crate) dry_run: bool,
}