use std::collections::{BTreeMap, BTreeSet};
use std::fs::{self, remove_dir_all, remove_file};
use std::io::{self, IsTerminal, Write};
use std::path::{Component, Path, PathBuf};
use std::process::{Child, Command, Stdio};
use std::sync::mpsc;
use std::thread;
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use aisling::{
Color as AislingColor, EffectKind, Gradient, GradientDirection, Loader, LoaderConfig,
LoaderKind, LoaderProgress,
};
use clap::{Args, Parser, Subcommand, ValueEnum};
use crossterm::event::{
self, Event, KeyCode, KeyEventKind, MouseButton, MouseEvent, MouseEventKind,
};
use reqwest::StatusCode;
use reqwest::blocking::{Client, Response};
use scrin::{
Color as TuiColor, Frame, Rect, Terminal, TerminalOptions,
core::buffer::Buffer,
effects::{EffectPlayer, LoaderPlayer},
layout::{Constraint, Direction, Layout},
style::{Modifier, Style},
widgets::{
Alignment, Block, Clear, Line, List, ListItem, ListState, Paragraph, Span, Text, Widget,
WrapMode, block::BorderStyle,
},
};
use serde::{Deserialize, Serialize};
use siteforge::{
ArchiveLayout as SiteforgeArchiveLayout, ArchiveOutputProfile,
ArchiveRunOptions as SiteforgeArchiveRunOptions, Config as SiteforgeConfig,
CrawlStats as SiteforgeCrawlStats, ExportFormat as SiteforgeExportFormat, Siteforge,
};
use thiserror::Error;
const GITHUB_API: &str = "https://api.github.com";
const HF_BASE: &str = "https://huggingface.co";
const HF_MIRROR_BASE: &str = "https://hf-mirror.com";
const OLLAMA_LOCAL: &str = "http://localhost:11434";
const CIVITAI_API: &str = "https://civitai.com/api/v1";
const DEFAULT_PAGE_SIZE: usize = 100;
const BATCH_SIZE_CHOICES: [usize; 5] = [1, 2, 3, 6, 8];
const MODE_OPTION_COUNT: usize = 3;
const SITE_OPTION_COUNT: usize = 5;
const THESA_FORMAT_VERSION: &str = "1";
const THESA_METADATA_DIR: &str = ".thesa";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum TuiScreen {
Mode,
Provider,
Target,
SiteOptions,
HfToken,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
enum Mode {
Repos,
Models,
Sites,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct SiteTuiOptions {
full_site: bool,
max_depth: usize,
max_pages: usize,
archive_profile: ArchiveOutputProfile,
render_js: bool,
render_js_auto: bool,
}
impl Default for SiteTuiOptions {
fn default() -> Self {
Self {
full_site: true,
max_depth: 2,
max_pages: 100,
archive_profile: ArchiveOutputProfile::Full,
render_js: false,
render_js_auto: false,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
enum SiteArchiveProfile {
Full,
Agent,
Lean,
}
impl SiteArchiveProfile {
fn as_output_profile(self) -> ArchiveOutputProfile {
match self {
Self::Full => ArchiveOutputProfile::Full,
Self::Agent => ArchiveOutputProfile::Agent,
Self::Lean => ArchiveOutputProfile::Lean,
}
}
}
impl From<ArchiveOutputProfile> for SiteArchiveProfile {
fn from(value: ArchiveOutputProfile) -> Self {
match value {
ArchiveOutputProfile::Full => Self::Full,
ArchiveOutputProfile::Agent => Self::Agent,
ArchiveOutputProfile::Lean => Self::Lean,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
enum ModelProvider {
Hf,
Ollama,
Civitai,
}
impl ModelProvider {
fn label(self) -> &'static str {
match self {
ModelProvider::Hf => "Hugging Face",
ModelProvider::Ollama => "Ollama",
ModelProvider::Civitai => "CivitAI",
}
}
fn short_label(self) -> &'static str {
match self {
ModelProvider::Hf => "HF",
ModelProvider::Ollama => "OLM",
ModelProvider::Civitai => "CIV",
}
}
}
impl std::fmt::Display for ModelProvider {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
ModelProvider::Hf => write!(f, "hf"),
ModelProvider::Ollama => write!(f, "ollama"),
ModelProvider::Civitai => write!(f, "civitai"),
}
}
}
impl From<ModelProvider> for modelforge::ModelProvider {
fn from(value: ModelProvider) -> Self {
match value {
ModelProvider::Hf => Self::Hf,
ModelProvider::Ollama => Self::Ollama,
ModelProvider::Civitai => Self::Civitai,
}
}
}
impl std::fmt::Display for Mode {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Mode::Repos => write!(f, "repos"),
Mode::Models => write!(f, "models"),
Mode::Sites => write!(f, "sites"),
}
}
}
#[derive(Parser, Clone)]
#[command(
name = "thesa",
version,
about = "A compendium to archive GitHub repositories, ML models, and websites",
subcommand_precedence_over_arg = true
)]
struct Cli {
#[command(subcommand)]
command: Option<Commands>,
target: Option<String>,
#[arg(short = 'o', long = "output", default_value = "./archives")]
output: PathBuf,
#[arg(short = 't', long = "token")]
token: Option<String>,
#[arg(long = "hf-token")]
hf_token: Option<String>,
#[arg(long = "civitai-token")]
civitai_token: Option<String>,
#[arg(long = "depth")]
depth: Option<u32>,
#[arg(short = 'c', long = "concurrency", default_value_t = 4)]
concurrency: usize,
#[arg(long = "skip-existing")]
skip_existing: bool,
#[arg(long = "filter")]
filter: Option<String>,
#[arg(long = "dry-run")]
dry_run: bool,
#[arg(short = 'T', long = "tui")]
tui: bool,
#[arg(
long = "mode",
value_enum,
default_value_t = Mode::Repos,
help = "Archive mode: repos (GitHub), models, or sites (Siteforge)"
)]
mode: Mode,
#[arg(long = "provider", value_enum, default_value_t = ModelProvider::Hf)]
provider: ModelProvider,
#[arg(long = "hf-mirror")]
hf_mirror: bool,
}
#[derive(Debug, Clone, Subcommand)]
enum Commands {
Archive {
#[arg(long = "dry-run")]
dry_run: bool,
#[command(subcommand)]
source: ArchiveCommands,
},
Verify {
archive: PathBuf,
#[arg(long)]
json: bool,
},
}
#[derive(Debug, Clone, Subcommand)]
enum ArchiveCommands {
#[command(alias = "update", alias = "pull")]
Refresh(GitRefreshArgs),
Site(SiteArchiveArgs),
Url(SiteArchiveArgs),
List(SiteforgeRootArgs),
Inspect(SiteforgeArchiveIdArgs),
Search(SiteforgeSearchArgs),
Export(SiteforgeExportArgs),
Pack(SiteforgePackArgs),
Verify(SiteforgeArchiveIdArgs),
Resume(SiteforgeArchiveIdArgs),
Diagnostics(SiteforgeDiagnosticsArgs),
}
#[derive(Debug, Clone, Args)]
struct GitRefreshArgs {
#[arg(long = "archive-root", short = 'r', default_value = "./archives")]
archive_root: PathBuf,
#[arg(short = 'c', long = "concurrency", default_value_t = 4)]
concurrency: usize,
#[arg(long = "filter")]
filter: Option<String>,
#[arg(long = "dry-run")]
dry_run: bool,
}
#[derive(Debug, Clone, Args)]
struct SiteforgeRootArgs {
#[arg(long = "archive-root", default_value = "./archives/sites")]
archive_root: PathBuf,
}
#[derive(Debug, Clone, Args)]
struct SiteforgeArchiveIdArgs {
archive_id: String,
#[command(flatten)]
root: SiteforgeRootArgs,
}
#[derive(Debug, Clone, Args)]
struct SiteforgeSearchArgs {
query: String,
#[arg(long = "archive-id")]
archive_id: Option<String>,
#[arg(long = "limit", default_value_t = 20)]
limit: usize,
#[command(flatten)]
root: SiteforgeRootArgs,
}
#[derive(Debug, Clone, Args)]
struct SiteforgeExportArgs {
archive_id: String,
#[arg(long = "format", value_enum)]
format: SiteforgeExportFormat,
#[arg(long = "output")]
output: Option<PathBuf>,
#[command(flatten)]
root: SiteforgeRootArgs,
}
#[derive(Debug, Clone, Args)]
struct SiteforgePackArgs {
archive_id: String,
#[arg(long = "max-tokens")]
max_tokens: Option<usize>,
#[command(flatten)]
root: SiteforgeRootArgs,
}
#[derive(Debug, Clone, Args)]
struct SiteforgeDiagnosticsArgs {
archive_id: Option<String>,
#[command(flatten)]
root: SiteforgeRootArgs,
}
#[derive(Debug, Clone, Args)]
struct SiteArchiveArgs {
#[arg(value_name = "URL")]
url: Option<String>,
#[arg(long = "input", value_name = "FILE")]
input: Option<PathBuf>,
#[arg(short = 'o', long = "output", default_value = "./archives/sites")]
output: PathBuf,
#[arg(long = "full-site")]
full_site: bool,
#[arg(long = "allow-cross-domain")]
allow_cross_domain: bool,
#[arg(long = "max-depth")]
max_depth: Option<usize>,
#[arg(long = "max-pages")]
max_pages: Option<usize>,
#[arg(short = 'c', short_alias = 'j', long = "concurrency", alias = "jobs")]
concurrency: Option<usize>,
#[arg(long = "delay-ms")]
delay_ms: Option<u64>,
#[arg(long = "archive-id")]
archive_id: Option<String>,
#[arg(long = "include", value_name = "GLOB")]
include_url_patterns: Vec<String>,
#[arg(long = "exclude", value_name = "GLOB")]
exclude_url_patterns: Vec<String>,
#[arg(long = "archive-profile", value_enum)]
archive_profile: Option<SiteArchiveProfile>,
#[arg(long = "max-page-size-bytes")]
max_page_size_bytes: Option<u64>,
#[arg(long = "max-asset-size-bytes")]
max_asset_size_bytes: Option<u64>,
#[arg(long = "max-total-archive-size-bytes")]
max_total_archive_size_bytes: Option<u64>,
#[arg(long = "render-js")]
render_js: bool,
#[arg(long = "render-js-auto", conflicts_with = "render_js")]
render_js_auto: bool,
#[arg(long = "browser-command", value_name = "COMMAND")]
browser_command: Option<String>,
#[arg(long = "browser-profile", value_name = "DIR")]
browser_profile: Option<PathBuf>,
#[arg(long = "render-wait-ms")]
render_wait_ms: Option<u64>,
#[arg(long = "header", value_name = "NAME: VALUE")]
headers: Vec<String>,
#[arg(long = "cookie", value_name = "COOKIE_HEADER")]
cookie: Option<String>,
#[arg(long = "user-agent", value_name = "VALUE")]
user_agent: Option<String>,
#[arg(long = "timeout-secs")]
timeout_secs: Option<u64>,
#[arg(long = "retry-count")]
retry_count: Option<usize>,
#[arg(long = "ocr")]
ocr_enabled: bool,
#[arg(long = "dry-run")]
dry_run: bool,
}
#[derive(Debug, Clone)]
enum Target {
Owner(String),
Repo { owner: String, repo: String },
}
impl Target {
fn archive_target_id(&self) -> String {
match self {
Target::Owner(owner) => owner.clone(),
Target::Repo { owner, repo } => format!("{owner}/{repo}"),
}
}
}
#[derive(Debug, Clone)]
struct RepoItem {
name: String,
full_name: String,
clone_url: String,
}
#[derive(Debug, Deserialize)]
struct RepoResponse {
name: String,
#[serde(rename = "full_name")]
full_name: String,
#[serde(rename = "clone_url")]
clone_url: String,
private: bool,
}
#[derive(Debug, Clone)]
#[allow(dead_code)]
struct ModelItem {
id: String,
downloads: u64,
pipeline_tag: Option<String>,
last_modified: Option<String>,
provider: ModelProvider,
download_url: Option<String>,
size: Option<u64>,
}
#[derive(Debug, Deserialize)]
struct HfModelResponse {
#[serde(rename = "modelId", default)]
model_id: Option<String>,
#[serde(default)]
id: String,
#[serde(default)]
downloads: u64,
#[serde(rename = "pipeline_tag", default)]
pipeline_tag: Option<String>,
#[serde(rename = "lastModified", default)]
last_modified: Option<String>,
}
#[derive(Debug, Deserialize)]
struct OllamaModelResponse {
models: Vec<OllamaModel>,
}
#[derive(Debug, Deserialize)]
struct OllamaModel {
name: String,
#[serde(default)]
size: u64,
#[serde(default)]
modified_at: String,
}
#[derive(Debug, Deserialize)]
struct CivitaiModelResponse {
items: Vec<CivitaiModel>,
}
#[derive(Debug, Deserialize)]
#[allow(dead_code)]
struct CivitaiModel {
id: u64,
name: String,
#[serde(default)]
downloads: u64,
#[serde(rename = "type", default)]
model_type: Option<String>,
#[serde(default)]
description: Option<String>,
model_versions: Option<Vec<CivitaiModelVersion>>,
}
#[derive(Debug, Deserialize)]
#[allow(dead_code)]
struct CivitaiModelVersion {
id: u64,
name: Option<String>,
#[serde(default)]
download_url: Option<String>,
#[serde(default)]
size: Option<u64>,
}
#[derive(Debug, Clone)]
enum CloneStatus {
Cloned,
Skipped,
}
#[derive(Debug, Clone, Copy)]
enum BatchState {
Pending,
Running,
Done,
Skipped,
Failed,
}
#[derive(Debug)]
enum CloneAction {
Skipped,
Running(Child),
}
#[derive(Debug, Default)]
struct CloneSummary {
cloned: usize,
skipped: usize,
failed: Vec<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
struct ArchiveManifest {
format_version: String,
tool: ManifestTool,
source: ManifestSource,
archive: ManifestArchive,
files: Vec<ManifestFile>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
struct ManifestTool {
name: String,
version: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
struct ManifestSource {
platform: String,
target: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
struct ManifestArchive {
created_at_unix: u64,
complete: bool,
file_count: usize,
total_bytes: u64,
checksum_algorithm: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
struct ManifestFile {
path: String,
size: u64,
#[serde(default)]
modified_unix_nanos: Option<u64>,
blake3: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
struct VerifyReport {
archive: String,
ok: bool,
files_verified: usize,
bytes_verified: u64,
manifest_valid: bool,
checksums_valid: bool,
path_safety_valid: bool,
warnings: Vec<String>,
errors: Vec<String>,
}
#[derive(Error, Debug)]
#[allow(dead_code)]
enum GitArchiverError {
#[error("invalid target '{0}'")]
InvalidTarget(String),
#[error("GitHub API returned {status}: {body}")]
ApiError { status: u16, body: String },
#[error("GitHub entity '{0}' was not found")]
NotFound(String),
#[error("repo '{0}' is private, only public repos can be cloned")]
PrivateRepo(String),
#[error("output path '{0}' is not a directory")]
OutputNotDirectory(String),
#[error("'git' executable not found in PATH")]
MissingGit,
#[error("clone failed for '{repo}': {message}")]
CloneFailed { repo: String, message: String },
#[error(
"Hugging Face API returned {status}: {body}\n hint: set HF_TOKEN env or pass --hf-token to authenticate (https://huggingface.co/settings/tokens)"
)]
HfApiError { status: u16, body: String },
#[error("Hugging Face model '{0}' was not found")]
HfNotFound(String),
#[error("Hugging Face download failed for '{model}': {message}")]
HfDownloadFailed { model: String, message: String },
#[error("Ollama API returned {status}: {body}")]
OllamaApiError { status: u16, body: String },
#[error("Ollama model '{0}' was not found")]
OllamaNotFound(String),
#[error("Ollama download failed for '{model}': {message}")]
OllamaDownloadFailed { model: String, message: String },
#[error("CivitAI API returned {status}: {body}")]
CivitaiApiError { status: u16, body: String },
#[error("CivitAI model '{0}' was not found")]
CivitaiNotFound(String),
#[error("CivitAI download failed for '{model}': {message}")]
CivitaiDownloadFailed { model: String, message: String },
#[error("'ollama' executable not found in PATH")]
MissingOllama,
#[error("{0}")]
Message(String),
#[error("I/O error: {0}")]
Io(#[from] io::Error),
#[error("request error: {0}")]
Request(#[from] reqwest::Error),
}
type Result<T> = std::result::Result<T, GitArchiverError>;
struct TerminalSession {
terminal: Terminal,
}
impl TerminalSession {
fn new() -> Result<Self> {
let mut terminal = Terminal::init_with(TerminalOptions {
mouse_capture: true,
..TerminalOptions::default()
})?;
terminal.clear();
Ok(Self { terminal })
}
}
impl Drop for TerminalSession {
fn drop(&mut self) {
let _ = self.terminal.restore();
}
}
fn draw_full_frame<F>(terminal: &mut Terminal, f: F) -> io::Result<()>
where
F: FnOnce(&mut Frame<'_>),
{
terminal.draw(f)?;
let frame = terminal.back_buffer().to_ansi_string();
let mut stdout = io::stdout();
write!(stdout, "\x1b[H{frame}")?;
stdout.flush()?;
Ok(())
}
trait FrameRenderExt {
fn render_block_widget<'item, W: Widget>(&mut self, area: Rect, widget: W, block: Block<'item>);
fn render_block_stateful_list<'item>(
&mut self,
area: Rect,
list: &List<'item>,
state: &mut ListState,
block: Block<'item>,
);
}
impl FrameRenderExt for Frame<'_> {
fn render_block_widget<'item, W: Widget>(
&mut self,
area: Rect,
widget: W,
block: Block<'item>,
) {
block.render(self.buffer(), area);
let inner = block.inner(area);
if inner.width > 0 && inner.height > 0 {
self.buffer()
.fill(inner, ' ', theme_white(), Some(theme_surface()));
widget.render(self.buffer(), inner);
normalize_bg(self.buffer(), inner, theme_surface());
}
}
fn render_block_stateful_list<'b>(
&mut self,
area: Rect,
list: &List<'b>,
state: &mut ListState,
block: Block<'b>,
) {
block.render(self.buffer(), area);
let inner = block.inner(area);
if inner.width > 0 && inner.height > 0 {
self.buffer()
.fill(inner, ' ', theme_white(), Some(theme_surface()));
list.render_stateful(self.buffer(), inner, state);
normalize_bg(self.buffer(), inner, theme_surface());
}
}
}
#[derive(Debug, Default)]
struct UiState {
show_details: bool,
abort_requested: bool,
panel_lines: usize,
}
fn main() {
if let Err(err) = run() {
eprintln!("error: {err}");
std::process::exit(1);
}
}
fn run() -> Result<()> {
let mut args = Cli::parse();
if let Some(command) = args.command.clone() {
return match command {
Commands::Archive { dry_run, source } => {
run_archive_command(source, args.dry_run || dry_run)
}
Commands::Verify { archive, json } => run_verify_command(&archive, json),
};
}
if args.token.is_none() {
args.token = std::env::var("GITHUB_TOKEN").ok();
}
if args.hf_token.is_none() {
args.hf_token = std::env::var("HF_TOKEN").ok();
}
if args.civitai_token.is_none() {
args.civitai_token = std::env::var("CIVITAI_TOKEN").ok();
}
let interactive = args.tui || args.target.is_none();
if args.tui && !io::stdout().is_terminal() {
return Err(GitArchiverError::Message(
"--tui requires an interactive terminal".to_string(),
));
}
let mut terminal_session = if interactive && io::stdout().is_terminal() {
Some(TerminalSession::new()?)
} else {
None
};
if interactive {
let mut screen = TuiScreen::Mode;
let mut site_options = SiteTuiOptions::default();
loop {
match screen {
TuiScreen::Mode => {
if let Some(session) = terminal_session.as_mut() {
match prompt_mode_tui(&mut session.terminal)? {
Some(mode) => {
args.mode = mode;
if mode == Mode::Models {
screen = TuiScreen::Provider;
} else {
screen = TuiScreen::Target;
}
}
None => return Ok(()),
}
} else {
args.mode = prompt_mode()?;
if args.mode == Mode::Models {
screen = TuiScreen::Provider;
} else {
screen = TuiScreen::Target;
}
}
}
TuiScreen::Provider => {
if let Some(session) = terminal_session.as_mut() {
match prompt_model_provider_tui(&mut session.terminal, args.provider)? {
Some(provider) => {
args.provider = provider;
screen = TuiScreen::Target;
}
None => screen = TuiScreen::Mode,
}
} else {
args.provider = prompt_model_provider(args.provider)?;
screen = TuiScreen::Target;
}
}
TuiScreen::Target => {
if args.target.is_some() {
screen = if args.mode == Mode::Sites {
TuiScreen::SiteOptions
} else {
TuiScreen::HfToken
};
continue;
}
if let Some(session) = terminal_session.as_mut() {
match prompt_target_tui_inner(
&mut session.terminal,
args.mode,
Some(args.provider),
)? {
Some(target) => {
args.target = Some(target);
screen = if args.mode == Mode::Sites {
TuiScreen::SiteOptions
} else {
TuiScreen::HfToken
};
}
None => {
if args.mode == Mode::Models {
screen = TuiScreen::Provider;
} else {
screen = TuiScreen::Mode;
}
}
}
} else {
args.target = Some(prompt_target(args.mode, Some(args.provider))?);
screen = if args.mode == Mode::Sites {
TuiScreen::SiteOptions
} else {
TuiScreen::HfToken
};
}
}
TuiScreen::SiteOptions => {
if let Some(session) = terminal_session.as_mut() {
match prompt_site_options_tui(
&mut session.terminal,
args.target.as_deref().unwrap_or_default(),
site_options,
)? {
Some(options) => {
site_options = options;
break;
}
None => screen = TuiScreen::Target,
}
} else {
site_options = prompt_site_options(site_options)?;
break;
}
}
TuiScreen::HfToken => {
if args.mode == Mode::Models
&& args.hf_token.is_none()
&& args.provider == ModelProvider::Hf
{
if let Some(session) = terminal_session.as_mut() {
match prompt_hf_token_tui(&mut session.terminal)? {
Some(token) => args.hf_token = Some(token),
None => {
screen = TuiScreen::Target;
continue;
}
}
} else if io::stdout().is_terminal() {
args.hf_token = prompt_hf_token()?;
}
}
break;
}
}
}
if args.mode == Mode::Sites {
return run_sites_mode(&mut args, &mut terminal_session, Some(site_options));
}
}
let client = Client::builder()
.user_agent("thesa")
.build()
.map_err(GitArchiverError::Request)?;
match args.mode {
Mode::Repos => run_repos_mode(&mut args, &client, &mut terminal_session),
Mode::Models => run_models_mode(&mut args, &client, &mut terminal_session),
Mode::Sites => run_sites_mode(&mut args, &mut terminal_session, None),
}
}
fn run_repos_mode(
args: &mut Cli,
client: &Client,
terminal_session: &mut Option<TerminalSession>,
) -> Result<()> {
let target = parse_target(args.target.as_deref().unwrap_or_default())?;
if args.output == Path::new("./archives") {
args.output = default_output_path(&target);
}
let output = prepare_output_dir(&args.output)?;
let mut repos = if let Some(session) = terminal_session.as_mut() {
fetch_repositories_tui(
&mut session.terminal,
client,
&target,
args.token.as_deref(),
)?
} else {
fetch_repositories_for_target(client, &target, args.token.as_deref())?
};
let interactive = args.tui || args.target.is_some();
if interactive {
repos = match &target {
Target::Owner(_) if terminal_session.is_some() => {
let session = terminal_session.as_mut().expect("checked above");
pick_repositories_tui_inner(&mut session.terminal, &repos, &mut args.concurrency)?
}
Target::Owner(_) => pick_repositories_interactively(&repos)?,
Target::Repo { .. } => repos,
};
} else if let Some(pattern) = args.filter.as_deref() {
repos.retain(|repo| repo.name.contains(pattern));
}
if repos.is_empty() {
*terminal_session = None;
println!("No repositories matched your request.");
return Ok(());
}
if args.dry_run {
*terminal_session = None;
println!("Dry run: {} repositories would be processed", repos.len());
for repo in &repos {
println!(" - {}", repo.full_name);
}
return Ok(());
}
let mut ui_state = UiState {
show_details: true,
..UiState::default()
};
let summary = if let Some(session) = terminal_session.as_mut() {
clone_repositories_scrin_inner(&mut session.terminal, &repos, &output, args, &mut ui_state)?
} else if io::stdout().is_terminal() {
clone_repositories_scrin(&repos, &output, args, &mut ui_state)?
} else {
let (ui_tx, ui_rx) = mpsc::channel::<char>();
let _ui_thread = thread::spawn(move || {
let stdin = io::stdin();
loop {
let mut line = String::new();
if stdin.read_line(&mut line).is_err() || line.is_empty() {
break;
}
let mut chars = line.chars().filter(|c| !c.is_whitespace());
if let Some(command) = chars.next() {
if ui_tx.send(command).is_err() {
break;
}
}
}
});
if args.concurrency > 1 {
clone_repositories_batched(&repos, &output, args, &ui_rx, &mut ui_state)?
} else {
clone_repositories_serial(&repos, &output, args, &ui_rx, &mut ui_state)?
}
};
*terminal_session = None;
if ui_state.abort_requested {
println!("Aborted by user input");
}
let manifest = write_archive_manifest(
&output,
"github",
&target.archive_target_id(),
!ui_state.abort_requested && summary.failed.is_empty(),
)?;
println!(
"\nCompleted: {} cloned, {} skipped, {} failed",
summary.cloned,
summary.skipped,
summary.failed.len()
);
println!(
"Manifest: {}/manifest.json ({} files, {} bytes)",
THESA_METADATA_DIR, manifest.archive.file_count, manifest.archive.total_bytes
);
if !summary.failed.is_empty() {
eprintln!("Failures:");
for item in summary.failed {
eprintln!(" - {item}");
}
return Err(GitArchiverError::Message(
"some repositories failed".to_string(),
));
}
Ok(())
}
fn run_models_mode(
args: &mut Cli,
client: &Client,
terminal_session: &mut Option<TerminalSession>,
) -> Result<()> {
let target =
parse_model_target_for_provider(args.target.as_deref().unwrap_or_default(), args.provider)?;
if args.output == Path::new("./archives") {
args.output = PathBuf::from("./archives").join("models");
}
let output = prepare_output_dir(&args.output)?;
let models = if let Some(session) = terminal_session.as_mut() {
fetch_models_tui(&mut session.terminal, client, &target, args)?
} else {
fetch_models_for_target(client, &target, args)?
};
let picked = if let Some(session) = terminal_session.as_mut() {
pick_models_tui_inner(&mut session.terminal, &models, &mut args.concurrency)?
} else if io::stdout().is_terminal() {
pick_models_interactively(&models)?
} else {
models
};
if picked.is_empty() {
*terminal_session = None;
println!("No models matched your request.");
return Ok(());
}
if args.dry_run {
*terminal_session = None;
println!("Dry run: {} models would be downloaded", picked.len());
for model in &picked {
println!(
" - [{}] {} (downloads: {}, size: {})",
model.provider.short_label(),
model.id,
format_downloads(model.downloads),
format_model_size(model.size),
);
}
return Ok(());
}
let mut ui_state = UiState {
show_details: true,
..UiState::default()
};
let summary = if let Some(session) = terminal_session.as_mut() {
download_models_scrin_inner(&mut session.terminal, &picked, &output, args, &mut ui_state)?
} else if io::stdout().is_terminal() {
download_models_scrin(&picked, &output, args, &mut ui_state)?
} else {
download_models_serial(&picked, &output, args, &mut ui_state)?
};
*terminal_session = None;
if ui_state.abort_requested {
println!("Aborted by user input");
}
let manifest = write_archive_manifest(
&output,
&format!("model:{}", args.provider),
args.target.as_deref().unwrap_or("models"),
!ui_state.abort_requested && summary.failed.is_empty(),
)?;
println!(
"\nCompleted: {} downloaded, {} skipped, {} failed",
summary.cloned,
summary.skipped,
summary.failed.len()
);
println!(
"Manifest: {}/manifest.json ({} files, {} bytes)",
THESA_METADATA_DIR, manifest.archive.file_count, manifest.archive.total_bytes
);
if !summary.failed.is_empty() {
eprintln!("Failures:");
for item in summary.failed {
eprintln!(" - {item}");
}
return Err(GitArchiverError::Message(
"some model downloads failed".to_string(),
));
}
Ok(())
}
fn run_archive_command(source: ArchiveCommands, dry_run: bool) -> Result<()> {
match source {
ArchiveCommands::Refresh(args) => run_git_archive_refresh(&args, dry_run || args.dry_run),
ArchiveCommands::Site(args) | ArchiveCommands::Url(args) => {
if dry_run || args.dry_run {
print_site_archive_dry_run(&args)
} else {
run_site_archive(args)
}
}
ArchiveCommands::List(args) => run_siteforge_list(&args),
ArchiveCommands::Inspect(args) => run_siteforge_inspect(&args),
ArchiveCommands::Search(args) => run_siteforge_search(&args),
ArchiveCommands::Export(args) => run_siteforge_export(&args, dry_run),
ArchiveCommands::Pack(args) => run_siteforge_pack(&args, dry_run),
ArchiveCommands::Verify(args) => run_siteforge_verify(&args),
ArchiveCommands::Resume(args) => run_siteforge_resume(&args, dry_run),
ArchiveCommands::Diagnostics(args) => run_siteforge_diagnostics(&args),
}
}
fn run_git_archive_refresh(args: &GitRefreshArgs, dry_run: bool) -> Result<()> {
let archives = repoforge::discover_git_archives(&args.archive_root, args.filter.as_deref())
.map_err(repoforge_err)?;
if archives.is_empty() {
println!("No Git archives found at {}", args.archive_root.display());
return Ok(());
}
if dry_run {
println!(
"Dry run: {} Git archives would be refreshed",
archives.len()
);
println!("Archive root: {}", args.archive_root.display());
if let Some(filter) = &args.filter {
println!("Filter: {filter}");
}
for archive in &archives {
let remote_note = if archive.remote_url.is_some() {
"origin"
} else {
"slug inferred from path"
};
println!(
" - {} ({}) [{}]",
archive.slug,
archive.path.display(),
remote_note
);
}
return Ok(());
}
println!(
"Refreshing {} Git archives under {}",
archives.len(),
args.archive_root.display()
);
let summary = repoforge::refresh_git_archives(&archives, args.concurrency.max(1));
let manifests =
write_git_refresh_manifests(&args.archive_root, &archives, &summary.failed_slugs())?;
let total = archives.len();
for (index, entry) in summary.entries.iter().enumerate() {
match entry.status {
repoforge::GitRefreshStatus::Updated => println!(
"[done] {} updated ({}/{}) {}",
entry.slug,
index + 1,
total,
entry.path.display()
),
repoforge::GitRefreshStatus::Unchanged => println!(
"[skip] {} up-to-date ({}/{}) {}",
entry.slug,
index + 1,
total,
entry.path.display()
),
}
}
for failure in &summary.failed {
eprintln!(
"[fail] {} {}: {}",
failure.slug,
failure.path.display(),
failure.message
);
}
println!(
"\nCompleted: {} updated, {} up-to-date, {} failed",
summary.updated,
summary.unchanged,
summary.failed.len()
);
if !manifests.is_empty() {
println!("Manifests refreshed:");
for manifest in &manifests {
println!(" - {}", manifest.display());
}
}
if !summary.failed.is_empty() {
eprintln!("Failures:");
for failure in &summary.failed {
eprintln!(" - {}: {}", failure.slug, failure.message);
}
return Err(GitArchiverError::Message(
"some Git archives failed to refresh".to_string(),
));
}
Ok(())
}
fn repoforge_err(err: repoforge::RepoForgeError) -> GitArchiverError {
GitArchiverError::Message(format!("repoforge refresh failed: {err}"))
}
fn write_git_refresh_manifests(
root: &Path,
archives: &[repoforge::GitArchiveCandidate],
failed_slugs: &BTreeSet<String>,
) -> Result<Vec<PathBuf>> {
let mut groups: BTreeMap<PathBuf, Vec<repoforge::GitArchiveCandidate>> = BTreeMap::new();
for archive in archives {
groups
.entry(git_archive_group_root(root, &archive.path))
.or_default()
.push(archive.clone());
}
let mut manifests = Vec::new();
for (group_root, group_archives) in groups {
let target = git_refresh_manifest_target(root, &group_root, &group_archives);
let complete = group_archives
.iter()
.all(|archive| !failed_slugs.contains(&archive.slug));
write_archive_manifest(&group_root, "github", &target, complete)?;
manifests.push(group_root.join(THESA_METADATA_DIR).join("manifest.json"));
}
Ok(manifests)
}
fn git_archive_group_root(root: &Path, repo_path: &Path) -> PathBuf {
if let Ok(relative) = repo_path.strip_prefix(root) {
for component in relative.components() {
if let Component::Normal(first) = component {
return root.join(first);
}
}
}
repo_path.to_path_buf()
}
fn git_refresh_manifest_target(
root: &Path,
group_root: &Path,
archives: &[repoforge::GitArchiveCandidate],
) -> String {
if let Ok(manifest) = read_archive_manifest_sidecar(group_root) {
if manifest.source.platform == "github" && !manifest.source.target.trim().is_empty() {
return manifest.source.target;
}
}
let group_name = group_root
.strip_prefix(root)
.ok()
.and_then(|relative| relative.components().next())
.and_then(|component| match component {
Component::Normal(value) => value.to_str().map(ToString::to_string),
_ => None,
})
.or_else(|| {
group_root
.file_name()
.and_then(|value| value.to_str())
.map(ToString::to_string)
})
.unwrap_or_else(|| "github".to_string());
let owners = archives
.iter()
.filter_map(|archive| {
archive
.slug
.split_once('/')
.map(|(owner, _)| owner.to_string())
})
.collect::<BTreeSet<_>>();
if owners.len() == 1 && owners.contains(&group_name) {
return group_name;
}
if archives.len() == 1 {
return archives[0].slug.clone();
}
group_name
}
fn siteforge_config_from_root(root: &SiteforgeRootArgs) -> SiteforgeConfig {
SiteforgeConfig {
archive_root: root.archive_root.clone(),
..SiteforgeConfig::default()
}
}
fn siteforge_err(context: &str, err: impl std::fmt::Display) -> GitArchiverError {
GitArchiverError::Message(format!("siteforge {context} failed: {err}"))
}
fn run_siteforge_list(args: &SiteforgeRootArgs) -> Result<()> {
let config = siteforge_config_from_root(args);
let root = config
.resolved_archive_root()
.map_err(|err| siteforge_err("list", err))?;
let archives = Siteforge::new(config)
.list_archives()
.map_err(|err| siteforge_err("list", err))?;
if archives.is_empty() {
println!("No Siteforge archives found at {}", root.display());
return Ok(());
}
for archive in archives {
println!(
"{} updated={} pages={}/{} failed={} seeds={}",
archive.archive_id,
archive.updated_at,
archive.stats.pages_parsed,
archive.stats.pages_discovered,
archive.stats.pages_failed,
archive.seeds.join(", ")
);
}
Ok(())
}
fn run_siteforge_inspect(args: &SiteforgeArchiveIdArgs) -> Result<()> {
let config = siteforge_config_from_root(&args.root);
let manifest = siteforge::archive::load_archive_manifest(&config, &args.archive_id)
.map_err(|err| siteforge_err("inspect", err))?;
let layout = SiteforgeArchiveLayout::new(
config
.resolved_archive_root()
.map_err(|err| siteforge_err("inspect", err))?,
args.archive_id.clone(),
);
println!("archive_id: {}", manifest.archive_id);
println!("created_at: {}", manifest.created_at);
println!("updated_at: {}", manifest.updated_at);
println!("path: {}", layout.base().display());
println!("seeds: {}", manifest.seeds.join(", "));
println!("full_site: {}", manifest.scope.full_site);
println!("same_domain: {}", manifest.scope.same_domain);
println!("max_depth: {}", manifest.scope.max_depth);
println!("archive_profile: {:?}", manifest.config.archive_profile);
println!("render_js: {}", manifest.config.render_js);
println!("render_js_auto: {}", manifest.config.render_js_auto);
println!("pages_discovered: {}", manifest.stats.pages_discovered);
println!("pages_parsed: {}", manifest.stats.pages_parsed);
println!("pages_failed: {}", manifest.stats.pages_failed);
println!("assets_downloaded: {}", manifest.stats.assets_downloaded);
if layout.database().exists() {
let storage = siteforge::storage::Storage::open(&layout.database())
.map_err(|err| siteforge_err("inspect", err))?;
println!(
"pages_indexed: {}",
storage
.page_count()
.map_err(|err| siteforge_err("inspect", err))?
);
println!(
"assets_indexed: {}",
storage
.asset_count()
.map_err(|err| siteforge_err("inspect", err))?
);
println!(
"asset_bytes: {}",
storage
.total_asset_size_bytes()
.map_err(|err| siteforge_err("inspect", err))?
);
let recent_pages = storage
.recent_pages_limited(5)
.map_err(|err| siteforge_err("inspect", err))?;
if !recent_pages.is_empty() {
println!("recent_pages:");
for page in recent_pages {
println!(" - [{}] {} ({})", page.status, page.title, page.source_url);
}
}
}
Ok(())
}
fn run_siteforge_search(args: &SiteforgeSearchArgs) -> Result<()> {
let config = siteforge_config_from_root(&args.root);
let results = Siteforge::new(config)
.search(args.archive_id.as_deref(), &args.query, args.limit)
.map_err(|err| siteforge_err("search", err))?;
if results.is_empty() {
println!("No matches.");
return Ok(());
}
for result in results {
println!("{} | {}", result.archive_id, result.title);
println!(" {}", result.source_url);
if let Some(heading) = result.matched_heading.as_deref() {
println!(" heading: {heading}");
}
println!(" snippet: {}", result.snippet.replace('\n', " "));
if let Some(path) = result.local_file_path.as_deref() {
println!(" local: {path}");
}
}
Ok(())
}
fn run_siteforge_export(args: &SiteforgeExportArgs, dry_run: bool) -> Result<()> {
if dry_run {
println!("Dry run: siteforge would export {}", args.archive_id);
println!("Archive root: {}", args.root.archive_root.display());
println!("Format: {}", args.format.as_str());
if let Some(output) = &args.output {
println!("Output: {}", output.display());
}
return Ok(());
}
let config = siteforge_config_from_root(&args.root);
let path = Siteforge::new(config)
.export(&args.archive_id, args.format, args.output.as_deref())
.map_err(|err| siteforge_err("export", err))?;
println!("Exported: {}", path.display());
Ok(())
}
fn run_siteforge_pack(args: &SiteforgePackArgs, dry_run: bool) -> Result<()> {
if dry_run {
println!(
"Dry run: siteforge would create AI packs for {}",
args.archive_id
);
println!("Archive root: {}", args.root.archive_root.display());
if let Some(max_tokens) = args.max_tokens {
println!("Max tokens: {max_tokens}");
}
return Ok(());
}
let config = siteforge_config_from_root(&args.root);
let packs = Siteforge::new(config)
.create_ai_packs(&args.archive_id, args.max_tokens)
.map_err(|err| siteforge_err("pack", err))?;
for pack in packs {
println!("Pack: {}", pack.display());
}
Ok(())
}
fn run_siteforge_verify(args: &SiteforgeArchiveIdArgs) -> Result<()> {
let config = siteforge_config_from_root(&args.root);
let problems = Siteforge::new(config)
.verify(&args.archive_id)
.map_err(|err| siteforge_err("verify", err))?;
if problems.is_empty() {
println!("Siteforge archive verified: {}", args.archive_id);
} else {
println!(
"Siteforge archive verification found {} problem(s):",
problems.len()
);
for problem in problems {
println!("- {problem}");
}
}
Ok(())
}
fn run_siteforge_resume(args: &SiteforgeArchiveIdArgs, dry_run: bool) -> Result<()> {
if dry_run {
println!("Dry run: siteforge would resume {}", args.archive_id);
println!("Archive root: {}", args.root.archive_root.display());
return Ok(());
}
let config = siteforge_config_from_root(&args.root);
let runtime = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.map_err(|err| {
GitArchiverError::Message(format!("failed to start async runtime: {err}"))
})?;
let summary = runtime
.block_on(Siteforge::new(config).resume(&args.archive_id))
.map_err(|err| siteforge_err("resume", err))?;
println!("Resumed archive: {}", summary.archive_id);
println!("Archive: {}", summary.archive_path.display());
println!(
"Pages: fetched={} parsed={} failed={}",
summary.stats.pages_fetched, summary.stats.pages_parsed, summary.stats.pages_failed
);
Ok(())
}
fn run_siteforge_diagnostics(args: &SiteforgeDiagnosticsArgs) -> Result<()> {
let config = siteforge_config_from_root(&args.root);
let root = config
.resolved_archive_root()
.map_err(|err| siteforge_err("diagnostics", err))?;
println!("archive_root: {}", root.display());
println!("default_concurrency: {}", config.default_concurrency);
println!("archive_profile: {:?}", config.archive_profile);
println!("render_js: {}", config.render_js);
println!("render_js_auto: {}", config.render_js_auto);
println!("max_pages: {}", config.max_pages);
println!("max_page_size_bytes: {}", config.max_page_size_bytes);
println!("max_asset_size_bytes: {}", config.max_asset_size_bytes);
println!(
"max_total_archive_size_bytes: {}",
config.max_total_archive_size_bytes
);
if let Some(archive_id) = &args.archive_id {
let manifest = siteforge::archive::load_archive_manifest(&config, archive_id)
.map_err(|err| siteforge_err("diagnostics", err))?;
let layout = SiteforgeArchiveLayout::new(root, archive_id.clone());
println!("archive_id: {}", manifest.archive_id);
println!("archive_path: {}", layout.base().display());
println!(
"archive_size_bytes: {}",
siteforge::archive::archive_size_bytes(&layout)
.map_err(|err| siteforge_err("diagnostics", err))?
);
if layout.database().exists() {
let storage = siteforge::storage::Storage::open(&layout.database())
.map_err(|err| siteforge_err("diagnostics", err))?;
println!(
"jobs: {}",
storage
.job_count()
.map_err(|err| siteforge_err("diagnostics", err))?
);
for job in storage
.job_snapshots_limited(20)
.map_err(|err| siteforge_err("diagnostics", err))?
{
println!(
"- job={} status={} active_url={}",
job.job_id,
job.status,
job.active_url.as_deref().unwrap_or("none")
);
}
}
return Ok(());
}
let archives = Siteforge::new(config)
.list_archives()
.map_err(|err| siteforge_err("diagnostics", err))?;
println!("archives: {}", archives.len());
for archive in archives.into_iter().take(50) {
println!(
"- {} updated={} parsed={} failed={}",
archive.archive_id,
archive.updated_at,
archive.stats.pages_parsed,
archive.stats.pages_failed
);
}
Ok(())
}
fn print_site_archive_dry_run(args: &SiteArchiveArgs) -> Result<()> {
let seeds = load_site_seed_urls(args.url.as_deref(), args.input.as_deref())?;
let headers = parse_site_headers(&args.headers)?;
println!(
"Dry run: siteforge would archive {}",
site_seed_display(&seeds)
);
println!("Output root: {}", args.output.display());
if let Some(input) = &args.input {
println!("Input seeds: {}", input.display());
}
println!(
"Mode: {}",
if args.full_site {
"same-domain full-site crawl"
} else {
"single page"
}
);
println!(
"Cross-domain: {}",
if args.allow_cross_domain {
"allowed"
} else {
"blocked"
}
);
println!(
"Max depth: {}",
args.max_depth
.map(|value| value.to_string())
.unwrap_or_else(|| "config default".to_string())
);
println!(
"Max pages: {}",
args.max_pages
.map(|value| value.to_string())
.unwrap_or_else(|| "config default".to_string())
);
println!(
"Concurrency: {}",
args.concurrency
.map(|value| value.to_string())
.unwrap_or_else(|| "config default".to_string())
);
println!(
"Delay: {}",
args.delay_ms
.map(|value| format!("{value} ms"))
.unwrap_or_else(|| "config default".to_string())
);
if !args.include_url_patterns.is_empty() {
println!("Include globs: {}", args.include_url_patterns.join(", "));
}
if !args.exclude_url_patterns.is_empty() {
println!("Exclude globs: {}", args.exclude_url_patterns.join(", "));
}
if let Some(archive_profile) = args.archive_profile {
println!(
"Archive profile: {}",
site_archive_profile_label(archive_profile.as_output_profile())
);
}
if let Some(max_page_size_bytes) = args.max_page_size_bytes {
println!("Max page size: {max_page_size_bytes} bytes");
}
if let Some(max_asset_size_bytes) = args.max_asset_size_bytes {
println!("Max asset size: {max_asset_size_bytes} bytes");
}
if let Some(max_total_archive_size_bytes) = args.max_total_archive_size_bytes {
println!("Max total archive size: {max_total_archive_size_bytes} bytes");
}
println!(
"Render JS: {}",
site_render_mode_label(args.render_js, args.render_js_auto)
);
if let Some(browser_command) = &args.browser_command {
println!("Browser command: {browser_command}");
}
if let Some(browser_profile) = &args.browser_profile {
println!("Browser profile: {}", browser_profile.display());
}
if let Some(render_wait_ms) = args.render_wait_ms {
println!("Render wait: {render_wait_ms} ms");
}
if let Some(user_agent) = &args.user_agent {
println!("User-Agent: {user_agent}");
}
if let Some(timeout_secs) = args.timeout_secs {
println!("Timeout: {timeout_secs} seconds");
}
if let Some(retry_count) = args.retry_count {
println!("Retries: {retry_count}");
}
println!(
"OCR: {}",
if args.ocr_enabled {
"enabled"
} else {
"config default"
}
);
if !headers.is_empty() {
let names = headers
.iter()
.map(|(name, _)| name.as_str())
.collect::<Vec<_>>()
.join(", ");
println!("Headers: {} configured ({names})", headers.len());
}
if args.cookie.is_some() {
println!("Cookie: configured (redacted)");
}
if let Some(archive_id) = &args.archive_id {
println!("Archive ID: {archive_id}");
}
Ok(())
}
fn load_site_seed_urls(primary: Option<&str>, input: Option<&Path>) -> Result<Vec<reqwest::Url>> {
let mut seeds = Vec::new();
if let Some(primary) = primary {
seeds.push(parse_site_url(primary)?);
}
if let Some(input) = input {
let raw = fs::read_to_string(input)?;
for (line_index, line) in raw.lines().enumerate() {
let line = line.trim();
if line.is_empty() || line.starts_with('#') {
continue;
}
let seed = parse_site_url(line).map_err(|err| {
GitArchiverError::Message(format!(
"invalid site URL in {} at line {}: {err}",
input.display(),
line_index + 1
))
})?;
seeds.push(seed);
}
}
if seeds.is_empty() {
return Err(GitArchiverError::Message(
"siteforge archive requires a URL or --input seed file".to_string(),
));
}
Ok(seeds)
}
fn site_seed_display(seeds: &[reqwest::Url]) -> String {
match seeds {
[] => "no seeds".to_string(),
[seed] => seed.as_str().to_string(),
[seed, rest @ ..] => format!("{} (+{} more seeds)", seed.as_str(), rest.len()),
}
}
fn site_source_target(seeds: &[reqwest::Url]) -> String {
match seeds {
[] => "siteforge".to_string(),
[seed] => seed.as_str().to_string(),
[seed, rest @ ..] => format!("{} (+{} seeds)", seed.as_str(), rest.len()),
}
}
fn parse_site_headers(values: &[String]) -> Result<Vec<(String, String)>> {
values
.iter()
.map(|value| {
let (name, header_value) = value.split_once(':').ok_or_else(|| {
GitArchiverError::Message(format!(
"invalid header {value:?}; expected 'Name: Value'"
))
})?;
let name = name.trim();
let header_value = header_value.trim();
if name.is_empty() || header_value.is_empty() {
return Err(GitArchiverError::Message(format!(
"invalid header {value:?}; name and value are required"
)));
}
Ok((name.to_string(), header_value.to_string()))
})
.collect()
}
fn run_sites_mode(
args: &mut Cli,
terminal_session: &mut Option<TerminalSession>,
tui_options: Option<SiteTuiOptions>,
) -> Result<()> {
let url = args
.target
.as_deref()
.unwrap_or_default()
.trim()
.to_string();
parse_site_url(&url)?;
*terminal_session = None;
let output = if args.output == Path::new("./archives") {
PathBuf::from("./archives/sites")
} else {
args.output.clone()
};
let full_site = tui_options.map(|options| options.full_site).unwrap_or(true);
let max_depth = tui_options.map(|options| options.max_depth);
let max_pages = tui_options.map(|options| options.max_pages);
let archive_profile = tui_options.map(|options| options.archive_profile);
let render_js = tui_options
.map(|options| options.render_js)
.unwrap_or(false);
let render_js_auto = tui_options
.map(|options| options.render_js_auto)
.unwrap_or(false);
if args.dry_run {
println!("Dry run: siteforge would archive {url}");
println!("Output root: {}", output.display());
println!(
"Mode: {}",
if full_site {
"same-domain full-site crawl"
} else {
"single page"
}
);
if let Some(max_depth) = max_depth {
println!("Max depth: {max_depth}");
}
if let Some(max_pages) = max_pages {
println!("Max pages: {max_pages}");
}
if let Some(archive_profile) = archive_profile {
println!(
"Archive profile: {}",
site_archive_profile_label(archive_profile)
);
}
println!(
"Render JS: {}",
site_render_mode_label(render_js, render_js_auto)
);
return Ok(());
}
run_site_archive(SiteArchiveArgs {
url: Some(url),
input: None,
output,
full_site,
allow_cross_domain: false,
max_depth,
max_pages,
concurrency: Some(args.concurrency),
delay_ms: None,
archive_id: None,
include_url_patterns: Vec::new(),
exclude_url_patterns: Vec::new(),
archive_profile: archive_profile.map(SiteArchiveProfile::from),
max_page_size_bytes: None,
max_asset_size_bytes: None,
max_total_archive_size_bytes: None,
render_js,
render_js_auto,
browser_command: None,
browser_profile: None,
render_wait_ms: None,
headers: Vec::new(),
cookie: None,
user_agent: None,
timeout_secs: None,
retry_count: None,
ocr_enabled: false,
dry_run: false,
})
}
fn run_site_archive(args: SiteArchiveArgs) -> Result<()> {
let mut config = SiteforgeConfig {
archive_root: args.output.clone(),
..SiteforgeConfig::default()
};
if let Some(concurrency) = args.concurrency {
config.default_concurrency = concurrency.max(1);
}
if let Some(delay_ms) = args.delay_ms {
config.default_delay_ms = delay_ms;
}
if let Some(max_depth) = args.max_depth {
config.max_depth = max_depth;
}
if let Some(max_pages) = args.max_pages {
config.max_pages = max_pages;
}
if let Some(archive_profile) = args.archive_profile {
config.archive_profile = archive_profile.as_output_profile();
}
if let Some(max_page_size_bytes) = args.max_page_size_bytes {
config.max_page_size_bytes = max_page_size_bytes;
}
if let Some(max_asset_size_bytes) = args.max_asset_size_bytes {
config.max_asset_size_bytes = max_asset_size_bytes;
}
if let Some(max_total_archive_size_bytes) = args.max_total_archive_size_bytes {
config.max_total_archive_size_bytes = max_total_archive_size_bytes;
}
if args.render_js {
config.render_js = true;
config.render_js_auto = false;
}
if args.render_js_auto {
config.render_js = true;
config.render_js_auto = true;
}
if let Some(browser_command) = &args.browser_command {
config.browser_command = Some(browser_command.clone());
}
if let Some(browser_profile) = &args.browser_profile {
config.browser_profile = Some(browser_profile.clone());
}
if let Some(render_wait_ms) = args.render_wait_ms {
config.render_wait_ms = render_wait_ms;
}
if let Some(user_agent) = &args.user_agent {
config.user_agent = user_agent.clone();
}
if let Some(timeout_secs) = args.timeout_secs {
config.timeout_secs = timeout_secs;
}
if let Some(retry_count) = args.retry_count {
config.retry_count = retry_count;
}
if args.ocr_enabled {
config.ocr_enabled = true;
}
if !args.include_url_patterns.is_empty() {
config.include_url_patterns = args.include_url_patterns.clone();
}
if !args.exclude_url_patterns.is_empty() {
config
.exclude_url_patterns
.extend(args.exclude_url_patterns.clone());
}
let seeds = load_site_seed_urls(args.url.as_deref(), args.input.as_deref())?;
let source_target = site_source_target(&seeds);
let seed_display = site_seed_display(&seeds);
let headers = parse_site_headers(&args.headers)?;
let options = SiteforgeArchiveRunOptions {
seeds,
full_site: args.full_site,
same_domain: !args.allow_cross_domain,
max_depth: args.max_depth.unwrap_or(config.max_depth),
max_pages: args.max_pages.unwrap_or(config.max_pages),
concurrency: args
.concurrency
.unwrap_or(config.default_concurrency)
.max(1),
delay_ms: args.delay_ms.unwrap_or(config.default_delay_ms),
headers,
cookie: args.cookie.clone(),
archive_id: args.archive_id.clone(),
};
println!("Archiving website with siteforge: {seed_display}");
let runtime = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.map_err(|err| {
GitArchiverError::Message(format!("failed to start async runtime: {err}"))
})?;
let siteforge = Siteforge::new(config);
let summary = runtime
.block_on(siteforge.archive(options))
.map_err(|err| {
GitArchiverError::Message(siteforge_error_message(&err.to_string(), &args.output))
})?;
let manifest = read_archive_manifest_sidecar(&summary.archive_path).or_else(|_| {
write_archive_manifest(&summary.archive_path, "siteforge", &source_target, true)
})?;
print_siteforge_summary(
&summary.archive_id,
&summary.archive_path,
&manifest,
&summary.stats,
);
Ok(())
}
fn siteforge_error_message(message: &str, output_root: &Path) -> String {
if message.contains("already exists") {
format!(
"siteforge archive failed: {message}\n output root: {}\n suggestion: pass --archive-id with a new id, remove the existing archive, or use siteforge resume directly",
output_root.display()
)
} else {
format!("siteforge archive failed: {message}")
}
}
fn print_siteforge_summary(
archive_id: &str,
archive_path: &Path,
manifest: &ArchiveManifest,
stats: &SiteforgeCrawlStats,
) {
let layout = SiteforgeArchiveLayout::new(
archive_path
.parent()
.map(Path::to_path_buf)
.unwrap_or_else(|| archive_path.to_path_buf()),
archive_id.to_string(),
);
println!("Archive ID: {archive_id}");
println!("Archive: {}", archive_path.display());
println!("Open/read first:");
print_existing_path(" AGENTS.md", &layout.agents_md());
print_existing_path(" Agent index", &layout.agent_index_json());
print_existing_path(" Readable markdown", &layout.readable_markdown_dir());
print_existing_path(" Readable text", &layout.readable_text_dir());
print_existing_path(" Chunks JSONL", &layout.chunks_jsonl());
println!("Machine/index data:");
print_existing_path(" Siteforge manifest", &layout.manifest());
print_existing_path(" Siteforge checksums", &layout.checksums());
print_existing_path(" Crawl database", &layout.database());
print_existing_path(" Raw pages", &layout.raw_pages_dir());
print_existing_path(" Raw assets", &layout.raw_assets_dir());
println!("Pages parsed: {}", stats.pages_parsed);
println!("Pages failed: {}", stats.pages_failed);
println!("Assets downloaded: {}", stats.assets_downloaded);
println!(
"Manifest: {}/manifest.json ({} files, {} bytes)",
THESA_METADATA_DIR, manifest.archive.file_count, manifest.archive.total_bytes
);
}
fn print_existing_path(label: &str, path: &Path) {
if path.exists() {
println!("{label}: {}", path.display());
} else {
println!("{label}: {} (not written)", path.display());
}
}
#[derive(Debug, Clone)]
enum ModelTarget {
Org(String),
Model(String),
Top,
Latest,
}
impl From<modelforge::ModelTarget> for ModelTarget {
fn from(value: modelforge::ModelTarget) -> Self {
match value {
modelforge::ModelTarget::Org(org) => Self::Org(org),
modelforge::ModelTarget::Model(model) => Self::Model(model),
modelforge::ModelTarget::Top => Self::Top,
modelforge::ModelTarget::Latest => Self::Latest,
}
}
}
fn parse_model_target_for_provider(input: &str, provider: ModelProvider) -> Result<ModelTarget> {
modelforge::parse_model_target_for_provider(input, provider.into())
.map(ModelTarget::from)
.map_err(|err| GitArchiverError::InvalidTarget(err.target().to_string()))
}
fn hf_auth_header(token: Option<&str>) -> Option<String> {
token.map(|t| format!("Bearer {t}"))
}
fn hf_api_base(mirror: bool) -> &'static str {
if mirror { HF_MIRROR_BASE } else { HF_BASE }
}
fn list_models_for_org(
client: &Client,
org: &str,
token: Option<&str>,
mirror: bool,
) -> Result<Vec<ModelItem>> {
list_hf_models_with_sort(client, Some(org), "downloads", -1, token, mirror)
}
fn list_hf_top_models(
client: &Client,
token: Option<&str>,
mirror: bool,
) -> Result<Vec<ModelItem>> {
list_hf_models_with_sort(client, None, "downloads", -1, token, mirror)
}
fn list_hf_latest_models(
client: &Client,
token: Option<&str>,
mirror: bool,
) -> Result<Vec<ModelItem>> {
list_hf_models_with_sort(client, None, "lastModified", -1, token, mirror)
}
fn list_hf_models_with_sort(
client: &Client,
org: Option<&str>,
sort: &str,
direction: i8,
token: Option<&str>,
mirror: bool,
) -> Result<Vec<ModelItem>> {
let base = hf_api_base(mirror);
let mut page = 1usize;
let mut models = Vec::new();
loop {
let mut url = format!(
"{}/api/models?sort={}&direction={}&limit={}&page={}",
base,
urlencoding::encode(sort),
direction,
DEFAULT_PAGE_SIZE,
page
);
if let Some(org) = org {
url.push_str("&author=");
url.push_str(&urlencoding::encode(org));
}
let mut request = client.get(&url);
if let Some(auth) = hf_auth_header(token) {
request = request.header("Authorization", auth);
}
let response = request.send()?;
let status = response.status();
if status == StatusCode::NOT_FOUND {
if let Some(org) = org {
return Err(GitArchiverError::HfNotFound(org.to_string()));
}
return Err(GitArchiverError::HfApiError {
status: status.as_u16(),
body: "empty response from Hugging Face listing".to_string(),
});
}
if !status.is_success() {
return Err(parse_hf_api_error(response));
}
let batch: Vec<HfModelResponse> = response.json()?;
let count = batch.len();
models.extend(batch.into_iter().map(|m| ModelItem {
id: if let Some(ref mid) = m.model_id {
mid.clone()
} else {
m.id
},
downloads: m.downloads,
pipeline_tag: m.pipeline_tag,
last_modified: m.last_modified,
provider: ModelProvider::Hf,
download_url: None,
size: None,
}));
if count < DEFAULT_PAGE_SIZE {
break;
}
page = page.saturating_add(1);
}
Ok(models)
}
fn fetch_single_model(
client: &Client,
model_id: &str,
token: Option<&str>,
mirror: bool,
) -> Result<ModelItem> {
let base = hf_api_base(mirror);
let url = format!("{}/api/models/{model_id}", base);
let mut request = client.get(&url);
if let Some(auth) = hf_auth_header(token) {
request = request.header("Authorization", auth);
}
let response = request.send()?;
if response.status() == StatusCode::NOT_FOUND {
return Err(GitArchiverError::HfNotFound(model_id.to_string()));
}
if !response.status().is_success() {
return Err(parse_hf_api_error(response));
}
let data: HfModelResponse = response.json()?;
Ok(ModelItem {
id: if let Some(ref mid) = data.model_id {
mid.clone()
} else {
data.id
},
downloads: data.downloads,
pipeline_tag: data.pipeline_tag,
last_modified: data.last_modified,
provider: ModelProvider::Hf,
download_url: None,
size: None,
})
}
fn fetch_models_for_target(
client: &Client,
target: &ModelTarget,
args: &Cli,
) -> Result<Vec<ModelItem>> {
match args.provider {
ModelProvider::Hf => match target {
ModelTarget::Org(org) => {
list_models_for_org(client, org, args.hf_token.as_deref(), args.hf_mirror)
}
ModelTarget::Model(id) => Ok(vec![fetch_single_model(
client,
id,
args.hf_token.as_deref(),
args.hf_mirror,
)?]),
ModelTarget::Top => {
list_hf_top_models(client, args.hf_token.as_deref(), args.hf_mirror)
}
ModelTarget::Latest => {
list_hf_latest_models(client, args.hf_token.as_deref(), args.hf_mirror)
}
},
ModelProvider::Ollama => match target {
ModelTarget::Org(org) => list_ollama_models(client, org),
ModelTarget::Model(id) => Ok(vec![fetch_single_ollama_model(client, id)?]),
ModelTarget::Top => list_ollama_top_models(client),
ModelTarget::Latest => list_ollama_latest_models(client),
},
ModelProvider::Civitai => match target {
ModelTarget::Org(org) if org.chars().all(|c| c.is_ascii_digit()) => {
if let Ok(model) =
fetch_single_civitai_model(client, &org, args.civitai_token.as_deref())
{
return Ok(vec![model]);
}
list_civitai_models(
client,
Some(&org),
"Most+Downloaded",
args.civitai_token.as_deref(),
)
}
ModelTarget::Org(org) => list_civitai_models(
client,
Some(org),
"Most+Downloaded",
args.civitai_token.as_deref(),
),
ModelTarget::Model(id) => Ok(vec![fetch_single_civitai_model(
client,
id,
args.civitai_token.as_deref(),
)?]),
ModelTarget::Top => list_civitai_models(
client,
None,
"Most+Downloaded",
args.civitai_token.as_deref(),
),
ModelTarget::Latest => {
list_civitai_models(client, None, "Newest", args.civitai_token.as_deref())
}
},
}
}
fn list_ollama_models(client: &Client, org: &str) -> Result<Vec<ModelItem>> {
list_ollama_models_by_query(client, Some(org), true)
}
fn list_ollama_top_models(client: &Client) -> Result<Vec<ModelItem>> {
let mut models = list_ollama_models_by_query(client, None, false)?;
models.sort_by(|a, b| b.size.cmp(&a.size));
Ok(models)
}
fn list_ollama_latest_models(client: &Client) -> Result<Vec<ModelItem>> {
let mut models = list_ollama_models_by_query(client, None, false)?;
models.sort_by(|a, b| {
let b_modified = b.last_modified.as_deref().unwrap_or("");
let a_modified = a.last_modified.as_deref().unwrap_or("");
b_modified.cmp(a_modified)
});
Ok(models)
}
fn list_ollama_models_by_query(
client: &Client,
query: Option<&str>,
fallback_on_empty: bool,
) -> Result<Vec<ModelItem>> {
let local_url = format!("{}/api/tags", OLLAMA_LOCAL);
let query = query.map(str::to_lowercase).unwrap_or_default();
if let Ok(resp) = client
.get(&local_url)
.timeout(Duration::from_secs(2))
.send()
{
if resp.status().is_success() {
if let Ok(data) = resp.json::<OllamaModelResponse>() {
let models = data
.models
.into_iter()
.filter(|m| query.is_empty() || m.name.to_lowercase().contains(&query))
.map(|m| ModelItem {
id: m.name.clone(),
downloads: 0,
pipeline_tag: None,
last_modified: Some(m.modified_at),
provider: ModelProvider::Ollama,
download_url: None,
size: Some(m.size),
})
.collect::<Vec<_>>();
if !models.is_empty() {
return Ok(models);
}
}
}
}
if fallback_on_empty && !query.is_empty() {
return Ok(vec![ModelItem {
id: query.to_string(),
downloads: 0,
pipeline_tag: None,
last_modified: None,
provider: ModelProvider::Ollama,
download_url: None,
size: None,
}]);
}
Ok(Vec::new())
}
fn fetch_single_ollama_model(_client: &Client, model_id: &str) -> Result<ModelItem> {
Ok(ModelItem {
id: model_id.to_string(),
downloads: 0,
pipeline_tag: None,
last_modified: None,
provider: ModelProvider::Ollama,
download_url: None,
size: None,
})
}
fn list_civitai_models(
client: &Client,
query: Option<&str>,
sort: &str,
token: Option<&str>,
) -> Result<Vec<ModelItem>> {
let mut url = format!(
"{}/models?limit={}&sort={}",
CIVITAI_API, DEFAULT_PAGE_SIZE, sort
);
if let Some(query) = query {
if !query.is_empty() {
url.push_str("&query=");
url.push_str(&urlencoding::encode(query));
}
}
let mut request = client.get(&url);
if let Some(token) = token {
request = request.header("Authorization", format!("Bearer {token}"));
}
let response = request.send()?;
let status = response.status();
if !status.is_success() {
let body = response
.text()
.unwrap_or_else(|_| "<unable to read body>".to_string());
return Err(GitArchiverError::CivitaiApiError {
status: status.as_u16(),
body,
});
}
let data: CivitaiModelResponse = response.json()?;
let models = data
.items
.into_iter()
.map(|m| {
let (download_url, size) = m
.model_versions
.as_ref()
.and_then(|v| v.first())
.map(|v| (v.download_url.clone(), v.size))
.unwrap_or((None, None));
ModelItem {
id: m.name.clone(),
downloads: m.downloads,
pipeline_tag: m.model_type,
last_modified: None,
provider: ModelProvider::Civitai,
download_url,
size,
}
})
.collect::<Vec<_>>();
if models.is_empty() {
return Err(GitArchiverError::CivitaiNotFound(
query.unwrap_or("civitai top models").to_string(),
));
}
Ok(models)
}
fn fetch_single_civitai_model(
client: &Client,
model_id: &str,
token: Option<&str>,
) -> Result<ModelItem> {
let numeric_id: u64 = model_id.parse().map_err(|_| {
GitArchiverError::CivitaiNotFound(format!(
"{model_id} (CivitAI requires numeric model ID, e.g. 827184)"
))
})?;
let url = format!("{}/models/{}", CIVITAI_API, numeric_id);
let mut request = client.get(&url);
if let Some(token) = token {
request = request.header("Authorization", format!("Bearer {token}"));
}
let response = request.send()?;
let status = response.status();
if status == StatusCode::NOT_FOUND {
return Err(GitArchiverError::CivitaiNotFound(model_id.to_string()));
}
if !status.is_success() {
let body = response
.text()
.unwrap_or_else(|_| "<unable to read body>".to_string());
return Err(GitArchiverError::CivitaiApiError {
status: status.as_u16(),
body,
});
}
let m: CivitaiModel = response.json()?;
let (download_url, size) = m
.model_versions
.as_ref()
.and_then(|v| v.first())
.map(|v| (v.download_url.clone(), v.size))
.unwrap_or((None, None));
Ok(ModelItem {
id: m.name.clone(),
downloads: m.downloads,
pipeline_tag: m.model_type,
last_modified: None,
provider: ModelProvider::Civitai,
download_url,
size,
})
}
fn parse_hf_api_error(response: Response) -> GitArchiverError {
let status = response.status();
let body = response
.text()
.unwrap_or_else(|_| "<unable to read body>".to_string());
GitArchiverError::HfApiError {
status: status.as_u16(),
body,
}
}
fn download_model(model: &ModelItem, output: &Path, args: &Cli) -> Result<CloneStatus> {
match model.provider {
ModelProvider::Hf => download_hf_model(model, output, args),
ModelProvider::Ollama => download_ollama_model(model, output, args),
ModelProvider::Civitai => download_civitai_model(model, output, args),
}
}
fn download_hf_model(model: &ModelItem, output: &Path, args: &Cli) -> Result<CloneStatus> {
let destination = output.join(&model.id.replace('/', "_"));
if destination.exists() {
if args.skip_existing {
return Ok(CloneStatus::Skipped);
}
if destination.is_dir() {
remove_dir_all(&destination)?;
} else {
remove_file(&destination)?;
}
}
let child = spawn_hf_clone(model, output, args)?;
let status = child.wait_with_output()?;
if status.status.success() {
Ok(CloneStatus::Cloned)
} else {
let code = status
.status
.code()
.map_or_else(|| "signal/unknown".to_string(), |c| c.to_string());
Err(GitArchiverError::HfDownloadFailed {
model: model.id.clone(),
message: format!("exit code {code}"),
})
}
}
fn download_ollama_model(model: &ModelItem, output: &Path, args: &Cli) -> Result<CloneStatus> {
let destination = output.join(&model.id.replace('/', "_").replace(':', "_"));
if destination.exists() {
if args.skip_existing {
return Ok(CloneStatus::Skipped);
}
if destination.is_dir() {
remove_dir_all(&destination)?;
} else {
remove_file(&destination)?;
}
}
fs::create_dir_all(&destination)?;
let command_args = vec!["pull".to_string(), model.id.clone()];
let child = match Command::new("ollama")
.current_dir(&destination)
.args(&command_args)
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
{
Ok(child) => child,
Err(error) if error.kind() == io::ErrorKind::NotFound => {
return Err(GitArchiverError::MissingOllama);
}
Err(error) => return Err(error.into()),
};
let status = child.wait_with_output()?;
if status.status.success() {
Ok(CloneStatus::Cloned)
} else {
let code = status
.status
.code()
.map_or_else(|| "signal/unknown".to_string(), |c| c.to_string());
Err(GitArchiverError::OllamaDownloadFailed {
model: model.id.clone(),
message: format!("exit code {code}"),
})
}
}
fn download_civitai_model(model: &ModelItem, output: &Path, args: &Cli) -> Result<CloneStatus> {
let dir_name = model.id.replace('/', "_");
let destination = output.join(&dir_name);
if destination.exists() {
if args.skip_existing {
return Ok(CloneStatus::Skipped);
}
if destination.is_dir() {
remove_dir_all(&destination)?;
} else {
remove_file(&destination)?;
}
}
let url =
model
.download_url
.as_deref()
.ok_or_else(|| GitArchiverError::CivitaiDownloadFailed {
model: model.id.clone(),
message: "no download URL available".to_string(),
})?;
let mut request = client_get().get(url);
if let Some(token) = args.civitai_token.as_deref() {
request = request.header("Authorization", format!("Bearer {token}"));
}
let response = request.send()?;
let status = response.status();
if !status.is_success() {
let body = response
.text()
.unwrap_or_else(|_| "<unable to read body>".to_string());
return Err(GitArchiverError::CivitaiDownloadFailed {
model: model.id.clone(),
message: format!("HTTP {status}: {body}"),
});
}
let bytes = response.bytes()?;
fs::create_dir_all(&destination)?;
let filename = url.rsplit('/').next().unwrap_or("model.safetensors");
let filepath = destination.join(filename);
fs::write(&filepath, &bytes)?;
Ok(CloneStatus::Cloned)
}
fn client_get() -> Client {
Client::builder()
.user_agent("thesa")
.build()
.unwrap_or_else(|_| Client::new())
}
fn spawn_hf_clone(model: &ModelItem, output: &Path, args: &Cli) -> Result<Child> {
let url = format!("https://huggingface.co/{}", model.id);
let dir_name = model.id.replace('/', "_");
let mut command_args = vec!["clone".to_string()];
if let Some(depth) = args.depth {
command_args.push("--depth".to_string());
command_args.push(depth.to_string());
}
command_args.push(url);
command_args.push(dir_name);
match Command::new("git")
.current_dir(output)
.args(&command_args)
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
{
Ok(child) => Ok(child),
Err(error) if error.kind() == io::ErrorKind::NotFound => Err(GitArchiverError::MissingGit),
Err(error) => Err(error.into()),
}
}
fn prompt_mode_tui(terminal: &mut Terminal) -> Result<Option<Mode>> {
let mut cursor = 0usize;
let mut frame_index = 0usize;
loop {
draw_full_frame(terminal, |frame| {
draw_mode_selector_tui(frame, cursor, frame_index);
})?;
frame_index = frame_index.wrapping_add(1);
if !event::poll(Duration::from_millis(50))? {
continue;
}
match event::read()? {
Event::Mouse(mouse) => {
if let Some(inner) = mode_selector_list_inner(terminal.size()) {
match list_mouse_action(inner, MODE_OPTION_COUNT, cursor, &mouse) {
Some(ListMouseAction::Hover(index)) => cursor = index,
Some(ListMouseAction::Click(index)) => {
cursor = index;
return Ok(Some(mode_from_index(cursor)));
}
Some(ListMouseAction::ScrollUp) => cursor = cursor.saturating_sub(1),
Some(ListMouseAction::ScrollDown) => {
if cursor + 1 < MODE_OPTION_COUNT {
cursor += 1;
}
}
None => {}
}
}
}
Event::Key(key) => {
if key.kind != KeyEventKind::Press {
continue;
}
match key.code {
KeyCode::Esc => return Ok(None),
KeyCode::Up | KeyCode::Left => cursor = cursor.saturating_sub(1),
KeyCode::Down | KeyCode::Right => {
if cursor + 1 < MODE_OPTION_COUNT {
cursor += 1;
}
}
KeyCode::Enter => {
return Ok(Some(mode_from_index(cursor)));
}
KeyCode::Char('r') => return Ok(Some(Mode::Repos)),
KeyCode::Char('m') => return Ok(Some(Mode::Models)),
KeyCode::Char('s') | KeyCode::Char('u') => return Ok(Some(Mode::Sites)),
_ => {}
}
}
_ => {}
}
}
}
fn mode_from_index(index: usize) -> Mode {
match index {
1 => Mode::Models,
2 => Mode::Sites,
_ => Mode::Repos,
}
}
fn prompt_model_provider_tui(
terminal: &mut Terminal,
current: ModelProvider,
) -> Result<Option<ModelProvider>> {
let mut cursor = model_provider_index(current);
let mut frame_index = 0usize;
loop {
draw_full_frame(terminal, |frame| {
draw_model_provider_selector_tui(frame, cursor, frame_index);
})?;
frame_index = frame_index.wrapping_add(1);
if !event::poll(Duration::from_millis(50))? {
continue;
}
match event::read()? {
Event::Mouse(mouse) => {
if let Some(inner) = provider_selector_list_inner(terminal.size()) {
match list_mouse_action(inner, 3, cursor, &mouse) {
Some(ListMouseAction::Hover(index)) => cursor = index,
Some(ListMouseAction::Click(index)) => {
cursor = index;
return Ok(Some(model_provider_from_index(cursor)));
}
Some(ListMouseAction::ScrollUp) => cursor = cursor.saturating_sub(1),
Some(ListMouseAction::ScrollDown) => {
if cursor + 1 < 3 {
cursor += 1;
}
}
None => {}
}
}
}
Event::Key(key) => {
if key.kind != KeyEventKind::Press {
continue;
}
match key.code {
KeyCode::Esc => return Ok(None),
KeyCode::Up | KeyCode::Left => cursor = cursor.saturating_sub(1),
KeyCode::Down | KeyCode::Right => {
if cursor + 1 < 3 {
cursor += 1;
}
}
KeyCode::Enter => return Ok(Some(model_provider_from_index(cursor))),
KeyCode::Char('h') | KeyCode::Char('1') => return Ok(Some(ModelProvider::Hf)),
KeyCode::Char('o') | KeyCode::Char('2') => {
return Ok(Some(ModelProvider::Ollama));
}
KeyCode::Char('c') | KeyCode::Char('3') => {
return Ok(Some(ModelProvider::Civitai));
}
_ => {}
}
}
_ => {}
}
}
}
fn model_provider_index(provider: ModelProvider) -> usize {
match provider {
ModelProvider::Hf => 0,
ModelProvider::Ollama => 1,
ModelProvider::Civitai => 2,
}
}
fn model_provider_from_index(index: usize) -> ModelProvider {
match index {
1 => ModelProvider::Ollama,
2 => ModelProvider::Civitai,
_ => ModelProvider::Hf,
}
}
fn prompt_target_tui_inner(
terminal: &mut Terminal,
mode: Mode,
provider: Option<ModelProvider>,
) -> Result<Option<String>> {
let mut input = String::new();
let mut frame_index = 0usize;
let mut status = match mode {
Mode::Repos => "enter GitHub owner or repo name".to_string(),
Mode::Models => match provider.unwrap_or(ModelProvider::Hf) {
ModelProvider::Hf => "enter Hugging Face org, org/model, top, or latest".to_string(),
ModelProvider::Ollama => "enter Ollama model name, top, or latest".to_string(),
ModelProvider::Civitai => "enter CivitAI model id, keyword, top, or latest".to_string(),
},
Mode::Sites => "enter a website URL for Siteforge".to_string(),
};
loop {
draw_full_frame(terminal, |frame| {
draw_target_tui(
frame,
&input,
&status,
frame_index,
mode,
if matches!(mode, Mode::Models) {
provider
} else {
None
},
);
})?;
frame_index = frame_index.wrapping_add(1);
if !event::poll(Duration::from_millis(50))? {
continue;
}
let Event::Key(key) = event::read()? else {
continue;
};
if key.kind != KeyEventKind::Press {
continue;
}
match key.code {
KeyCode::Esc if input.is_empty() => return Ok(None),
KeyCode::Esc => {
input.clear();
status = match mode {
Mode::Repos => "enter GitHub owner or repo name".to_string(),
Mode::Models => match provider.unwrap_or(ModelProvider::Hf) {
ModelProvider::Hf => {
"enter Hugging Face org, org/model, top, or latest".to_string()
}
ModelProvider::Ollama => {
"enter Ollama model name, top, or latest".to_string()
}
ModelProvider::Civitai => {
"enter CivitAI model id, keyword, top, or latest".to_string()
}
},
Mode::Sites => "enter a website URL for Siteforge".to_string(),
};
}
KeyCode::Enter => {
let target = input.trim().to_string();
if target.is_empty() {
status = "name is required".to_string();
continue;
}
let parse_result = match mode {
Mode::Repos => parse_target(&target).map(|_| target.clone()),
Mode::Models => parse_model_target_for_provider(
&target,
provider.unwrap_or(ModelProvider::Hf),
)
.map(|_| target.clone()),
Mode::Sites => parse_site_url(&target).map(|_| target.clone()),
};
if parse_result.is_ok() {
return Ok(Some(target));
}
status = match mode {
Mode::Repos => "try octocat or octocat/Spoon-Knife".to_string(),
Mode::Models => match provider.unwrap_or(ModelProvider::Hf) {
ModelProvider::Hf => "try meta-llama or meta-llama/Llama-2-7b".to_string(),
ModelProvider::Ollama => "try llama3, latest, or qwen2.5:7b".to_string(),
ModelProvider::Civitai => {
"try 827184 or top, realistic-vision-v6".to_string()
}
},
Mode::Sites => "try https://example.com or http://127.0.0.1:8765/".to_string(),
};
}
KeyCode::Backspace => {
input.pop();
}
KeyCode::Char(ch) => {
input.push(ch);
}
_ => {}
}
}
}
fn prompt_site_options_tui(
terminal: &mut Terminal,
url: &str,
initial: SiteTuiOptions,
) -> Result<Option<SiteTuiOptions>> {
let mut options = initial;
let mut cursor = 0usize;
let mut frame_index = 0usize;
let mut status = "tune Siteforge crawl options, then press Enter".to_string();
loop {
draw_full_frame(terminal, |frame| {
draw_site_options_tui(frame, url, options, cursor, &status, frame_index);
})?;
frame_index = frame_index.wrapping_add(1);
if !event::poll(Duration::from_millis(50))? {
continue;
}
let input_event = event::read()?;
if let Event::Mouse(mouse) = input_event {
if let Some(inner) = site_options_list_inner(terminal.size()) {
match list_mouse_action(inner, SITE_OPTION_COUNT, cursor, &mouse) {
Some(ListMouseAction::Hover(index)) => cursor = index,
Some(ListMouseAction::Click(index)) => {
cursor = index;
status = apply_site_option_row(&mut options, cursor);
}
Some(ListMouseAction::ScrollUp) => cursor = cursor.saturating_sub(1),
Some(ListMouseAction::ScrollDown) => {
if cursor + 1 < SITE_OPTION_COUNT {
cursor += 1;
}
}
None => {}
}
}
continue;
}
let Event::Key(key) = input_event else {
continue;
};
if key.kind != KeyEventKind::Press {
continue;
}
match key.code {
KeyCode::Esc => return Ok(None),
KeyCode::Enter => return Ok(Some(options)),
KeyCode::Up => cursor = cursor.saturating_sub(1),
KeyCode::Down => {
if cursor + 1 < SITE_OPTION_COUNT {
cursor += 1;
}
}
KeyCode::Left | KeyCode::Right | KeyCode::Char(' ') => {
status = apply_site_option_row(&mut options, cursor);
}
KeyCode::Char('f') => {
options.full_site = !options.full_site;
status = site_crawl_status(options);
}
KeyCode::Char('d') => {
options.max_depth = next_site_depth(options.max_depth);
status = format!("max depth set to {}", options.max_depth);
}
KeyCode::Char('p') => {
options.max_pages = next_site_pages(options.max_pages);
status = format!("max pages set to {}", options.max_pages);
}
KeyCode::Char('j') => status = cycle_site_render_mode(&mut options),
KeyCode::Char('l') => {
options.archive_profile = next_site_archive_profile(options.archive_profile);
status = format!(
"archive profile set to {}",
site_archive_profile_label(options.archive_profile)
);
}
_ => {}
}
}
}
fn prompt_site_options(initial: SiteTuiOptions) -> Result<SiteTuiOptions> {
println!("Siteforge options:");
println!(
" crawl: {}",
if initial.full_site {
"same-domain full-site"
} else {
"single page"
}
);
println!(" max depth: {}", initial.max_depth);
println!(" max pages: {}", initial.max_pages);
println!(
" archive profile: {}",
site_archive_profile_label(initial.archive_profile)
);
println!(
" render JS: {}",
site_render_mode_label(initial.render_js, initial.render_js_auto)
);
let value = prompt_line("Use defaults? [Y/n]: ")?;
if matches!(value.as_str(), "n" | "N" | "no" | "No") {
println!("Using safe defaults for now; use command-form flags for detailed tuning.");
}
Ok(initial)
}
fn apply_site_option_row(options: &mut SiteTuiOptions, row: usize) -> String {
match row {
0 => {
options.full_site = !options.full_site;
site_crawl_status(*options)
}
1 => {
options.max_depth = next_site_depth(options.max_depth);
format!("max depth set to {}", options.max_depth)
}
2 => {
options.max_pages = next_site_pages(options.max_pages);
format!("max pages set to {}", options.max_pages)
}
3 => cycle_site_render_mode(options),
4 => {
options.archive_profile = next_site_archive_profile(options.archive_profile);
format!(
"archive profile set to {}",
site_archive_profile_label(options.archive_profile)
)
}
_ => "press Enter to run Siteforge".to_string(),
}
}
fn cycle_site_render_mode(options: &mut SiteTuiOptions) -> String {
match (options.render_js, options.render_js_auto) {
(false, false) => {
options.render_js = false;
options.render_js_auto = true;
}
(false, true) => {
options.render_js = true;
options.render_js_auto = false;
}
_ => {
options.render_js = false;
options.render_js_auto = false;
}
}
format!(
"JavaScript rendering set to {}",
site_render_mode_label(options.render_js, options.render_js_auto)
)
}
fn site_render_mode_label(render_js: bool, render_js_auto: bool) -> &'static str {
match (render_js, render_js_auto) {
(_, true) => "auto",
(true, false) => "all",
(false, false) => "off",
}
}
fn next_site_archive_profile(current: ArchiveOutputProfile) -> ArchiveOutputProfile {
match current {
ArchiveOutputProfile::Full => ArchiveOutputProfile::Agent,
ArchiveOutputProfile::Agent => ArchiveOutputProfile::Lean,
ArchiveOutputProfile::Lean => ArchiveOutputProfile::Full,
}
}
fn site_archive_profile_label(profile: ArchiveOutputProfile) -> &'static str {
match profile {
ArchiveOutputProfile::Full => "full",
ArchiveOutputProfile::Agent => "agent",
ArchiveOutputProfile::Lean => "lean",
}
}
fn site_crawl_status(options: SiteTuiOptions) -> String {
if options.full_site {
"same-domain full-site crawl enabled".to_string()
} else {
"single-page archive selected".to_string()
}
}
fn next_site_depth(current: usize) -> usize {
match current {
1 => 2,
2 => 4,
4 => 8,
_ => 1,
}
}
fn next_site_pages(current: usize) -> usize {
match current {
25 => 100,
100 => 500,
500 => 1000,
_ => 25,
}
}
fn pick_models_tui_inner(
terminal: &mut Terminal,
models: &[ModelItem],
batch_size: &mut usize,
) -> Result<Vec<ModelItem>> {
if models.is_empty() {
return Ok(vec![]);
}
let mut query = String::new();
let mut selected = BTreeSet::new();
let mut cursor = 0usize;
let mut frame_index = 0usize;
let mut batch_overlay = false;
let mut status =
"Space checks models; Enter downloads checked models, or visible if none checked"
.to_string();
loop {
let filtered = filtered_model_indices(models, &query);
if cursor >= filtered.len() {
cursor = filtered.len().saturating_sub(1);
}
let page_step = picker_page_step(terminal.size().height);
draw_full_frame(terminal, |frame| {
draw_model_picker_tui(
frame,
models,
&filtered,
&selected,
cursor,
&query,
&status,
frame_index,
*batch_size,
batch_overlay,
);
})?;
frame_index = frame_index.wrapping_add(1);
if !event::poll(Duration::from_millis(50))? {
continue;
}
let input_event = event::read()?;
if let Event::Mouse(mouse) = input_event {
if !batch_overlay {
if let Some(inner) = picker_list_inner(terminal.size(), 104) {
match list_mouse_action(inner, filtered.len(), cursor, &mouse) {
Some(ListMouseAction::Hover(index)) => cursor = index,
Some(ListMouseAction::Click(index)) => {
cursor = index;
if let Some(model_index) = filtered.get(cursor).copied() {
if !selected.insert(model_index) {
selected.remove(&model_index);
}
let checked = selected.len();
status = if checked == 0 {
"nothing checked; Enter downloads visible models".to_string()
} else {
format!(
"{checked} checked; Enter downloads checked models only"
)
};
}
}
Some(ListMouseAction::ScrollUp) => cursor = cursor.saturating_sub(1),
Some(ListMouseAction::ScrollDown) => {
if cursor + 1 < filtered.len() {
cursor += 1;
}
}
None => {}
}
}
}
continue;
}
let Event::Key(key) = input_event else {
continue;
};
if key.kind != KeyEventKind::Press {
continue;
}
if batch_overlay {
match key.code {
KeyCode::Esc | KeyCode::Char('b') | KeyCode::Enter => {
batch_overlay = false;
status = format!("batch size set to {}; Enter downloads models", *batch_size);
}
KeyCode::Char(ch) => {
if let Some(value) = batch_size_from_key(ch) {
*batch_size = value;
batch_overlay = false;
status = format!("batch size set to {value}; Enter downloads models");
}
}
KeyCode::Left | KeyCode::Up => {
*batch_size = previous_batch_size(*batch_size);
status = format!("batch size set to {}; Enter accepts", *batch_size);
}
KeyCode::Right | KeyCode::Down => {
*batch_size = next_batch_size(*batch_size);
status = format!("batch size set to {}; Enter accepts", *batch_size);
}
_ => {}
}
continue;
}
match key.code {
KeyCode::Esc => return Err(GitArchiverError::Message("aborted by user".to_string())),
KeyCode::Enter => {
let picked = if selected.is_empty() {
filtered
} else {
selected.iter().copied().collect::<Vec<_>>()
};
if picked.is_empty() {
status = "no models selected or visible".to_string();
continue;
}
return Ok(picked
.into_iter()
.map(|index| models[index].clone())
.collect());
}
KeyCode::Up => cursor = cursor.saturating_sub(1),
KeyCode::Down => {
if cursor + 1 < filtered.len() {
cursor += 1;
}
}
KeyCode::PageUp => {
let step = page_step.min(filtered.len().max(1));
cursor = cursor.saturating_sub(step);
status = if filtered.is_empty() {
"no models to move to".to_string()
} else {
format!("moved up by {step}")
};
}
KeyCode::PageDown => {
if filtered.is_empty() {
status = "no models to move to".to_string();
} else {
let step = page_step.min(filtered.len());
cursor = (cursor + step).min(filtered.len() - 1);
status = format!("moved down by {step}");
}
}
KeyCode::Home => {
if !filtered.is_empty() {
cursor = 0;
status = "jumped to first model".to_string();
} else {
status = "no models to move to".to_string();
}
}
KeyCode::End => {
if !filtered.is_empty() {
cursor = filtered.len() - 1;
status = "jumped to last model".to_string();
} else {
status = "no models to move to".to_string();
}
}
KeyCode::Char(' ') => {
if let Some(model_index) = filtered.get(cursor).copied() {
if !selected.insert(model_index) {
selected.remove(&model_index);
}
let checked = selected.len();
status = if checked == 0 {
"nothing checked; Enter downloads visible models".to_string()
} else {
format!("{checked} checked; Enter downloads checked models only")
};
}
}
KeyCode::Char('a') => {
for model_index in &filtered {
selected.insert(*model_index);
}
status = format!("checked {} visible models", filtered.len());
}
KeyCode::Char('c') => {
selected.clear();
query.clear();
cursor = 0;
status = "cleared filter and selection".to_string();
}
KeyCode::Char('b') => {
batch_overlay = true;
status = "choose batch size: 1 2 3 6 8".to_string();
}
KeyCode::Backspace => {
query.pop();
cursor = 0;
}
KeyCode::Char(ch) => {
query.push(ch);
cursor = 0;
}
_ => {}
}
}
}
fn filtered_model_indices(models: &[ModelItem], query: &str) -> Vec<usize> {
if query.trim().is_empty() {
return (0..models.len()).collect();
}
let query = query.to_lowercase();
models
.iter()
.enumerate()
.filter(|(_, m)| m.id.to_lowercase().contains(&query))
.map(|(index, _)| index)
.collect()
}
fn download_models_scrin(
models: &[ModelItem],
output: &Path,
args: &Cli,
ui_state: &mut UiState,
) -> Result<CloneSummary> {
let mut session = TerminalSession::new()?;
download_models_scrin_inner(&mut session.terminal, models, output, args, ui_state)
}
fn download_models_scrin_inner(
terminal: &mut Terminal,
models: &[ModelItem],
output: &Path,
args: &Cli,
ui_state: &mut UiState,
) -> Result<CloneSummary> {
let batch_size = args.concurrency.max(1);
let total = models.len();
let total_batches = total.div_ceil(batch_size);
let mut summary = CloneSummary::default();
let mut effect_kind = EffectKind::Waves;
let started_at = Instant::now();
let mut tick = 0usize;
let mut batch_index = 1;
while batch_index <= total {
let batch_end = (batch_index + batch_size - 1).min(total);
let batch = &models[batch_index - 1..batch_end];
let names: Vec<String> = batch.iter().map(|m| m.id.clone()).collect();
let mut state = vec![BatchState::Pending; batch.len()];
let (tx, rx) = mpsc::channel::<(usize, BatchEvent)>();
let mut handles = Vec::new();
for (offset, model) in batch.iter().enumerate() {
let model = model.clone();
let output = output.to_path_buf();
let args = args.clone();
let tx = tx.clone();
handles.push((
model.id.clone(),
thread::spawn(move || {
let started = tx.send((offset, BatchEvent::Started));
if started.is_err() {
return;
}
let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
download_model(&model, &output, &args)
}));
let event = match outcome {
Ok(Ok(CloneStatus::Cloned)) => BatchEvent::Result(BatchResult::Done),
Ok(Ok(CloneStatus::Skipped)) => BatchEvent::Result(BatchResult::Skipped),
Ok(Err(err)) => BatchEvent::Result(BatchResult::Error(err.to_string())),
Err(_) => BatchEvent::Result(BatchResult::Error(
"worker thread panicked while downloading".to_string(),
)),
};
let _ = tx.send((offset, event));
}),
));
}
drop(tx);
let mut completed = 0;
while completed < batch.len() {
while event::poll(Duration::from_millis(0))? {
if let Event::Key(key) = event::read()? {
match key.code {
KeyCode::Char('q') | KeyCode::Esc => ui_state.abort_requested = true,
KeyCode::Char('t') => ui_state.show_details = !ui_state.show_details,
KeyCode::Char('m') => effect_kind = EffectKind::Waves,
KeyCode::Char('b') => effect_kind = EffectKind::Burn,
KeyCode::Char('w') => effect_kind = EffectKind::Wipe,
_ => {}
}
}
}
while let Ok((offset, event)) = rx.try_recv() {
match event {
BatchEvent::Started => state[offset] = BatchState::Running,
BatchEvent::Result(BatchResult::Done) => {
state[offset] = BatchState::Done;
summary.cloned += 1;
completed += 1;
}
BatchEvent::Result(BatchResult::Skipped) => {
state[offset] = BatchState::Skipped;
summary.skipped += 1;
completed += 1;
}
BatchEvent::Result(BatchResult::Error(message)) => {
state[offset] = BatchState::Failed;
summary
.failed
.push(format!("{}: {}", names[offset], message));
completed += 1;
}
}
}
draw_full_frame(terminal, |frame| {
render_download_dashboard(
frame,
&names,
&state,
&summary,
batch_index,
batch_size,
total,
total_batches,
tick,
started_at.elapsed(),
effect_kind,
ui_state,
)
})?;
tick = tick.wrapping_add(1);
if completed < batch.len() {
thread::sleep(Duration::from_millis(80));
}
}
for (model_name, handle) in handles {
if handle.join().is_err() {
summary
.failed
.push(format!("{model_name}: worker thread panicked"));
}
}
if ui_state.abort_requested {
break;
}
batch_index = batch_end + 1;
}
Ok(summary)
}
fn download_models_serial(
models: &[ModelItem],
output: &Path,
args: &Cli,
ui_state: &mut UiState,
) -> Result<CloneSummary> {
let mut summary = CloneSummary::default();
let total = models.len();
for (index, model) in models.iter().enumerate() {
if ui_state.abort_requested {
break;
}
match download_model(model, output, args) {
Ok(CloneStatus::Cloned) => {
println!("[done] {} ({}/{})", model.id, index + 1, total);
summary.cloned += 1;
}
Ok(CloneStatus::Skipped) => {
println!("[skip] {} ({}/{})", model.id, index + 1, total);
summary.skipped += 1;
}
Err(err) => {
eprintln!("[fail] {} ({}/{}): {err}", model.id, index + 1, total);
summary.failed.push(format!("{}: {err}", model.id));
}
}
}
Ok(summary)
}
fn pick_repositories_interactively(repos: &[RepoItem]) -> Result<Vec<RepoItem>> {
if repos.is_empty() {
return Ok(vec![]);
}
let snapshot: Vec<RepoItem> = repos.to_vec();
loop {
println!("Found {} public repositories", snapshot.len());
let query = prompt_line("Search public repos (blank = all, q = quit): ")?;
if query.eq_ignore_ascii_case("q") {
return Err(GitArchiverError::Message("aborted by user".to_string()));
}
let filtered = if query.is_empty() {
snapshot.clone()
} else {
let normalized_query = query.to_lowercase();
snapshot
.iter()
.filter(|repo| repo.name.to_lowercase().contains(&normalized_query))
.cloned()
.collect::<Vec<_>>()
};
if filtered.is_empty() {
println!("No matches for '{query}'.");
continue;
}
for (index, repo) in filtered.iter().enumerate() {
println!("{:>3}. {}", index + 1, repo.full_name);
}
let selection = prompt_line("Clone [A]ll, specific indexes (1,3-5) or [q] to quit [a]: ")?;
if selection.is_empty() || selection.eq_ignore_ascii_case("a") {
return Ok(filtered);
}
if selection.eq_ignore_ascii_case("q") {
return Err(GitArchiverError::Message("aborted by user".to_string()));
}
match parse_indices(&selection, filtered.len()) {
Ok(indices) => {
let selected = indices
.into_iter()
.map(|idx| filtered[idx].clone())
.collect::<Vec<_>>();
return Ok(selected);
}
Err(message) => {
println!("{message}");
}
}
}
}
fn pick_models_interactively(models: &[ModelItem]) -> Result<Vec<ModelItem>> {
if models.is_empty() {
return Ok(vec![]);
}
let snapshot = models.to_vec();
loop {
println!("Found {} models", snapshot.len());
let query = prompt_line("Search models (blank = all, q = quit): ")?;
if query.eq_ignore_ascii_case("q") {
return Err(GitArchiverError::Message("aborted by user".to_string()));
}
let filtered = if query.is_empty() {
snapshot.clone()
} else {
let q = query.to_lowercase();
snapshot
.iter()
.filter(|m| m.id.to_lowercase().contains(&q))
.cloned()
.collect::<Vec<_>>()
};
if filtered.is_empty() {
println!("No matches for '{query}'.");
continue;
}
for (index, model) in filtered.iter().enumerate() {
println!("{:>3}. {} [dl: {}]", index + 1, model.id, model.downloads);
}
let selection =
prompt_line("Download [A]ll, specific indexes (1,3-5) or [q] to quit [a]: ")?;
if selection.is_empty() || selection.eq_ignore_ascii_case("a") {
return Ok(filtered);
}
if selection.eq_ignore_ascii_case("q") {
return Err(GitArchiverError::Message("aborted by user".to_string()));
}
match parse_indices(&selection, filtered.len()) {
Ok(indices) => {
return Ok(indices
.into_iter()
.map(|idx| filtered[idx].clone())
.collect());
}
Err(message) => {
println!("{message}");
}
}
}
}
fn draw_hf_token_tui(
frame: &mut Frame<'_>,
input: &str,
status: &str,
tick: usize,
show_plain: bool,
) {
let area = frame.area();
paint_surface(frame, area);
if area.width < 24 || area.height < 8 {
let line = if show_plain {
format!("HF token> {input}\n{status}")
} else {
format!("HF token> {}\n{status}", "*".repeat(input.len()))
};
frame.render_widget(
Paragraph::from_text(Text::new(vec![Line::styled(
&line,
Style::default().fg(theme_white()),
)])),
area,
);
return;
}
let layout = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(header_height(area)),
Constraint::Min(5),
Constraint::Length(2),
])
.split(area);
draw_thesa_header(frame, layout[0], tick, status, "hf auth");
let display_input = if show_plain {
input.to_string()
} else {
"*".repeat(input.len())
};
let body = Paragraph::from_text(Text::new(vec![
Line::new(vec![
Span::styled("HF token> ", prompt_style()),
Span::styled(&display_input, Style::default().fg(theme_white())),
]),
Line::raw(""),
Line::new(vec![
Span::styled("hint ", Style::default().fg(theme_dim())),
Span::styled(
"https://huggingface.co/settings/tokens",
Style::default().fg(theme_blue()),
),
]),
Line::raw(""),
Line::styled(
"Press Enter with blank to skip (will rate-limit).",
Style::default().fg(theme_dim()),
),
]))
.wrap(WrapMode::Character { trim: false });
frame.render_block_widget(
layout[1],
body,
knott_block(" Hugging Face Token ", theme_blue()),
);
draw_thesa_bottom_bar(frame, layout[2], status, &["Enter accept", "Esc back"]);
let input_area = panel_inner(layout[1]);
let cursor_x = input_area
.x
.saturating_add(10)
.saturating_add(input.chars().count() as u16)
.min(input_area.x + input_area.width.saturating_sub(1));
frame.set_cursor_position(cursor_x, input_area.y);
}
fn draw_mode_selector_tui(frame: &mut Frame<'_>, cursor: usize, tick: usize) {
let area = frame.area();
paint_surface(frame, area);
if area.width < 28 || area.height < 10 {
let line = format!(
"select mode: repos={} models={} sites={}",
cursor == 0,
cursor == 1,
cursor == 2,
);
frame.render_widget(
Paragraph::from_text(Text::new(vec![Line::styled(
&line,
Style::default().fg(theme_white()),
)])),
area,
);
return;
}
let layout = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(header_height(area)),
Constraint::Min(8),
Constraint::Length(2),
])
.split(area);
draw_thesa_header(
frame,
layout[0],
tick,
"choose an archive source",
"landing",
);
let body = Layout::default()
.direction(Direction::Horizontal)
.constraints(if layout[1].width >= 94 {
[Constraint::Percentage(62), Constraint::Percentage(38)]
} else {
[Constraint::Percentage(100), Constraint::Percentage(0)]
})
.split(layout[1]);
let modes = [
(
"repos",
"GitHub repositories (owner/org/repo)",
"r",
theme_blue(),
"clone + manifest",
),
(
"models",
"Hugging Face, Ollama, CivitAI models",
"m",
theme_green(),
"model artifacts",
),
(
"siteforge",
"Website archive from a URL",
"s",
TuiColor::rgb(163, 113, 247),
"site crawl",
),
];
let items = modes
.iter()
.enumerate()
.map(|(i, (name, desc, key, color, tag))| {
let selected = cursor == i;
let marker = if selected { ">>" } else { " " };
let style = if selected {
selected_style(tick)
} else {
Style::default().fg(theme_white())
};
ListItem::with_line(Line::new(vec![
Span::styled(
&format!(" {marker} "),
style.add_modifier(if selected {
Modifier::BOLD
} else {
Modifier::empty()
}),
),
Span::styled(
&format!("[{key}]"),
Style::default().fg(*color).add_modifier(Modifier::BOLD),
),
Span::raw(" "),
Span::styled(&name.to_uppercase(), style.add_modifier(Modifier::BOLD)),
Span::raw(" "),
Span::styled(desc, Style::default().fg(theme_dim())),
Span::raw(" "),
Span::styled(
&format!("{tag:<16}"),
Style::default().fg(*color).add_modifier(Modifier::BOLD),
),
]))
})
.collect::<Vec<_>>();
let list = List::new(&items)
.with_highlight_style(selected_style(tick))
.with_highlight_symbol(">>");
let mut state = ListState::default();
state.select(Some(cursor));
frame.render_block_stateful_list(
body[0],
&list,
&mut state,
knott_block(" Archive Source ", theme_blue()),
);
if body[1].width > 0 {
render_aisling_text_panel(
frame,
body[1],
" thesa command signal ",
EffectKind::Waves,
tick,
"thesa\narchive\nverify\nrestore",
vec![
section_line("Preservation Routes"),
help_pair_line("repos", "GitHub clone batches"),
help_pair_line("models", "HF/Ollama/CivitAI"),
help_pair_line("siteforge", "same-domain web crawl"),
help_pair_line("metadata", ".thesa manifest + BLAKE3"),
Line::raw(""),
section_line("Controls"),
help_pair_line("r / m / s", "jump source"),
help_pair_line("Enter", "open selected route"),
help_pair_line("Mouse", "click row"),
],
theme_blue(),
);
}
draw_thesa_bottom_bar(
frame,
layout[2],
"Up/Down or r/m/s to select, Enter to confirm, Esc back/quit",
&[
"Up/Down move",
"r repos",
"m models",
"s siteforge",
"Mouse click",
"Enter select",
"Esc back",
],
);
}
fn draw_model_provider_selector_tui(frame: &mut Frame<'_>, cursor: usize, tick: usize) {
let area = frame.area();
paint_surface(frame, area);
if area.width < 24 || area.height < 8 {
frame.render_widget(
Paragraph::from_text(Text::new(vec![Line::styled(
"Model provider\nhf\nollama\ncivitai",
Style::default().fg(theme_white()),
)])),
area,
);
return;
}
let layout = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(header_height(area)),
Constraint::Min(8),
Constraint::Length(3),
])
.split(area);
draw_thesa_header(
frame,
layout[0],
tick,
"select model provider before choosing target",
"model setup",
);
let providers = [
(
"hugging face",
"remote model hub: orgs, org/model, top, latest",
"h/1",
ModelProvider::Hf,
),
(
"ollama",
"local Ollama library: installed names, top, latest",
"o/2",
ModelProvider::Ollama,
),
(
"civitai",
"CivitAI models: numeric IDs, keyword search, top, latest",
"c/3",
ModelProvider::Civitai,
),
];
let items = providers
.iter()
.enumerate()
.map(|(i, (name, desc, key, provider))| {
let selected = cursor == i;
let marker = if selected { ">>" } else { " " };
let style = if selected {
selected_style(tick)
} else {
Style::default().fg(theme_white())
};
ListItem::with_line(Line::new(vec![
Span::styled(&format!(" {marker} "), style),
Span::styled(
&format!("[{key}]"),
Style::default()
.fg(model_provider_color(*provider))
.add_modifier(Modifier::BOLD),
),
Span::raw(" "),
Span::styled(&name.to_uppercase(), style.add_modifier(Modifier::BOLD)),
Span::raw(" "),
Span::styled(desc, Style::default().fg(theme_dim())),
]))
})
.collect::<Vec<_>>();
let list = List::new(&items)
.with_highlight_style(selected_style(tick))
.with_highlight_symbol(">>");
let mut state = ListState::default();
state.select(Some(cursor));
frame.render_block_stateful_list(
layout[1],
&list,
&mut state,
knott_block(" Provider ", theme_green()),
);
draw_thesa_bottom_bar(
frame,
layout[2],
"Choose where model archives come from",
&[
"Up/Down move",
"h/o/c choose",
"1/2/3 choose",
"Mouse click",
"Enter continue",
"Esc back",
],
);
}
fn draw_target_tui(
frame: &mut Frame<'_>,
input: &str,
status: &str,
tick: usize,
mode: Mode,
provider: Option<ModelProvider>,
) {
let area = frame.area();
paint_surface(frame, area);
let prompt_label = match mode {
Mode::Repos => "repo/owner> ",
Mode::Models => "model target> ",
Mode::Sites => "site url> ",
};
if area.width < 24 || area.height < 8 {
let line = format!("{prompt_label}{input}\n{status}");
frame.render_widget(
Paragraph::from_text(Text::new(vec![Line::styled(
&line,
Style::default().fg(theme_white()),
)])),
area,
);
return;
}
let layout = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(header_height(area)),
Constraint::Min(5),
Constraint::Length(2),
])
.split(area);
let model_provider = provider.unwrap_or(ModelProvider::Hf);
let mode_label = match mode {
Mode::Repos => "github repos",
Mode::Models => match model_provider {
ModelProvider::Hf => "huggingface models",
ModelProvider::Ollama => "ollama models",
ModelProvider::Civitai => "civitai models",
},
Mode::Sites => "siteforge websites",
};
draw_thesa_header(frame, layout[0], tick, status, mode_label);
let examples = match mode {
Mode::Repos => vec![
Span::styled("examples ", Style::default().fg(theme_dim())),
Span::styled("octocat", Style::default().fg(theme_blue())),
Span::raw(" "),
Span::styled("octocat/Spoon-Knife", Style::default().fg(theme_green())),
Span::raw(" "),
Span::styled(
"https://github.com/owner/repo",
Style::default().fg(theme_red()),
),
],
Mode::Models => match model_provider {
ModelProvider::Hf => vec![
Span::styled("examples ", Style::default().fg(theme_dim())),
Span::styled("meta-llama", Style::default().fg(theme_blue())),
Span::raw(" "),
Span::styled("meta-llama/Llama-2-7b", Style::default().fg(theme_green())),
Span::raw(" "),
Span::styled("top", Style::default().fg(theme_red())),
Span::raw(" "),
Span::styled(
"https://huggingface.co/openai-community/gpt2",
Style::default().fg(theme_red()),
),
],
ModelProvider::Ollama => vec![
Span::styled("examples ", Style::default().fg(theme_dim())),
Span::styled("llama3:8b", Style::default().fg(theme_blue())),
Span::raw(" "),
Span::styled("qwen2.5:7b", Style::default().fg(theme_green())),
Span::raw(" "),
Span::styled("latest", Style::default().fg(theme_red())),
Span::raw(" "),
Span::styled("phi3:3.8b", Style::default().fg(theme_red())),
],
ModelProvider::Civitai => vec![
Span::styled("examples ", Style::default().fg(theme_dim())),
Span::styled("827184", Style::default().fg(theme_blue())),
Span::raw(" "),
Span::styled("realistic-vision-v6", Style::default().fg(theme_green())),
Span::raw(" "),
Span::styled("top", Style::default().fg(theme_red())),
Span::raw(" "),
Span::styled(
"https://civitai.com/models/827184",
Style::default().fg(theme_red()),
),
],
},
Mode::Sites => vec![
Span::styled("examples ", Style::default().fg(theme_dim())),
Span::styled("https://example.com", Style::default().fg(theme_blue())),
Span::raw(" "),
Span::styled("https://docs.rs", Style::default().fg(theme_green())),
Span::raw(" "),
Span::styled("http://127.0.0.1:8765/", Style::default().fg(theme_red())),
],
};
let body = Paragraph::from_text(Text::new(vec![
Line::new(vec![
Span::styled(prompt_label, prompt_style()),
Span::styled(input, Style::default().fg(theme_white())),
]),
Line::raw(""),
Line::new(examples),
]))
.wrap(WrapMode::Character { trim: false });
let title = match mode {
Mode::Sites => " Siteforge URL ",
_ => " Target ",
};
let color = match mode {
Mode::Sites => theme_green(),
_ => theme_blue(),
};
frame.render_block_widget(layout[1], body, knott_block(title, color));
draw_thesa_bottom_bar(
frame,
layout[2],
status,
match mode {
Mode::Sites => &["Enter crawl", "Esc back"],
_ => &["Enter accept", "Esc back"],
},
);
let input_area = panel_inner(layout[1]);
let cursor_x = input_area
.x
.saturating_add(prompt_label.len() as u16)
.saturating_add(input.chars().count() as u16)
.min(input_area.x + input_area.width.saturating_sub(1));
frame.set_cursor_position(cursor_x, input_area.y);
}
fn draw_site_options_tui(
frame: &mut Frame<'_>,
url: &str,
options: SiteTuiOptions,
cursor: usize,
status: &str,
tick: usize,
) {
let area = frame.area();
paint_surface(frame, area);
if area.width < 32 || area.height < 10 {
let line = format!(
"siteforge\n{}\ndepth {} pages {} profile {} js {}\n{}",
if options.full_site {
"full-site"
} else {
"single-page"
},
options.max_depth,
options.max_pages,
site_archive_profile_label(options.archive_profile),
site_render_mode_label(options.render_js, options.render_js_auto),
status,
);
frame.render_widget(
Paragraph::from_text(Text::new(vec![Line::styled(
&line,
Style::default().fg(theme_white()),
)])),
area,
);
return;
}
let layout = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(header_height(area)),
Constraint::Min(8),
Constraint::Length(2),
])
.split(area);
draw_thesa_header(frame, layout[0], tick, status, "siteforge options");
let body = Layout::default()
.direction(Direction::Horizontal)
.constraints(if layout[1].width >= 94 {
[Constraint::Percentage(64), Constraint::Percentage(36)]
} else {
[Constraint::Percentage(100), Constraint::Percentage(0)]
})
.split(layout[1]);
let rows: Vec<(&str, &str, String, &str)> = vec![
(
"f",
"crawl",
if options.full_site {
"same-domain full-site".to_string()
} else {
"single page only".to_string()
},
"toggle scope",
),
(
"d",
"depth",
format_site_option_value(options.max_depth),
"cycle 1/2/4/8",
),
(
"p",
"pages",
format_site_option_value(options.max_pages),
"cycle 25/100/500/1000",
),
(
"j",
"js mode",
site_render_mode_label(options.render_js, options.render_js_auto).to_string(),
"off/auto/all",
),
(
"l",
"profile",
site_archive_profile_label(options.archive_profile).to_string(),
"full/agent/lean",
),
];
let items = rows
.iter()
.enumerate()
.map(|(index, (key, label, value, detail))| {
let selected = cursor == index;
let style = if selected {
selected_style(tick)
} else {
Style::default().fg(theme_white())
};
ListItem::with_line(Line::new(vec![
Span::styled(
&format!("[{key}] "),
Style::default()
.fg(theme_green())
.add_modifier(Modifier::BOLD),
),
Span::styled(&format!("{label:<10}"), style.add_modifier(Modifier::BOLD)),
Span::styled(&format!(" {value:<24}"), Style::default().fg(theme_white())),
Span::styled(detail, Style::default().fg(theme_dim())),
]))
})
.collect::<Vec<_>>();
let mut state = ListState::default();
state.select(Some(cursor));
let list = List::new(&items)
.with_highlight_style(selected_style(tick))
.with_highlight_symbol("> ");
frame.render_block_stateful_list(
body[0],
&list,
&mut state,
knott_block(" Siteforge Run Plan ", theme_green()),
);
if body[1].width > 0 {
render_aisling_text_panel(
frame,
body[1],
" siteforge crawl signal ",
EffectKind::Wipe,
tick,
"siteforge\ncrawl\nreadable\nindex",
vec![
section_line("URL"),
Line::styled(
&truncate_to_width(url, body[1].width.saturating_sub(4) as usize),
Style::default().fg(theme_white()),
),
Line::raw(""),
help_pair_line("output", "./archives/sites unless overridden"),
help_pair_line("domain", "same-domain only"),
help_pair_line("readable", "AGENTS.md + markdown/text"),
help_pair_line("verify", ".thesa manifest after crawl"),
],
theme_green(),
);
}
draw_thesa_bottom_bar(
frame,
layout[2],
status,
&[
"Enter run",
"Esc back",
"Space adjust",
"f scope",
"d depth",
"p pages",
"j js",
"l profile",
],
);
}
fn format_site_option_value(value: usize) -> String {
value.to_string()
}
#[allow(clippy::too_many_arguments)]
fn draw_model_picker_tui(
frame: &mut Frame<'_>,
models: &[ModelItem],
filtered: &[usize],
selected: &BTreeSet<usize>,
cursor: usize,
query: &str,
status: &str,
tick: usize,
batch_size: usize,
batch_overlay: bool,
) {
let area = frame.area();
paint_surface(frame, area);
if area.width < 28 || area.height < 8 {
let line = format!(
"models {} visible / {} total\nfilter> {}\n{}",
filtered.len(),
models.len(),
query,
status,
);
frame.render_widget(
Paragraph::from_text(Text::new(vec![Line::styled(
&line,
Style::default().fg(theme_white()),
)])),
area,
);
return;
}
let layout = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(header_height(area)),
Constraint::Min(8),
Constraint::Length(2),
])
.split(area);
draw_thesa_header(frame, layout[0], tick, status, "model picker");
let body = Layout::default()
.direction(Direction::Horizontal)
.constraints(if layout[1].width >= 104 {
[Constraint::Percentage(72), Constraint::Percentage(28)]
} else {
[Constraint::Percentage(100), Constraint::Percentage(0)]
})
.split(layout[1]);
let clone_scope = if selected.is_empty() {
format!("visible: {}", filtered.len())
} else {
format!("checked only: {}", selected.len())
};
let max_name_width = if body[0].width > 56 {
(body[0].width as usize - 52).max(20)
} else {
(body[0].width as usize - 30).max(10)
};
let provider_counts = format!(
"hf:{} ollama:{} civitai:{}",
models
.iter()
.filter(|m| m.provider == ModelProvider::Hf)
.count(),
models
.iter()
.filter(|m| m.provider == ModelProvider::Ollama)
.count(),
models
.iter()
.filter(|m| m.provider == ModelProvider::Civitai)
.count(),
);
let items = filtered
.iter()
.map(|model_index| {
let model = &models[*model_index];
let marker = if selected.contains(model_index) {
"[x]"
} else {
"[ ]"
};
let tag = model.pipeline_tag.as_deref().unwrap_or("-");
let dl = format_downloads(model.downloads);
let size = format_model_size(model.size);
let name = truncate_to_width(&model.id, max_name_width);
let tag = truncate_to_width(tag, 10);
ListItem::with_line(Line::new(vec![
Span::styled(
marker,
Style::default()
.fg(theme_green())
.add_modifier(Modifier::BOLD),
),
Span::raw(" "),
Span::styled(
&format!("[{}]", model.provider.short_label()),
Style::default()
.fg(model_provider_color(model.provider))
.add_modifier(Modifier::BOLD),
),
Span::raw(" "),
Span::styled(
&format!("{name:<max_name_width$}"),
Style::default().fg(theme_white()),
),
Span::styled(&format!(" {dl:>7} "), Style::default().fg(theme_blue())),
Span::styled(&format!(" {size:>9} "), Style::default().fg(theme_dim())),
Span::styled(&format!(" [{tag}]"), Style::default().fg(theme_dim())),
]))
})
.collect::<Vec<_>>();
let mut state = ListState::default();
if !filtered.is_empty() {
state.select(Some(cursor));
}
let list_title = format!(
" Models | filter: {} | {} | batch: {} | {} ",
if query.is_empty() { "all" } else { query },
clone_scope,
batch_size,
provider_counts,
);
let list = List::new(&items)
.with_highlight_style(selected_style(tick))
.with_highlight_symbol("> ");
frame.render_block_stateful_list(
body[0],
&list,
&mut state,
knott_block(&list_title, theme_blue()),
);
if body[1].width > 0 {
let side = Layout::default()
.direction(Direction::Vertical)
.constraints([Constraint::Percentage(65), Constraint::Percentage(35)])
.split(body[1]);
let mut sidebar_lines = vec![
section_line("Controls"),
help_pair_line("type", "filter models"),
help_pair_line("mouse", "hover focus, click check"),
help_pair_line("Space", "check/uncheck model"),
help_pair_line("a", "check visible"),
help_pair_line("b", "batch size overlay"),
help_pair_line("c", "clear"),
help_pair_line("PgUp/PgDown", "jump pages"),
help_pair_line("Home/End", "first/last"),
help_pair_line("Enter", "checked only, or visible"),
Line::raw(""),
section_line("Selection"),
help_pair_line("checked", &selected.len().to_string()),
help_pair_line("filtered", &filtered.len().to_string()),
];
if let Some(model) = filtered.get(cursor).and_then(|i| models.get(*i)) {
sidebar_lines.push(Line::raw(""));
sidebar_lines.push(section_line("Active"));
sidebar_lines.push(help_pair_line("provider", model.provider.label()));
sidebar_lines.push(Line::new(vec![
Span::styled(
"name ",
Style::default()
.fg(theme_blue())
.add_modifier(Modifier::BOLD),
),
Span::styled(
&truncate_to_width(&model.id, (body[1].width as usize).saturating_sub(16)),
Style::default().fg(theme_white()),
),
]));
sidebar_lines.push(help_pair_line(
"tag",
model.pipeline_tag.as_deref().unwrap_or("-"),
));
sidebar_lines.push(help_pair_line(
"downloads",
&format_downloads(model.downloads),
));
sidebar_lines.push(help_pair_line("size", &format_model_size(model.size)));
sidebar_lines.push(help_pair_line(
"updated",
model.last_modified.as_deref().unwrap_or("n/a"),
));
} else {
sidebar_lines.push(Line::raw(""));
sidebar_lines.push(section_line("Active"));
sidebar_lines.push(Line::raw("no model selected"));
}
let sidebar = Paragraph::from_text(Text::new(sidebar_lines))
.wrap(WrapMode::Character { trim: false });
frame.render_block_widget(
side[0],
sidebar,
knott_block(" controls + selection ", theme_dim()),
);
render_aisling_panel(
frame,
side[1],
" scrin aisling burn ",
EffectKind::Burn,
tick,
"models",
);
}
draw_thesa_bottom_bar(
frame,
layout[2],
status,
&[
"Up/Down move",
"PgUp/PgDown jump",
"Home/End jump",
"Mouse hover/click",
"Space check",
"b batch",
"Enter download",
"Esc quit",
],
);
if batch_overlay {
draw_batch_overlay(frame, area, batch_size);
}
}
fn picker_page_step(terminal_height: u16) -> usize {
let header_rows = if terminal_height >= 30 { 5 } else { 3 };
let list_rows = terminal_height.saturating_sub(header_rows + 2);
(list_rows.max(1) as usize).max(5) / 2
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ListMouseAction {
Hover(usize),
Click(usize),
ScrollUp,
ScrollDown,
}
fn mode_selector_list_inner(area: Rect) -> Option<Rect> {
if area.width < 28 || area.height < 10 {
return None;
}
let layout = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(header_height(area)),
Constraint::Min(8),
Constraint::Length(2),
])
.split(area);
let body = Layout::default()
.direction(Direction::Horizontal)
.constraints(if layout[1].width >= 94 {
[Constraint::Percentage(62), Constraint::Percentage(38)]
} else {
[Constraint::Percentage(100), Constraint::Percentage(0)]
})
.split(layout[1]);
Some(panel_inner(body[0]))
}
fn provider_selector_list_inner(area: Rect) -> Option<Rect> {
selector_list_inner(area, 24, 8, 3)
}
fn site_options_list_inner(area: Rect) -> Option<Rect> {
if area.width < 32 || area.height < 10 {
return None;
}
let layout = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(header_height(area)),
Constraint::Min(8),
Constraint::Length(2),
])
.split(area);
let body = Layout::default()
.direction(Direction::Horizontal)
.constraints(if layout[1].width >= 94 {
[Constraint::Percentage(64), Constraint::Percentage(36)]
} else {
[Constraint::Percentage(100), Constraint::Percentage(0)]
})
.split(layout[1]);
Some(panel_inner(body[0]))
}
fn selector_list_inner(
area: Rect,
min_width: u16,
min_height: u16,
footer_height: u16,
) -> Option<Rect> {
if area.width < min_width || area.height < min_height {
return None;
}
let layout = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(header_height(area)),
Constraint::Min(8),
Constraint::Length(footer_height),
])
.split(area);
Some(panel_inner(layout[1]))
}
fn picker_list_inner(area: Rect, sidebar_threshold: u16) -> Option<Rect> {
if area.width < 28 || area.height < 8 {
return None;
}
let layout = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(header_height(area)),
Constraint::Min(8),
Constraint::Length(2),
])
.split(area);
let body = Layout::default()
.direction(Direction::Horizontal)
.constraints(if layout[1].width >= sidebar_threshold {
[Constraint::Percentage(72), Constraint::Percentage(28)]
} else {
[Constraint::Percentage(100), Constraint::Percentage(0)]
})
.split(layout[1]);
Some(panel_inner(body[0]))
}
fn list_mouse_action(
inner: Rect,
item_count: usize,
cursor: usize,
mouse: &MouseEvent,
) -> Option<ListMouseAction> {
match mouse.kind {
MouseEventKind::Moved | MouseEventKind::Drag(MouseButton::Left) => {
visible_list_item_at(inner, mouse.column, mouse.row, item_count, cursor)
.map(ListMouseAction::Hover)
}
MouseEventKind::Down(MouseButton::Left) => {
visible_list_item_at(inner, mouse.column, mouse.row, item_count, cursor)
.map(ListMouseAction::Click)
}
MouseEventKind::ScrollUp if rect_contains_point(inner, mouse.column, mouse.row) => {
Some(ListMouseAction::ScrollUp)
}
MouseEventKind::ScrollDown if rect_contains_point(inner, mouse.column, mouse.row) => {
Some(ListMouseAction::ScrollDown)
}
_ => None,
}
}
fn visible_list_item_at(
inner: Rect,
x: u16,
y: u16,
item_count: usize,
cursor: usize,
) -> Option<usize> {
if item_count == 0 || inner.height == 0 || !rect_contains_point(inner, x, y) {
return None;
}
let visible_rows = inner.height as usize;
let offset = list_render_offset(item_count, cursor, visible_rows);
let row = y.saturating_sub(inner.y) as usize;
let index = offset + row;
(index < item_count).then_some(index)
}
fn list_render_offset(item_count: usize, cursor: usize, visible_rows: usize) -> usize {
if item_count == 0 || visible_rows == 0 {
return 0;
}
let cursor = cursor.min(item_count - 1);
if cursor >= visible_rows {
cursor + 1 - visible_rows
} else {
0
}
}
fn rect_contains_point(area: Rect, x: u16, y: u16) -> bool {
x >= area.x
&& y >= area.y
&& x < area.x.saturating_add(area.width)
&& y < area.y.saturating_add(area.height)
}
fn render_download_dashboard(
frame: &mut Frame,
names: &[String],
state: &[BatchState],
summary: &CloneSummary,
batch_index: usize,
batch_size: usize,
total_models: usize,
total_batches: usize,
tick: usize,
elapsed: Duration,
effect_kind: EffectKind,
ui_state: &UiState,
) {
let area = frame.area();
paint_surface(frame, area);
let overall_done = summary.cloned + summary.skipped + summary.failed.len();
let status = if ui_state.abort_requested {
"stopping after current batch"
} else {
"downloading models"
};
if area.width < 36 || area.height < 10 {
let loader = Loader::with_config(
LoaderKind::Tqdm,
LoaderConfig::default()
.with_size(area.width.saturating_sub(4) as usize, 1)
.with_label("thesa")
.with_unit("model")
.with_fraction(true),
);
let line = loader.line(
tick,
LoaderProgress::from_counts(overall_done as u64, total_models as u64),
);
let panel = Paragraph::from_text(Text::new(vec![
Line::styled(
"thesa",
Style::default()
.fg(theme_white())
.add_modifier(Modifier::BOLD),
),
Line::styled(&line, Style::default().fg(theme_white())),
Line::styled(status, Style::default().fg(theme_dim())),
]));
frame.render_block_widget(area, panel, knott_block(" thesa models ", theme_blue()));
return;
}
let root = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(header_height(area)),
Constraint::Min(8),
Constraint::Length(2),
])
.split(area);
draw_thesa_header(frame, root[0], tick, status, "model download");
let body = Layout::default()
.direction(Direction::Horizontal)
.constraints(if root[1].width >= 104 {
[Constraint::Percentage(68), Constraint::Percentage(32)]
} else {
[Constraint::Percentage(100), Constraint::Percentage(0)]
})
.split(root[1]);
let left = Layout::default()
.direction(Direction::Vertical)
.constraints([Constraint::Length(5), Constraint::Min(5)])
.split(body[0]);
let current_batch = ((batch_index - 1) / batch_size) + 1;
render_command_progress_panel(
frame,
left[0],
" command progress ",
CommandProgress {
verb: "download models",
unit: "model",
done_label: "downloaded",
done: summary.cloned,
skipped: summary.skipped,
failed: summary.failed.len(),
total: total_models,
batch_index: current_batch,
batch_size,
total_batches,
elapsed,
},
tick,
);
let model_loader = Loader::with_config(
LoaderKind::Tqdm,
LoaderConfig::default()
.with_size(left[1].width.saturating_sub(34).max(10) as usize, 1)
.with_label("model")
.with_unit("step")
.with_fraction(true),
);
let mut rows = Vec::new();
if ui_state.show_details {
for (index, (name, model_state)) in names.iter().zip(state.iter()).enumerate() {
rows.push(repo_loader_row(
&model_loader,
tick + index,
name,
*model_state,
));
}
} else {
rows.push(Line::styled(
"details hidden; press t to show concurrent model loaders",
Style::default().fg(theme_dim()),
));
}
let panel = Paragraph::from_text(Text::new(rows)).wrap(WrapMode::Character { trim: false });
frame.render_block_widget(
left[1],
panel,
knott_block(" Concurrent Downloads | Aisling Tqdm ", theme_blue()),
);
if body[1].width > 0 {
let (pending, running, done, skipped, failed) = batch_state_counts(state);
let side_layout = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(8),
Constraint::Length(8),
Constraint::Min(4),
])
.split(body[1]);
render_command_info_panel(
frame,
side_layout[0],
" telemetry ",
vec![
section_line("Batch State"),
help_pair_line("running", &running.to_string()),
help_pair_line("pending", &pending.to_string()),
help_pair_line("done", &done.to_string()),
help_pair_line("skipped", &skipped.to_string()),
help_pair_line("failed", &failed.to_string()),
],
theme_blue(),
);
let side_lines = vec![
section_line("Controls"),
help_pair_line("q / Esc", "stop after current batch"),
help_pair_line("t", "toggle model rows"),
help_pair_line("m", "motion effect"),
help_pair_line("b", "burn effect"),
help_pair_line("w", "wipe effect"),
];
render_command_info_panel(
frame,
side_layout[1],
" command keys ",
side_lines,
theme_dim(),
);
render_aisling_panel(
frame,
side_layout[2],
&format!(" scrin aisling {} ", effect_kind.name()),
effect_kind,
tick,
"models\ndownload",
);
}
draw_thesa_bottom_bar(
frame,
root[2],
status,
&["q stop", "t details", "m motion", "b burn", "w wipe"],
);
}
fn format_downloads(n: u64) -> String {
if n >= 1_000_000 {
format!("{:.1}M", n as f64 / 1_000_000.0)
} else if n >= 1_000 {
format!("{:.1}K", n as f64 / 1_000.0)
} else {
format!("{n}")
}
}
fn format_model_size(size: Option<u64>) -> String {
match size {
None => "n/a".to_string(),
Some(bytes) => {
let gb = 1024f64 * 1024f64 * 1024f64;
let mb = 1024f64 * 1024f64;
let kb = 1024f64;
if bytes as f64 >= gb {
format!("{:.1}GB", bytes as f64 / gb)
} else if bytes as f64 >= mb {
format!("{:.1}MB", bytes as f64 / mb)
} else if bytes as f64 >= kb {
format!("{:.1}KB", bytes as f64 / kb)
} else {
format!("{bytes} B")
}
}
}
}
fn truncate_to_width(value: &str, width: usize) -> String {
if width == 0 {
return String::new();
}
let chars: Vec<char> = value.chars().collect();
if chars.len() <= width {
value.to_string()
} else if width <= 3 {
chars.iter().take(width).copied().collect()
} else {
let mut out: String = chars.iter().take(width - 3).copied().collect();
out.push_str("...");
out
}
}
fn model_provider_color(provider: ModelProvider) -> TuiColor {
match provider {
ModelProvider::Hf => TuiColor::rgb(130, 190, 255),
ModelProvider::Ollama => TuiColor::rgb(130, 255, 130),
ModelProvider::Civitai => TuiColor::rgb(255, 200, 130),
}
}
fn fetch_repositories_for_target(
client: &Client,
target: &Target,
token: Option<&str>,
) -> Result<Vec<RepoItem>> {
match target {
Target::Owner(owner) => match list_repos_for_owner(client, owner, token) {
Ok(items) => Ok(items),
Err(GitArchiverError::NotFound(_)) => list_repos_for_org(client, owner, token),
Err(other) => Err(other),
},
Target::Repo { owner, repo } => Ok(vec![fetch_single_repo(client, owner, repo, token)?]),
}
}
fn fetch_repositories_tui(
terminal: &mut Terminal,
client: &Client,
target: &Target,
token: Option<&str>,
) -> Result<Vec<RepoItem>> {
let client = client.clone();
let target = target.clone();
let token = token.map(str::to_string);
let loading_label = match &target {
Target::Owner(owner) => format!("loading public repos for {owner}"),
Target::Repo { owner, repo } => format!("loading repo {owner}/{repo}"),
};
let (tx, rx) = mpsc::channel();
thread::spawn(move || {
let result = fetch_repositories_for_target(&client, &target, token.as_deref());
let _ = tx.send(result);
});
let mut tick = 0usize;
loop {
if let Ok(result) = rx.try_recv() {
return result;
}
draw_full_frame(terminal, |frame| {
draw_loading_tui(frame, &loading_label, tick, Mode::Repos, None)
})?;
tick = tick.wrapping_add(1);
if event::poll(Duration::from_millis(80))? {
let Event::Key(key) = event::read()? else {
continue;
};
if key.kind == KeyEventKind::Press && matches!(key.code, KeyCode::Esc) {
return Err(GitArchiverError::Message("aborted by user".to_string()));
}
}
}
}
fn fetch_models_tui(
terminal: &mut Terminal,
client: &Client,
target: &ModelTarget,
args: &Cli,
) -> Result<Vec<ModelItem>> {
let client = client.clone();
let target = target.clone();
let args_clone = args.clone();
let loading_label = match &target {
ModelTarget::Org(org) => format!("loading models for {org} from {}", args.provider.label()),
ModelTarget::Model(id) => format!("loading model {id} from {}", args.provider.label()),
ModelTarget::Top => format!("loading top models from {}", args.provider.label()),
ModelTarget::Latest => {
format!("loading latest models from {}", args.provider.label())
}
};
let (tx, rx) = mpsc::channel();
thread::spawn(move || {
let result = fetch_models_for_target(&client, &target, &args_clone);
let _ = tx.send(result);
});
let mut tick = 0usize;
loop {
if let Ok(result) = rx.try_recv() {
return result;
}
draw_full_frame(terminal, |frame| {
draw_loading_tui(
frame,
&loading_label,
tick,
Mode::Models,
Some(args.provider),
)
})?;
tick = tick.wrapping_add(1);
if event::poll(Duration::from_millis(80))? {
let Event::Key(key) = event::read()? else {
continue;
};
if key.kind == KeyEventKind::Press && matches!(key.code, KeyCode::Esc) {
return Err(GitArchiverError::Message("aborted by user".to_string()));
}
}
}
}
fn draw_loading_tui(
frame: &mut Frame<'_>,
label: &str,
tick: usize,
mode: Mode,
provider: Option<ModelProvider>,
) {
let area = frame.area();
paint_surface(frame, area);
if area.width < 30 || area.height < 8 {
let message = format!("thesa\n{label}");
frame.render_widget(
Paragraph::from_text(Text::new(vec![Line::styled(
&message,
Style::default().fg(theme_white()),
)])),
area,
);
return;
}
let layout = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(header_height(area)),
Constraint::Min(5),
Constraint::Length(2),
])
.split(area);
let mode_label = match mode {
Mode::Repos => "github",
Mode::Models => match provider.unwrap_or(ModelProvider::Hf) {
ModelProvider::Hf => "huggingface",
ModelProvider::Ollama => "ollama",
ModelProvider::Civitai => "civitai",
},
Mode::Sites => "siteforge",
};
draw_thesa_header(frame, layout[0], tick, label, mode_label);
let loader = Loader::with_config(
LoaderKind::Tqdm,
LoaderConfig::default()
.with_size(layout[1].width.saturating_sub(4).max(12) as usize, 1)
.with_label(mode_label)
.with_unit(match mode {
Mode::Repos => "repo",
Mode::Models => "model",
Mode::Sites => "site",
})
.with_fraction(false),
);
let loader_line = loader.line(
tick,
LoaderProgress::from_counts((tick % 97) as u64 + 1, 100),
);
let body = Layout::default()
.direction(Direction::Horizontal)
.constraints(if layout[1].width >= 74 {
[Constraint::Percentage(55), Constraint::Percentage(45)]
} else {
[Constraint::Percentage(100), Constraint::Percentage(0)]
})
.split(layout[1]);
let panel = Paragraph::from_text(Text::new(vec![
section_line("Loading"),
Line::styled(label, Style::default().fg(theme_white())),
Line::raw(&loader_line),
]))
.wrap(WrapMode::Character { trim: false });
frame.render_block_widget(
body[0],
panel,
knott_block(" Aisling Loader ", theme_blue()),
);
if body[1].width > 0 {
render_aisling_panel(
frame,
body[1],
" scrin aisling wipe ",
EffectKind::Wipe,
tick,
label,
);
}
draw_thesa_bottom_bar(frame, layout[2], label, &["Esc cancel"]);
}
fn pick_repositories_tui_inner(
terminal: &mut Terminal,
repos: &[RepoItem],
batch_size: &mut usize,
) -> Result<Vec<RepoItem>> {
if repos.is_empty() {
return Ok(vec![]);
}
let mut query = String::new();
let mut selected = BTreeSet::new();
let mut cursor = 0usize;
let mut frame_index = 0usize;
let mut batch_overlay = false;
let mut status =
"Space checks repos; Enter clones checked repos, or visible if none checked".to_string();
loop {
let filtered = filtered_repo_indices(repos, &query);
if cursor >= filtered.len() {
cursor = filtered.len().saturating_sub(1);
}
let page_step = picker_page_step(terminal.size().height);
draw_full_frame(terminal, |frame| {
draw_repo_picker_tui(
frame,
repos,
&filtered,
&selected,
cursor,
&query,
&status,
frame_index,
*batch_size,
batch_overlay,
);
})?;
frame_index = frame_index.wrapping_add(1);
if !event::poll(Duration::from_millis(50))? {
continue;
}
let input_event = event::read()?;
if let Event::Mouse(mouse) = input_event {
if !batch_overlay {
if let Some(inner) = picker_list_inner(terminal.size(), 96) {
match list_mouse_action(inner, filtered.len(), cursor, &mouse) {
Some(ListMouseAction::Hover(index)) => cursor = index,
Some(ListMouseAction::Click(index)) => {
cursor = index;
if let Some(repo_index) = filtered.get(cursor).copied() {
if !selected.insert(repo_index) {
selected.remove(&repo_index);
}
let checked = selected.len();
status = if checked == 0 {
"nothing checked; Enter clones visible repos".to_string()
} else {
format!("{checked} checked; Enter clones checked repos only")
};
}
}
Some(ListMouseAction::ScrollUp) => cursor = cursor.saturating_sub(1),
Some(ListMouseAction::ScrollDown) => {
if cursor + 1 < filtered.len() {
cursor += 1;
}
}
None => {}
}
}
}
continue;
}
let Event::Key(key) = input_event else {
continue;
};
if key.kind != KeyEventKind::Press {
continue;
}
if batch_overlay {
match key.code {
KeyCode::Esc | KeyCode::Char('b') | KeyCode::Enter => {
batch_overlay = false;
status = format!("batch size set to {}; Enter clones repos", *batch_size);
}
KeyCode::Char(ch) => {
if let Some(value) = batch_size_from_key(ch) {
*batch_size = value;
batch_overlay = false;
status = format!("batch size set to {value}; Enter clones repos");
}
}
KeyCode::Left | KeyCode::Up => {
*batch_size = previous_batch_size(*batch_size);
status = format!("batch size set to {}; Enter accepts", *batch_size);
}
KeyCode::Right | KeyCode::Down => {
*batch_size = next_batch_size(*batch_size);
status = format!("batch size set to {}; Enter accepts", *batch_size);
}
_ => {}
}
continue;
}
match key.code {
KeyCode::Esc => return Err(GitArchiverError::Message("aborted by user".to_string())),
KeyCode::Enter => {
let picked = if selected.is_empty() {
filtered
} else {
selected.iter().copied().collect::<Vec<_>>()
};
if picked.is_empty() {
status = "no repositories selected or visible".to_string();
continue;
}
return Ok(picked
.into_iter()
.map(|index| repos[index].clone())
.collect());
}
KeyCode::Up => cursor = cursor.saturating_sub(1),
KeyCode::Down => {
if cursor + 1 < filtered.len() {
cursor += 1;
}
}
KeyCode::PageUp => {
let step = page_step.min(filtered.len().max(1));
cursor = cursor.saturating_sub(step);
status = if filtered.is_empty() {
"no repositories to move to".to_string()
} else {
format!("moved up by {step}")
};
}
KeyCode::PageDown => {
if filtered.is_empty() {
status = "no repositories to move to".to_string();
} else {
let step = page_step.min(filtered.len());
cursor = (cursor + step).min(filtered.len() - 1);
status = format!("moved down by {step}");
}
}
KeyCode::Home => {
if !filtered.is_empty() {
cursor = 0;
status = "jumped to first repo".to_string();
} else {
status = "no repositories to move to".to_string();
}
}
KeyCode::End => {
if !filtered.is_empty() {
cursor = filtered.len() - 1;
status = "jumped to last repo".to_string();
} else {
status = "no repositories to move to".to_string();
}
}
KeyCode::Char(' ') => {
if let Some(repo_index) = filtered.get(cursor).copied() {
if !selected.insert(repo_index) {
selected.remove(&repo_index);
}
let checked = selected.len();
status = if checked == 0 {
"nothing checked; Enter clones visible repos".to_string()
} else {
format!("{checked} checked; Enter clones checked repos only")
};
}
}
KeyCode::Char('a') => {
for repo_index in &filtered {
selected.insert(*repo_index);
}
status = format!("checked {} visible repos", filtered.len());
}
KeyCode::Char('c') => {
selected.clear();
query.clear();
cursor = 0;
status = "cleared filter and selection".to_string();
}
KeyCode::Char('b') => {
batch_overlay = true;
status = "choose batch size: 1 2 3 6 8".to_string();
}
KeyCode::Backspace => {
query.pop();
cursor = 0;
}
KeyCode::Char(ch) => {
query.push(ch);
cursor = 0;
}
_ => {}
}
}
}
fn filtered_repo_indices(repos: &[RepoItem], query: &str) -> Vec<usize> {
if query.trim().is_empty() {
return (0..repos.len()).collect();
}
let query = query.to_lowercase();
repos
.iter()
.enumerate()
.filter(|(_, repo)| {
repo.name.to_lowercase().contains(&query)
|| repo.full_name.to_lowercase().contains(&query)
})
.map(|(index, _)| index)
.collect()
}
#[allow(clippy::too_many_arguments)]
fn draw_repo_picker_tui(
frame: &mut Frame<'_>,
repos: &[RepoItem],
filtered: &[usize],
selected: &BTreeSet<usize>,
cursor: usize,
query: &str,
status: &str,
tick: usize,
batch_size: usize,
batch_overlay: bool,
) {
let area = frame.area();
paint_surface(frame, area);
if area.width < 28 || area.height < 8 {
let line = format!(
"repos {} visible / {} total\nfilter> {}\n{}",
filtered.len(),
repos.len(),
query,
status,
);
frame.render_widget(
Paragraph::from_text(Text::new(vec![Line::styled(
&line,
Style::default().fg(theme_white()),
)])),
area,
);
return;
}
let layout = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(header_height(area)),
Constraint::Min(8),
Constraint::Length(2),
])
.split(area);
draw_thesa_header(frame, layout[0], tick, status, "repo picker");
let body = Layout::default()
.direction(Direction::Horizontal)
.constraints(if layout[1].width >= 96 {
[Constraint::Percentage(72), Constraint::Percentage(28)]
} else {
[Constraint::Percentage(100), Constraint::Percentage(0)]
})
.split(layout[1]);
let clone_scope = if selected.is_empty() {
format!("visible: {}", filtered.len())
} else {
format!("checked only: {}", selected.len())
};
let items = filtered
.iter()
.map(|repo_index| {
let repo = &repos[*repo_index];
let marker = if selected.contains(repo_index) {
"[x]"
} else {
"[ ]"
};
ListItem::with_line(Line::new(vec![
Span::styled(
marker,
Style::default()
.fg(theme_green())
.add_modifier(Modifier::BOLD),
),
Span::raw(" "),
Span::styled(&repo.full_name, Style::default().fg(theme_white())),
]))
})
.collect::<Vec<_>>();
let mut state = ListState::default();
if !filtered.is_empty() {
state.select(Some(cursor));
}
let list_title = format!(
" Repositories | filter: {} | {} | batch: {} ",
if query.is_empty() { "all" } else { query },
clone_scope,
batch_size,
);
let list = List::new(&items)
.with_highlight_style(selected_style(tick))
.with_highlight_symbol("> ");
frame.render_block_stateful_list(
body[0],
&list,
&mut state,
knott_block(&list_title, theme_blue()),
);
if body[1].width > 0 {
let side = Layout::default()
.direction(Direction::Vertical)
.constraints([Constraint::Percentage(62), Constraint::Percentage(38)])
.split(body[1]);
let sidebar = Paragraph::from_text(Text::new(vec![
section_line("Controls"),
help_pair_line("type", "filter public repos"),
help_pair_line("mouse", "hover focus, click check"),
help_pair_line("Space", "check/uncheck repo"),
help_pair_line("a", "check visible"),
help_pair_line("b", "batch size overlay"),
help_pair_line("c", "clear"),
help_pair_line("PgUp/PgDown", "jump pages"),
help_pair_line("Home/End", "first/last"),
help_pair_line("Enter", "checked only, or visible"),
]));
frame.render_block_widget(side[0], sidebar, knott_block(" controls ", theme_dim()));
render_aisling_panel(
frame,
side[1],
" scrin aisling waves ",
EffectKind::Waves,
tick,
"repos",
);
}
draw_thesa_bottom_bar(
frame,
layout[2],
status,
&[
"Up/Down move",
"PgUp/PgDown jump",
"Home/End jump",
"Mouse hover/click",
"Space check",
"b batch",
"Enter clone",
"Esc quit",
],
);
if batch_overlay {
draw_batch_overlay(frame, area, batch_size);
}
}
fn clone_repositories_scrin(
repos: &[RepoItem],
output: &Path,
args: &Cli,
ui_state: &mut UiState,
) -> Result<CloneSummary> {
let mut session = TerminalSession::new()?;
clone_repositories_scrin_inner(&mut session.terminal, repos, output, args, ui_state)
}
fn clone_repositories_scrin_inner(
terminal: &mut Terminal,
repos: &[RepoItem],
output: &Path,
args: &Cli,
ui_state: &mut UiState,
) -> Result<CloneSummary> {
let batch_size = args.concurrency.max(1);
let total = repos.len();
let total_batches = total.div_ceil(batch_size);
let mut summary = CloneSummary::default();
let mut effect_kind = EffectKind::Waves;
let started_at = Instant::now();
let mut tick = 0usize;
let mut batch_index = 1;
while batch_index <= total {
let batch_end = (batch_index + batch_size - 1).min(total);
let batch = &repos[batch_index - 1..batch_end];
let names = batch
.iter()
.map(|repo| repo.full_name.clone())
.collect::<Vec<_>>();
let mut state = vec![BatchState::Pending; batch.len()];
let (tx, rx) = mpsc::channel::<(usize, BatchEvent)>();
let mut handles = Vec::new();
for (offset, repo) in batch.iter().enumerate() {
let repo = repo.clone();
let output = output.to_path_buf();
let args = args.clone();
let tx = tx.clone();
handles.push((
repo.full_name.clone(),
thread::spawn(move || {
let started = tx.send((offset, BatchEvent::Started));
if started.is_err() {
return;
}
let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
clone_repository(&repo, &output, &args)
}));
let event = match outcome {
Ok(Ok(CloneStatus::Cloned)) => BatchEvent::Result(BatchResult::Done),
Ok(Ok(CloneStatus::Skipped)) => BatchEvent::Result(BatchResult::Skipped),
Ok(Err(err)) => BatchEvent::Result(BatchResult::Error(err.to_string())),
Err(_) => BatchEvent::Result(BatchResult::Error(
"worker thread panicked while cloning".to_string(),
)),
};
let _ = tx.send((offset, event));
}),
));
}
drop(tx);
let mut completed = 0;
while completed < batch.len() {
while event::poll(Duration::from_millis(0))? {
if let Event::Key(key) = event::read()? {
match key.code {
KeyCode::Char('q') | KeyCode::Esc => ui_state.abort_requested = true,
KeyCode::Char('t') => ui_state.show_details = !ui_state.show_details,
KeyCode::Char('m') => effect_kind = EffectKind::Waves,
KeyCode::Char('b') => effect_kind = EffectKind::Burn,
KeyCode::Char('w') => effect_kind = EffectKind::Wipe,
_ => {}
}
}
}
while let Ok((offset, event)) = rx.try_recv() {
match event {
BatchEvent::Started => state[offset] = BatchState::Running,
BatchEvent::Result(BatchResult::Done) => {
state[offset] = BatchState::Done;
summary.cloned += 1;
completed += 1;
}
BatchEvent::Result(BatchResult::Skipped) => {
state[offset] = BatchState::Skipped;
summary.skipped += 1;
completed += 1;
}
BatchEvent::Result(BatchResult::Error(message)) => {
state[offset] = BatchState::Failed;
summary
.failed
.push(format!("{}: {}", names[offset], message));
completed += 1;
}
}
}
draw_full_frame(terminal, |frame| {
render_clone_dashboard(
frame,
&names,
&state,
&summary,
batch_index,
batch_size,
total,
total_batches,
tick,
started_at.elapsed(),
effect_kind,
ui_state,
)
})?;
tick = tick.wrapping_add(1);
if completed < batch.len() {
thread::sleep(Duration::from_millis(80));
}
}
for (repo_name, handle) in handles {
if handle.join().is_err() {
summary
.failed
.push(format!("{repo_name}: worker thread panicked"));
}
}
draw_full_frame(terminal, |frame| {
render_clone_dashboard(
frame,
&names,
&state,
&summary,
batch_index,
batch_size,
total,
total_batches,
tick,
started_at.elapsed(),
effect_kind,
ui_state,
)
})?;
if ui_state.abort_requested {
break;
}
batch_index = batch_end + 1;
}
Ok(summary)
}
#[allow(clippy::too_many_arguments)]
fn render_clone_dashboard(
frame: &mut Frame,
names: &[String],
state: &[BatchState],
summary: &CloneSummary,
batch_index: usize,
batch_size: usize,
total_repos: usize,
total_batches: usize,
tick: usize,
elapsed: Duration,
effect_kind: EffectKind,
ui_state: &UiState,
) {
let area = frame.area();
paint_surface(frame, area);
let overall_done = summary.cloned + summary.skipped + summary.failed.len();
let status = if ui_state.abort_requested {
"stopping after current batch"
} else {
"cloning"
};
if area.width < 36 || area.height < 10 {
let loader = Loader::with_config(
LoaderKind::Tqdm,
LoaderConfig::default()
.with_size(area.width.saturating_sub(4) as usize, 1)
.with_label("thesa")
.with_unit("repo")
.with_fraction(true),
);
let line = loader.line(
tick,
LoaderProgress::from_counts(overall_done as u64, total_repos as u64),
);
let panel = Paragraph::from_text(Text::new(vec![
Line::styled(
"thesa",
Style::default()
.fg(theme_white())
.add_modifier(Modifier::BOLD),
),
Line::styled(&line, Style::default().fg(theme_white())),
Line::styled(status, Style::default().fg(theme_dim())),
]));
frame.render_block_widget(area, panel, knott_block(" thesa ", theme_blue()));
return;
}
let root = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(header_height(area)),
Constraint::Min(8),
Constraint::Length(2),
])
.split(area);
draw_thesa_header(frame, root[0], tick, status, "clone dashboard");
let body = Layout::default()
.direction(Direction::Horizontal)
.constraints(if root[1].width >= 104 {
[Constraint::Percentage(68), Constraint::Percentage(32)]
} else {
[Constraint::Percentage(100), Constraint::Percentage(0)]
})
.split(root[1]);
let left = Layout::default()
.direction(Direction::Vertical)
.constraints([Constraint::Length(5), Constraint::Min(5)])
.split(body[0]);
let current_batch = ((batch_index - 1) / batch_size) + 1;
render_command_progress_panel(
frame,
left[0],
" command progress ",
CommandProgress {
verb: "clone repositories",
unit: "repo",
done_label: "cloned",
done: summary.cloned,
skipped: summary.skipped,
failed: summary.failed.len(),
total: total_repos,
batch_index: current_batch,
batch_size,
total_batches,
elapsed,
},
tick,
);
let repo_loader = Loader::with_config(
LoaderKind::Tqdm,
LoaderConfig::default()
.with_size(left[1].width.saturating_sub(34).max(10) as usize, 1)
.with_label("repo")
.with_unit("step")
.with_fraction(true),
);
let mut rows = Vec::new();
if ui_state.show_details {
for (index, (name, repo_state)) in names.iter().zip(state.iter()).enumerate() {
rows.push(repo_loader_row(
&repo_loader,
tick + index,
name,
*repo_state,
));
}
} else {
rows.push(Line::styled(
"details hidden; press t to show concurrent repo loaders",
Style::default().fg(theme_dim()),
));
}
let repos = Paragraph::from_text(Text::new(rows)).wrap(WrapMode::Character { trim: false });
frame.render_block_widget(
left[1],
repos,
knott_block(" Concurrent Pulls | Aisling Tqdm ", theme_blue()),
);
if body[1].width > 0 {
let (pending, running, done, skipped, failed) = batch_state_counts(state);
let side_layout = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(8),
Constraint::Length(8),
Constraint::Min(4),
])
.split(body[1]);
render_command_info_panel(
frame,
side_layout[0],
" telemetry ",
vec![
section_line("Batch State"),
help_pair_line("running", &running.to_string()),
help_pair_line("pending", &pending.to_string()),
help_pair_line("done", &done.to_string()),
help_pair_line("skipped", &skipped.to_string()),
help_pair_line("failed", &failed.to_string()),
],
theme_blue(),
);
let side_lines = vec![
section_line("Controls"),
help_pair_line("q / Esc", "stop after current batch"),
help_pair_line("t", "toggle repo rows"),
help_pair_line("m", "motion effect"),
help_pair_line("b", "burn effect"),
help_pair_line("w", "wipe effect"),
];
render_command_info_panel(
frame,
side_layout[1],
" command keys ",
side_lines,
theme_dim(),
);
render_aisling_panel(
frame,
side_layout[2],
&format!(" scrin aisling {} ", effect_kind.name()),
effect_kind,
tick,
"repos\nclone",
);
}
draw_thesa_bottom_bar(
frame,
root[2],
status,
&["q stop", "t details", "m motion", "b burn", "w wipe"],
);
}
fn batch_size_from_key(ch: char) -> Option<usize> {
match ch {
'1' => Some(1),
'2' => Some(2),
'3' => Some(3),
'6' => Some(6),
'8' => Some(8),
_ => None,
}
}
fn batch_state_counts(state: &[BatchState]) -> (usize, usize, usize, usize, usize) {
let mut pending = 0;
let mut running = 0;
let mut done = 0;
let mut skipped = 0;
let mut failed = 0;
for item in state {
match item {
BatchState::Pending => pending += 1,
BatchState::Running => running += 1,
BatchState::Done => done += 1,
BatchState::Skipped => skipped += 1,
BatchState::Failed => failed += 1,
}
}
(pending, running, done, skipped, failed)
}
fn previous_batch_size(current: usize) -> usize {
let index = BATCH_SIZE_CHOICES
.iter()
.position(|value| *value == current)
.unwrap_or(0);
BATCH_SIZE_CHOICES[index.saturating_sub(1)]
}
fn next_batch_size(current: usize) -> usize {
let index = BATCH_SIZE_CHOICES
.iter()
.position(|value| *value == current)
.unwrap_or(0);
BATCH_SIZE_CHOICES[(index + 1).min(BATCH_SIZE_CHOICES.len() - 1)]
}
fn draw_batch_overlay(frame: &mut Frame<'_>, area: Rect, batch_size: usize) {
let width = area.width.min(58).max(30.min(area.width));
let height = area.height.min(11).max(7.min(area.height));
let x = area.x + area.width.saturating_sub(width) / 2;
let y = area.y + area.height.saturating_sub(height) / 2;
let overlay = Rect::new(x, y, width, height);
frame.render_widget(Clear::new(), overlay);
let mut choice_spans = Vec::new();
for value in BATCH_SIZE_CHOICES {
if !choice_spans.is_empty() {
choice_spans.push(Span::raw(" "));
}
let style = if value == batch_size {
Style::default()
.bg(TuiColor::WHITE)
.fg(TuiColor::BLACK)
.add_modifier(Modifier::BOLD)
} else {
Style::default()
.fg(theme_blue())
.add_modifier(Modifier::BOLD)
};
choice_spans.push(Span::styled(&format!(" {value} "), style));
}
let panel = Paragraph::from_text(Text::new(vec![
section_line("Batch Size"),
Line::styled(
"How many items to process at once",
Style::default().fg(theme_dim()),
),
Line::raw(""),
Line::new(choice_spans),
Line::raw(""),
help_pair_line("1/2/3/6/8", "select and close"),
help_pair_line("Left/Right", "cycle choices"),
help_pair_line("b / Enter", "close overlay"),
]))
.wrap(WrapMode::Character { trim: false });
frame.render_block_widget(overlay, panel, knott_block(" Batch Options ", theme_red()));
}
fn draw_thesa_header(frame: &mut Frame<'_>, area: Rect, tick: usize, status: &str, mode: &str) {
if area.height == 0 {
return;
}
if area.height >= 5 && area.width >= 88 {
let deck = Layout::default()
.direction(Direction::Horizontal)
.constraints([Constraint::Percentage(62), Constraint::Percentage(38)])
.split(area);
draw_thesa_header_text(frame, deck[0], tick, status, mode);
render_aisling_panel(
frame,
deck[1],
" scrin aisling matrix ",
EffectKind::Matrix,
tick,
"thesa",
);
return;
}
draw_thesa_header_text(frame, area, tick, status, mode);
}
fn draw_thesa_header_text(
frame: &mut Frame<'_>,
area: Rect,
tick: usize,
status: &str,
mode: &str,
) {
let accent = animated_accent(tick);
let mut primary = Vec::new();
primary.push(Span::styled(
" thesa ",
Style::default()
.bg(accent)
.fg(TuiColor::BLACK)
.add_modifier(Modifier::BOLD),
));
append_status_chip(&mut primary, "mode", mode, theme_blue());
append_status_chip(&mut primary, "state", status, theme_green());
append_status_chip(&mut primary, "effect", "aisling", theme_red());
append_status_chip(&mut primary, "frame", tick.to_string(), theme_dim());
let mut lines = vec![Line::new(primary)];
if area.height >= 5 {
lines.push(Line::raw(""));
lines.push(Line::new(vec![
Span::styled("archive ", Style::default().fg(theme_dim())),
Span::styled("public repos", Style::default().fg(theme_blue())),
Span::styled(" + ", Style::default().fg(theme_dim())),
Span::styled("models", Style::default().fg(theme_green())),
Span::styled(" with live scrin panes", Style::default().fg(theme_dim())),
]));
}
let header = Paragraph::from_text(Text::new(lines)).alignment(Alignment::Left);
frame.render_block_widget(area, header, knott_block(" thesa command deck ", accent));
}
fn draw_thesa_bottom_bar(frame: &mut Frame<'_>, area: Rect, status: &str, keys: &[&str]) {
if area.height == 0 {
return;
}
frame
.buffer()
.fill(area, ' ', theme_white(), Some(theme_surface()));
let mut status_spans = vec![
Span::styled("status: ", Style::default().fg(theme_dim())),
Span::styled(status, Style::default().fg(theme_white())),
];
append_status_chip(&mut status_spans, "ui", "scrin", theme_blue());
append_status_chip(&mut status_spans, "fx", "aisling", theme_green());
append_status_chip(&mut status_spans, "hover", "active", theme_dim());
let key_line = Line::new(
keys.iter()
.flat_map(|key| {
let (head, detail) = key.split_once(' ').unwrap_or((key, ""));
let detail_spans = if detail.is_empty() {
vec![Span::raw(" ")]
} else {
vec![Span::raw(&format!(" {detail} "))]
};
let mut spans = Vec::new();
spans.push(Span::styled(head, key_style()));
spans.extend(detail_spans);
spans
})
.collect::<Vec<_>>(),
);
let lines = if area.height <= 1 {
vec![Line::new(status_spans)]
} else {
vec![Line::new(status_spans), key_line]
};
frame.render_widget(
Paragraph::from_text(Text::new(lines)).with_style(Style::default().fg(theme_white())),
area,
);
normalize_bg(frame.buffer(), area, theme_surface());
}
fn knott_block<'a>(title: &'a str, color: TuiColor) -> Block<'a> {
Block::new(title)
.with_borders(BorderStyle::Rounded)
.with_border_color(color)
.with_bg(theme_surface())
}
fn panel_inner(area: Rect) -> Rect {
knott_block("", theme_dim()).inner(area)
}
fn theme_white() -> TuiColor {
TuiColor::rgb(201, 209, 217)
}
fn theme_red() -> TuiColor {
TuiColor::rgb(248, 81, 73)
}
fn theme_blue() -> TuiColor {
TuiColor::rgb(88, 166, 255)
}
fn theme_green() -> TuiColor {
TuiColor::rgb(63, 185, 80)
}
fn theme_dim() -> TuiColor {
TuiColor::rgb(110, 118, 129)
}
fn theme_bg() -> TuiColor {
TuiColor::rgb(13, 17, 23)
}
fn theme_surface() -> TuiColor {
TuiColor::rgb(22, 27, 34)
}
fn prompt_style() -> Style {
Style::default()
.fg(theme_blue())
.add_modifier(Modifier::BOLD)
}
fn key_style() -> Style {
Style::default()
.bg(theme_blue())
.fg(TuiColor::BLACK)
.add_modifier(Modifier::BOLD)
}
fn selected_style(tick: usize) -> Style {
Style::default()
.bg(animated_accent(tick))
.fg(TuiColor::BLACK)
.add_modifier(Modifier::BOLD)
}
fn header_height(area: Rect) -> u16 {
if area.height >= 30 { 5 } else { 3 }
}
fn animated_accent(tick: usize) -> TuiColor {
let _ = tick;
theme_blue()
}
fn section_line(label: &str) -> Line {
Line::new(vec![Span::styled(
label,
Style::default()
.fg(theme_white())
.add_modifier(Modifier::BOLD),
)])
}
fn help_pair_line(head: &str, detail: &str) -> Line {
Line::new(vec![
Span::styled(
&format!("{head:<12}"),
Style::default()
.fg(theme_blue())
.add_modifier(Modifier::BOLD),
),
Span::styled(detail, Style::default().fg(theme_dim())),
])
}
fn append_status_chip(
spans: &mut Vec<Span>,
label: &str,
value: impl Into<String>,
color: TuiColor,
) {
if !spans.is_empty() {
spans.push(Span::raw(" "));
}
spans.push(Span::styled("[", Style::default().fg(theme_dim())));
spans.push(Span::styled(label, Style::default().fg(theme_dim())));
spans.push(Span::raw(" "));
let value = value.into();
spans.push(Span::styled(
&value,
Style::default().fg(color).add_modifier(Modifier::BOLD),
));
spans.push(Span::styled("]", Style::default().fg(theme_dim())));
}
fn paint_surface(frame: &mut Frame<'_>, area: Rect) {
if area.width == 0 || area.height == 0 {
return;
}
frame
.buffer()
.fill(area, ' ', theme_white(), Some(theme_bg()));
}
fn normalize_bg(buffer: &mut Buffer, area: Rect, bg: TuiColor) {
for y in area.y as usize..area.bottom() as usize {
for x in area.x as usize..area.right() as usize {
let Some(cell) = buffer.get(x, y).copied() else {
continue;
};
if cell.bg.is_none() {
buffer.set(
x,
y,
scrin::core::buffer::Cell {
bg: Some(bg),
..cell
},
);
}
}
}
}
fn render_aisling_panel(
frame: &mut Frame<'_>,
area: Rect,
title: &str,
kind: EffectKind,
tick: usize,
text: &str,
) {
if area.width < 4 || area.height < 4 {
return;
}
let block = knott_block(title, animated_accent(tick)).with_borders(BorderStyle::Thick);
block.render(frame.buffer(), area);
let inner = block.inner(area);
if inner.width == 0 || inner.height == 0 {
return;
}
frame
.buffer()
.fill(inner, ' ', theme_white(), Some(theme_surface()));
render_aisling_effect(frame, inner, kind, tick, text);
normalize_bg(frame.buffer(), inner, theme_surface());
}
fn render_aisling_text_panel(
frame: &mut Frame<'_>,
area: Rect,
title: &str,
kind: EffectKind,
tick: usize,
effect_text: &str,
lines: Vec<Line>,
color: TuiColor,
) {
if area.width < 4 || area.height < 4 {
return;
}
let block = knott_block(title, color).with_borders(BorderStyle::Thick);
block.render(frame.buffer(), area);
let inner = block.inner(area);
if inner.width == 0 || inner.height == 0 {
return;
}
frame
.buffer()
.fill(inner, ' ', theme_white(), Some(theme_surface()));
render_aisling_effect(frame, inner, kind, tick, effect_text);
let panel = Paragraph::from_text(Text::new(lines)).wrap(WrapMode::Character { trim: false });
panel.render(frame.buffer(), inner);
normalize_bg(frame.buffer(), inner, theme_surface());
}
fn render_aisling_effect(
frame: &mut Frame<'_>,
area: Rect,
kind: EffectKind,
tick: usize,
text: &str,
) {
if area.width == 0 || area.height == 0 {
return;
}
let mut player = EffectPlayer::new(kind, text)
.with_size(area.width as usize, area.height as usize)
.with_duration(96)
.with_seed(17)
.with_gradient_colors(
vec![
theme_blue(),
theme_green(),
theme_red(),
TuiColor::rgb(163, 113, 247),
],
45.0,
)
.with_accent(theme_blue());
player.set_frame(tick % player.total_frames().max(1));
player.render_to_buffer(frame.buffer(), area);
}
struct CommandProgress<'a> {
verb: &'a str,
unit: &'a str,
done_label: &'a str,
done: usize,
skipped: usize,
failed: usize,
total: usize,
batch_index: usize,
batch_size: usize,
total_batches: usize,
elapsed: Duration,
}
fn render_command_progress_panel(
frame: &mut Frame<'_>,
area: Rect,
title: &str,
data: CommandProgress<'_>,
tick: usize,
) {
if area.width < 4 || area.height < 4 {
return;
}
let block = knott_block(title, theme_green()).with_borders(BorderStyle::Thick);
block.render(frame.buffer(), area);
let inner = block.inner(area);
if inner.width == 0 || inner.height == 0 {
return;
}
frame
.buffer()
.fill(inner, ' ', theme_white(), Some(theme_surface()));
let completed = data.done + data.skipped + data.failed;
let progress_area = Rect::new(
inner.x,
inner.bottom().saturating_sub(1),
inner.width,
1.min(inner.height),
);
let text_height = inner.height.saturating_sub(progress_area.height);
let text_area = Rect::new(inner.x, inner.y, inner.width, text_height);
let mut chips = Vec::new();
append_status_chip(
&mut chips,
data.done_label,
data.done.to_string(),
theme_green(),
);
append_status_chip(
&mut chips,
"skipped",
data.skipped.to_string(),
TuiColor::YELLOW,
);
append_status_chip(&mut chips, "failed", data.failed.to_string(), theme_red());
append_status_chip(
&mut chips,
"elapsed",
format!("{}s", data.elapsed.as_secs()),
theme_dim(),
);
let text = Paragraph::from_text(Text::new(vec![
Line::new(chips),
Line::new(vec![
Span::styled("command ", Style::default().fg(theme_dim())),
Span::styled(data.verb, Style::default().fg(theme_white())),
]),
Line::styled(
&format!(
"batch {}/{} concurrency {} total {} {}",
data.batch_index, data.total_batches, data.batch_size, data.total, data.unit
),
Style::default().fg(theme_dim()),
),
]));
text.render(frame.buffer(), text_area);
normalize_bg(frame.buffer(), text_area, theme_surface());
let loader = LoaderPlayer::new(LoaderKind::Bar)
.with_size(progress_area.width as usize, 1)
.with_label("thesa".to_string())
.with_unit(data.unit)
.with_fraction(true)
.with_accent(theme_blue())
.with_gradient_colors(vec![theme_blue(), theme_green()], 0.0);
loader.render(
tick,
LoaderProgress::from_counts(completed as u64, data.total.max(1) as u64),
frame.buffer(),
progress_area,
);
normalize_bg(frame.buffer(), progress_area, theme_surface());
}
fn render_command_info_panel(
frame: &mut Frame<'_>,
area: Rect,
title: &str,
lines: Vec<Line>,
color: TuiColor,
) {
let panel = Paragraph::from_text(Text::new(lines)).wrap(WrapMode::Character { trim: false });
frame.render_block_widget(area, panel, knott_block(title, color));
}
fn repo_loader_row(loader: &Loader, tick: usize, name: &str, state: BatchState) -> Line {
let done = matches!(
state,
BatchState::Done | BatchState::Skipped | BatchState::Failed
);
let loader_line = loader.line(tick, LoaderProgress::from_counts(done as u64, 1));
let style = match state {
BatchState::Pending => Style::default().fg(TuiColor::DARK_GRAY),
BatchState::Running => Style::default().fg(TuiColor::MAGENTA),
BatchState::Done => Style::default().fg(TuiColor::CYAN),
BatchState::Skipped => Style::default().fg(TuiColor::YELLOW),
BatchState::Failed => Style::default().fg(TuiColor::RED),
};
let state_label = format!("{:<7}", state.label());
Line::new(vec![
Span::styled(&state_label, style.add_modifier(Modifier::BOLD)),
Span::raw(" "),
Span::raw(&loader_line),
Span::raw(" "),
Span::styled(name, style),
])
}
fn clone_repositories_serial(
repos: &[RepoItem],
output: &Path,
args: &Cli,
ui_rx: &mpsc::Receiver<char>,
ui_state: &mut UiState,
) -> Result<CloneSummary> {
let spinner = Loader::with_config(
LoaderKind::Tqdm,
LoaderConfig::default()
.with_size(54, 1)
.with_label("thesa")
.with_unit("repo")
.with_fraction(true),
);
let mut stdout = io::stdout();
let mut summary = CloneSummary::default();
let total = repos.len();
for (index, repo) in repos.iter().enumerate() {
if ui_state.abort_requested {
break;
}
apply_ui_events(ui_rx, ui_state);
match clone_repository_with_progress(
repo,
output,
args,
&mut stdout,
&spinner,
index + 1,
total,
) {
Ok(CloneStatus::Cloned) => summary.cloned += 1,
Ok(CloneStatus::Skipped) => summary.skipped += 1,
Err(err) => summary.failed.push(format!("{}: {}", repo.full_name, err)),
}
}
Ok(summary)
}
fn clone_repositories_batched(
repos: &[RepoItem],
output: &Path,
args: &Cli,
ui_rx: &mpsc::Receiver<char>,
ui_state: &mut UiState,
) -> Result<CloneSummary> {
let batch_size = args.concurrency.max(1);
let total = repos.len();
let mut summary = CloneSummary::default();
let mut batch_index = 1;
let total_batches = total.div_ceil(batch_size);
let mut stop_after_batch = false;
while batch_index <= total {
let batch_end = (batch_index + batch_size - 1).min(total);
let batch = &repos[batch_index - 1..batch_end];
let batch_positions: Vec<String> =
batch.iter().map(|repo| repo.full_name.clone()).collect();
let (tx, rx) = mpsc::channel::<(usize, BatchEvent)>();
println!(
"Starting batch {}/{} ({})...",
(batch_index - 1) / batch_size + 1,
total_batches,
batch.len()
);
let mut state = vec![BatchState::Pending; batch.len()];
apply_ui_events(ui_rx, ui_state);
if ui_state.abort_requested {
break;
}
let mut handles = Vec::new();
for (offset, repo) in batch.iter().enumerate() {
let repo = repo.clone();
let output = output.to_path_buf();
let args = args.clone();
let position = batch_index + offset;
let tx = tx.clone();
handles.push((
position,
repo.full_name.clone(),
thread::spawn(move || {
let started = tx.send((offset, BatchEvent::Started));
if started.is_err() {
return;
}
let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
clone_repository(&repo, &output, &args)
}));
let event = match outcome {
Ok(Ok(CloneStatus::Cloned)) => BatchEvent::Result(BatchResult::Done),
Ok(Ok(CloneStatus::Skipped)) => BatchEvent::Result(BatchResult::Skipped),
Ok(Err(err)) => BatchEvent::Result(BatchResult::Error(err.to_string())),
Err(_) => BatchEvent::Result(BatchResult::Error(
"worker thread panicked while cloning".to_string(),
)),
};
let _ = tx.send((offset, event));
}),
));
}
drop(tx);
let mut completed = 0;
while completed < batch.len() {
let mut progress_updated = false;
apply_ui_events(ui_rx, ui_state);
if ui_state.abort_requested {
stop_after_batch = true;
}
while let Ok((offset, event)) = rx.try_recv() {
match event {
BatchEvent::Started => {
state[offset] = BatchState::Running;
progress_updated = true;
}
BatchEvent::Result(BatchResult::Done) => {
state[offset] = BatchState::Done;
summary.cloned += 1;
completed += 1;
progress_updated = true;
}
BatchEvent::Result(BatchResult::Skipped) => {
state[offset] = BatchState::Skipped;
summary.skipped += 1;
completed += 1;
progress_updated = true;
}
BatchEvent::Result(BatchResult::Error(message)) => {
state[offset] = BatchState::Failed;
summary
.failed
.push(format!("{}: {}", batch_positions[offset], message));
completed += 1;
progress_updated = true;
}
}
}
if progress_updated {
render_batch_state(
&mut io::stdout(),
(batch_index - 1) / batch_size + 1,
total_batches,
batch_positions.as_slice(),
state.as_slice(),
ui_state.show_details,
&mut ui_state.panel_lines,
batch.len(),
total,
)?;
}
if completed < batch.len() {
thread::sleep(Duration::from_millis(100));
}
}
for (position, repo_name, handle) in handles {
if handle.join().is_err() {
summary.failed.push(format!(
"{repo_name} ({position}/{total}): worker thread panicked"
));
}
}
if stop_after_batch {
break;
}
batch_index = batch_end + 1;
}
Ok(summary)
}
fn apply_ui_events(ui_rx: &mpsc::Receiver<char>, ui_state: &mut UiState) {
while let Ok(command) = ui_rx.try_recv() {
match command {
't' | 'T' => {
ui_state.show_details = !ui_state.show_details;
}
'q' | 'Q' => {
ui_state.abort_requested = true;
}
_ => {}
}
}
}
fn render_batch_state(
stdout: &mut io::Stdout,
batch_number: usize,
total_batches: usize,
names: &[String],
state: &[BatchState],
show_details: bool,
panel_lines: &mut usize,
batch_size: usize,
total_repos: usize,
) -> io::Result<()> {
if !stdout.is_terminal() {
return Ok(());
}
let mut done = 0;
let mut skipped = 0;
let mut failed = 0;
let mut running = 0;
let mut pending = 0;
for entry in state {
match entry {
BatchState::Done => done += 1,
BatchState::Skipped => skipped += 1,
BatchState::Failed => failed += 1,
BatchState::Running => running += 1,
BatchState::Pending => pending += 1,
}
}
let completed = done + skipped + failed;
let loader = Loader::with_config(
LoaderKind::Pulse,
LoaderConfig::default()
.with_size(58, 1)
.with_label("thesa")
.with_unit("repo")
.with_fraction(true)
.with_gradient(thesa_gradient(), GradientDirection::Horizontal),
);
let loader_line = loader
.frame(
completed + running,
LoaderProgress::from_counts(completed as u64, batch_size as u64),
)
.to_ansi_string();
let cyan = AislingColor::rgb(0, 255, 210);
let violet = AislingColor::rgb(142, 92, 255);
let amber = AislingColor::rgb(255, 196, 92);
let rose = AislingColor::rgb(255, 80, 210);
let slate = AislingColor::rgb(132, 150, 168);
let mut lines = vec![
paint(
"+--------------------------------------------------------------------+",
violet,
),
format!(
"| {} {} | {} |",
paint_bold("THESA", cyan),
paint("public-repo collector", slate),
paint(
format!("batch {batch_number}/{total_batches} total {total_repos}"),
amber,
)
),
paint(
"+--------------------------------------------------------------------+",
violet,
),
format!("| {} |", loader_line.trim_end()),
format!(
"| {} {} {} {} {} {} {} {} {} {} |",
paint("done", cyan),
done,
paint("skip", amber),
skipped,
paint("fail", rose),
failed,
paint("run", violet),
running,
paint("wait", slate),
pending,
),
format!(
"| {} |",
paint(
"keys: t + Enter toggles detail, q + Enter stops after batch",
slate
)
),
paint(
"+--------------------------------------------------------------------+",
violet,
),
];
if show_details {
for (name, repo_state) in names.iter().zip(state.iter()) {
lines.push(format!(" {} {}", repo_state.icon(), name));
}
} else {
lines.push(paint(
" detail hidden; press t + Enter to show repos",
slate,
));
}
lines.push(String::new());
if *panel_lines > 0 {
write!(stdout, "\x1b[{}A", *panel_lines)?;
}
let written_lines = (*panel_lines).max(lines.len());
for index in 0..written_lines {
let line = lines.get(index).map(String::as_str).unwrap_or("");
write!(stdout, "\x1b[2K\r{line}\n")?;
}
*panel_lines = written_lines;
stdout.flush()?;
Ok(())
}
#[derive(Debug, Clone)]
enum BatchResult {
Done,
Skipped,
Error(String),
}
#[derive(Debug, Clone)]
enum BatchEvent {
Started,
Result(BatchResult),
}
impl BatchState {
fn label(self) -> &'static str {
match self {
BatchState::Pending => "queued",
BatchState::Running => "pulling",
BatchState::Done => "done",
BatchState::Skipped => "skip",
BatchState::Failed => "failed",
}
}
fn icon(self) -> &'static str {
match self {
BatchState::Pending => " [ ]",
BatchState::Running => "[-x]",
BatchState::Done => " [*]",
BatchState::Skipped => " [/]",
BatchState::Failed => " [!]",
}
}
}
fn parse_target(input: &str) -> Result<Target> {
let cleaned = input.trim().trim_end_matches(".git");
if cleaned.is_empty() {
return Err(GitArchiverError::InvalidTarget(input.to_string()));
}
if let Some(path) = cleaned.strip_prefix("https://github.com/") {
return parse_owner_or_repo_path(path);
}
if let Some(path) = cleaned.strip_prefix("http://github.com/") {
return parse_owner_or_repo_path(path);
}
if let Some(path) = cleaned.strip_prefix("git@github.com:") {
return parse_owner_or_repo_path(path);
}
parse_owner_or_repo_path(cleaned)
}
fn parse_owner_or_repo_path(input: &str) -> Result<Target> {
let parts: Vec<&str> = input
.trim_matches('/')
.split('/')
.filter(|part| !part.is_empty())
.collect();
if parts.is_empty() {
return Err(GitArchiverError::InvalidTarget(input.to_string()));
}
if parts.len() == 1 {
return Ok(Target::Owner(parts[0].to_string()));
}
Ok(Target::Repo {
owner: parts[0].to_string(),
repo: parts[1].to_string(),
})
}
fn default_output_path(target: &Target) -> PathBuf {
let slug = match target {
Target::Owner(owner) => owner.clone(),
Target::Repo { repo, .. } => repo.clone(),
};
PathBuf::from("./archives").join(slug)
}
fn prompt_target(mode: Mode, provider: Option<ModelProvider>) -> Result<String> {
loop {
let value = match mode {
Mode::Repos => prompt_line("GitHub target [owner or owner/repo]: ")?,
Mode::Models => match provider.unwrap_or(ModelProvider::Hf) {
ModelProvider::Hf => prompt_line("Model target [org, org/model, top, latest]: ")?,
ModelProvider::Ollama => {
prompt_line("Model target [name, keyword, top, latest]: ")?
}
ModelProvider::Civitai => prompt_line("Model target [id, keyword, top, latest]: ")?,
},
Mode::Sites => prompt_line("Siteforge URL [https://example.com]: ")?,
};
if value.is_empty() {
println!("Target is required.");
continue;
}
let ok = match mode {
Mode::Repos => parse_target(&value).is_ok(),
Mode::Models => {
parse_model_target_for_provider(&value, provider.unwrap_or(ModelProvider::Hf))
.is_ok()
}
Mode::Sites => parse_site_url(&value).is_ok(),
};
if ok {
return Ok(value);
}
match mode {
Mode::Repos => println!(
"Invalid target. Examples: torvalds, torvalds/linux, https://github.com/torvalds/linux"
),
Mode::Models => match provider.unwrap_or(ModelProvider::Hf) {
ModelProvider::Hf => println!(
"Invalid target. Examples: meta-llama, meta-llama/Llama-2-7b, top, latest"
),
ModelProvider::Ollama => {
println!("Invalid target. Examples: llama3, llama3:8b, latest")
}
ModelProvider::Civitai => {
println!("Invalid target. Examples: 827184, realistic-vision-v6, top, latest")
}
},
Mode::Sites => {
println!("Invalid URL. Examples: https://example.com, http://127.0.0.1:8765/")
}
}
}
}
fn parse_site_url(input: &str) -> Result<reqwest::Url> {
let seed = input.trim();
let parsed = reqwest::Url::parse(seed)
.map_err(|err| GitArchiverError::Message(format!("invalid site URL '{seed}': {err}")))?;
if !matches!(parsed.scheme(), "http" | "https") {
return Err(GitArchiverError::Message(format!(
"unsupported site URL scheme '{}': expected http or https",
parsed.scheme()
)));
}
Ok(parsed)
}
fn prompt_line(prompt: &str) -> Result<String> {
print!("{prompt}");
io::stdout().flush()?;
let mut value = String::new();
io::stdin()
.read_line(&mut value)
.map_err(GitArchiverError::Io)?;
Ok(value.trim().to_string())
}
fn prompt_mode() -> Result<Mode> {
println!("Archive mode:");
println!(" 1) repos - GitHub repositories");
println!(" 2) models - Hugging Face, Ollama, or CivitAI models");
println!(" 3) sites - Siteforge website archives");
let value = prompt_line("Select [1/2/3 or r/m/s, default: repos]: ")?;
match value.as_str() {
"2" | "m" | "models" => Ok(Mode::Models),
"3" | "s" | "site" | "sites" | "siteforge" | "url" => Ok(Mode::Sites),
_ => Ok(Mode::Repos),
}
}
fn prompt_model_provider(current: ModelProvider) -> Result<ModelProvider> {
println!("Model provider:");
println!(" 1) hf - Hugging Face models");
println!(" 2) ollama - local Ollama models");
println!(" 3) civitai - CivitAI models");
let value = prompt_line(&format!("Select [1/2/3 or h/o/c, default: {current}]: "))?;
match value.as_str() {
"2" | "o" | "ollama" => Ok(ModelProvider::Ollama),
"3" | "c" | "civitai" => Ok(ModelProvider::Civitai),
"1" | "h" | "hf" | "huggingface" | "hugging-face" => Ok(ModelProvider::Hf),
"" => Ok(current),
_ => {
println!("Invalid provider; keeping {current}.");
Ok(current)
}
}
}
fn prompt_hf_token() -> Result<Option<String>> {
println!("Hugging Face API requires a token for model access.");
println!("Get one at https://huggingface.co/settings/tokens");
let value = prompt_line("HF token (blank to skip, will rate-limit): ")?;
if value.is_empty() {
Ok(None)
} else {
Ok(Some(value))
}
}
fn prompt_hf_token_tui(terminal: &mut Terminal) -> Result<Option<String>> {
let mut input = String::new();
let mut frame_index = 0usize;
let status = "Hugging Face API requires a token for model access".to_string();
let mut show_plain = false;
loop {
draw_full_frame(terminal, |frame| {
draw_hf_token_tui(frame, &input, &status, frame_index, show_plain);
})?;
frame_index = frame_index.wrapping_add(1);
if !event::poll(Duration::from_millis(50))? {
continue;
}
let Event::Key(key) = event::read()? else {
continue;
};
if key.kind != KeyEventKind::Press {
continue;
}
match key.code {
KeyCode::Esc => return Ok(None),
KeyCode::Enter => {
if input.is_empty() {
return Ok(None);
}
return Ok(Some(input));
}
KeyCode::Backspace => {
input.pop();
}
KeyCode::Char(ch) => {
input.push(ch);
show_plain = true;
}
_ => {}
}
}
}
fn prepare_output_dir(output: &Path) -> Result<PathBuf> {
if output.exists() && !output.is_dir() {
return Err(GitArchiverError::OutputNotDirectory(
output.display().to_string(),
));
}
fs::create_dir_all(output)?;
Ok(output.to_path_buf())
}
fn write_archive_manifest(
output: &Path,
platform: &str,
target: &str,
complete: bool,
) -> Result<ArchiveManifest> {
let previous_manifest = read_archive_manifest_sidecar(output).ok();
let manifest = build_archive_manifest_with_previous(
output,
platform,
target,
complete,
previous_manifest.as_ref(),
)?;
let metadata_dir = output.join(THESA_METADATA_DIR);
fs::create_dir_all(&metadata_dir)?;
let manifest_path = metadata_dir.join("manifest.json");
let tmp_manifest_path = metadata_dir.join("manifest.json.tmp");
let manifest_json = serde_json::to_string(&manifest).map_err(|err| {
GitArchiverError::Message(format!("manifest serialization failed: {err}"))
})?;
fs::write(&tmp_manifest_path, manifest_json)?;
fs::rename(&tmp_manifest_path, &manifest_path)?;
let checksums_path = metadata_dir.join("checksums.blake3");
let tmp_checksums_path = metadata_dir.join("checksums.blake3.tmp");
let mut checksums = String::new();
for file in &manifest.files {
checksums.push_str(&file.blake3);
checksums.push_str(" ");
checksums.push_str(&file.path);
checksums.push('\n');
}
fs::write(&tmp_checksums_path, checksums)?;
fs::rename(&tmp_checksums_path, &checksums_path)?;
Ok(manifest)
}
#[cfg(test)]
fn build_archive_manifest(
output: &Path,
platform: &str,
target: &str,
complete: bool,
) -> Result<ArchiveManifest> {
build_archive_manifest_with_previous(output, platform, target, complete, None)
}
fn build_archive_manifest_with_previous(
output: &Path,
platform: &str,
target: &str,
complete: bool,
previous: Option<&ArchiveManifest>,
) -> Result<ArchiveManifest> {
let mut files = collect_manifest_files(output, previous)?;
files.sort_by(|a, b| a.path.cmp(&b.path));
let total_bytes = files.iter().map(|file| file.size).sum();
let created_at_unix = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_err(|err| {
GitArchiverError::Message(format!("system clock is before UNIX epoch: {err}"))
})?
.as_secs();
Ok(ArchiveManifest {
format_version: THESA_FORMAT_VERSION.to_string(),
tool: ManifestTool {
name: "thesa".to_string(),
version: env!("CARGO_PKG_VERSION").to_string(),
},
source: ManifestSource {
platform: platform.to_string(),
target: target.to_string(),
},
archive: ManifestArchive {
created_at_unix,
complete,
file_count: files.len(),
total_bytes,
checksum_algorithm: "blake3".to_string(),
},
files,
})
}
fn collect_manifest_files(
root: &Path,
previous: Option<&ArchiveManifest>,
) -> Result<Vec<ManifestFile>> {
let mut files = Vec::new();
let previous = previous_manifest_file_map(previous);
collect_manifest_files_inner(root, root, &previous, &mut files)?;
Ok(files)
}
fn previous_manifest_file_map(
previous: Option<&ArchiveManifest>,
) -> BTreeMap<String, ManifestFile> {
previous
.into_iter()
.flat_map(|manifest| manifest.files.iter().cloned())
.filter(|file| validate_manifest_relative_path(&file.path).is_ok())
.map(|file| (file.path.clone(), file))
.collect()
}
fn collect_manifest_files_inner(
root: &Path,
current: &Path,
previous: &BTreeMap<String, ManifestFile>,
files: &mut Vec<ManifestFile>,
) -> Result<()> {
for entry in fs::read_dir(current)? {
let entry = entry?;
let path = entry.path();
let name = entry.file_name();
if path.is_dir() {
if name == THESA_METADATA_DIR {
continue;
}
collect_manifest_files_inner(root, &path, previous, files)?;
continue;
}
if !path.is_file() {
continue;
}
let metadata = fs::metadata(&path)?;
let relative = path
.strip_prefix(root)
.map_err(|err| GitArchiverError::Message(format!("manifest path error: {err}")))?;
let path = manifest_path(relative)?;
let modified_unix_nanos = metadata_modified_unix_nanos(&metadata);
let blake3 = if let Some(cached) = previous.get(&path) {
if cached.size == metadata.len() && cached.modified_unix_nanos == modified_unix_nanos {
cached.blake3.clone()
} else {
blake3_hash_file(&path_from_root(root, &path)?)?
}
} else {
blake3_hash_file(&path_from_root(root, &path)?)?
};
files.push(ManifestFile {
path,
size: metadata.len(),
modified_unix_nanos,
blake3,
});
}
Ok(())
}
fn path_from_root(root: &Path, manifest_path: &str) -> Result<PathBuf> {
let relative = validate_manifest_relative_path(manifest_path).map_err(|message| {
GitArchiverError::Message(format!(
"invalid generated manifest path '{manifest_path}': {message}"
))
})?;
Ok(root.join(relative))
}
fn metadata_modified_unix_nanos(metadata: &fs::Metadata) -> Option<u64> {
metadata
.modified()
.ok()
.and_then(|time| time.duration_since(UNIX_EPOCH).ok())
.and_then(|duration| u64::try_from(duration.as_nanos()).ok())
}
fn blake3_hash_file(path: &Path) -> Result<String> {
let mut file = fs::File::open(path)?;
let mut hasher = blake3::Hasher::new();
io::copy(&mut file, &mut hasher)?;
Ok(hasher.finalize().to_hex().to_string())
}
fn manifest_path(path: &Path) -> Result<String> {
let normalized = path
.components()
.map(|component| component.as_os_str().to_string_lossy())
.collect::<Vec<_>>()
.join("/");
validate_manifest_relative_path(&normalized).map_err(|message| {
GitArchiverError::Message(format!(
"refusing to write unsafe manifest path '{normalized}': {message}"
))
})?;
Ok(normalized)
}
fn read_archive_manifest_sidecar(root: &Path) -> Result<ArchiveManifest> {
let manifest_path = root.join(THESA_METADATA_DIR).join("manifest.json");
let contents = fs::read_to_string(&manifest_path)?;
serde_json::from_str::<ArchiveManifest>(&contents).map_err(|err| {
GitArchiverError::Message(format!(
"unable to parse existing manifest {}: {err}",
manifest_path.display()
))
})
}
fn run_verify_command(archive: &Path, json: bool) -> Result<()> {
let report = verify_archive(archive);
if json {
let payload = serde_json::to_string_pretty(&report).map_err(|err| {
GitArchiverError::Message(format!("verification report serialization failed: {err}"))
})?;
println!("{payload}");
} else {
print_verify_report(&report);
}
if report.ok {
Ok(())
} else {
Err(GitArchiverError::Message(
"archive verification failed".to_string(),
))
}
}
fn verify_archive(input: &Path) -> VerifyReport {
let archive_root = archive_root_from_input(input);
let mut report = VerifyReport {
archive: archive_root.display().to_string(),
ok: false,
files_verified: 0,
bytes_verified: 0,
manifest_valid: false,
checksums_valid: false,
path_safety_valid: true,
warnings: Vec::new(),
errors: Vec::new(),
};
if !archive_root.is_dir() {
report.errors.push(format!(
"archive path is not a directory: {}",
archive_root.display()
));
return report;
}
let metadata_dir = archive_root.join(THESA_METADATA_DIR);
let manifest_path = metadata_dir.join("manifest.json");
let manifest = match fs::read_to_string(&manifest_path) {
Ok(contents) => match serde_json::from_str::<ArchiveManifest>(&contents) {
Ok(manifest) => manifest,
Err(err) => {
report
.errors
.push(format!("manifest is not valid JSON: {err}"));
return report;
}
},
Err(err) => {
report.errors.push(format!(
"unable to read manifest {}: {err}",
manifest_path.display()
));
return report;
}
};
report.manifest_valid = true;
if manifest.format_version != THESA_FORMAT_VERSION {
report.errors.push(format!(
"unsupported archive format version '{}', expected '{}'",
manifest.format_version, THESA_FORMAT_VERSION
));
}
if manifest.archive.checksum_algorithm != "blake3" {
report.errors.push(format!(
"unsupported checksum algorithm '{}', expected 'blake3'",
manifest.archive.checksum_algorithm
));
}
if !manifest.archive.complete {
report
.errors
.push("manifest marks this archive as incomplete".to_string());
}
if manifest.archive.file_count != manifest.files.len() {
report.errors.push(format!(
"manifest file_count {} does not match listed files {}",
manifest.archive.file_count,
manifest.files.len()
));
}
let manifest_total_bytes = manifest.files.iter().map(|file| file.size).sum::<u64>();
if manifest.archive.total_bytes != manifest_total_bytes {
report.errors.push(format!(
"manifest total_bytes {} does not match listed file bytes {}",
manifest.archive.total_bytes, manifest_total_bytes
));
}
let checksums_path = metadata_dir.join("checksums.blake3");
let checksums = match read_checksum_sidecar(&checksums_path) {
Ok(checksums) => {
report.checksums_valid = true;
checksums
}
Err(message) => {
report.errors.push(message);
BTreeMap::new()
}
};
let mut manifest_paths = BTreeSet::new();
for file in &manifest.files {
if !manifest_paths.insert(file.path.clone()) {
report
.errors
.push(format!("duplicate manifest path: {}", file.path));
continue;
}
let relative = match validate_manifest_relative_path(&file.path) {
Ok(relative) => relative,
Err(message) => {
report.path_safety_valid = false;
report
.errors
.push(format!("unsafe manifest path '{}': {message}", file.path));
continue;
}
};
if let Some(sidecar_hash) = checksums.get(&file.path) {
if !sidecar_hash.eq_ignore_ascii_case(&file.blake3) {
report.errors.push(format!(
"checksum sidecar disagrees with manifest for {}",
file.path
));
report.checksums_valid = false;
}
} else if !checksums.is_empty() {
report
.errors
.push(format!("checksum sidecar is missing path {}", file.path));
report.checksums_valid = false;
}
let path = archive_root.join(&relative);
let metadata = match fs::metadata(&path) {
Ok(metadata) if metadata.is_file() => metadata,
Ok(_) => {
report.errors.push(format!(
"manifest path is not a regular file: {}",
file.path
));
continue;
}
Err(err) => {
report
.errors
.push(format!("missing manifest file {}: {err}", file.path));
continue;
}
};
if metadata.len() != file.size {
report.errors.push(format!(
"size mismatch for {}: manifest {}, actual {}",
file.path,
file.size,
metadata.len()
));
continue;
}
if let Some(expected_mtime) = file.modified_unix_nanos {
let actual_mtime = metadata_modified_unix_nanos(&metadata);
if actual_mtime != Some(expected_mtime) {
report.errors.push(format!(
"modified timestamp mismatch for {}: manifest {:?}, actual {:?}",
file.path,
Some(expected_mtime),
actual_mtime
));
continue;
}
}
match blake3_hash_file(&path) {
Ok(actual) if actual.eq_ignore_ascii_case(&file.blake3) => {
report.files_verified += 1;
report.bytes_verified += file.size;
}
Ok(actual) => report.errors.push(format!(
"checksum mismatch for {}: manifest {}, actual {}",
file.path, file.blake3, actual
)),
Err(err) => report
.errors
.push(format!("unable to hash {}: {err}", file.path)),
}
}
for path in checksums.keys() {
if !manifest_paths.contains(path) {
report.errors.push(format!(
"checksum sidecar contains path not in manifest: {path}"
));
report.checksums_valid = false;
}
}
report.ok = report.errors.is_empty()
&& report.manifest_valid
&& report.checksums_valid
&& report.path_safety_valid
&& report.files_verified == manifest.files.len();
report
}
fn archive_root_from_input(input: &Path) -> PathBuf {
if input.file_name().and_then(|name| name.to_str()) == Some(THESA_METADATA_DIR) {
input.parent().unwrap_or(input).to_path_buf()
} else {
input.to_path_buf()
}
}
fn read_checksum_sidecar(path: &Path) -> std::result::Result<BTreeMap<String, String>, String> {
let contents = fs::read_to_string(path)
.map_err(|err| format!("unable to read checksum sidecar {}: {err}", path.display()))?;
let mut checksums = BTreeMap::new();
for (line_index, raw_line) in contents.lines().enumerate() {
let line = raw_line.trim();
if line.is_empty() {
continue;
}
let (hash, path) = line.split_once(" ").ok_or_else(|| {
format!(
"invalid checksum sidecar line {}: expected '<blake3> <path>'",
line_index + 1
)
})?;
if hash.len() != 64 || !hash.chars().all(|ch| ch.is_ascii_hexdigit()) {
return Err(format!(
"invalid BLAKE3 hash on checksum sidecar line {}",
line_index + 1
));
}
validate_manifest_relative_path(path).map_err(|message| {
format!(
"unsafe path in checksum sidecar line {}: {path}: {message}",
line_index + 1
)
})?;
if checksums
.insert(path.to_string(), hash.to_ascii_lowercase())
.is_some()
{
return Err(format!(
"duplicate path in checksum sidecar line {}: {path}",
line_index + 1
));
}
}
Ok(checksums)
}
fn validate_manifest_relative_path(path: &str) -> std::result::Result<PathBuf, String> {
if path.is_empty() {
return Err("path is empty".to_string());
}
if path.contains('\\') {
return Err("backslashes are not allowed in manifest paths".to_string());
}
let path_ref = Path::new(path);
if path_ref.is_absolute() {
return Err("absolute paths are not allowed".to_string());
}
let mut safe = PathBuf::new();
for (index, component) in path_ref.components().enumerate() {
match component {
Component::Normal(value) => {
if index == 0 && value == THESA_METADATA_DIR {
return Err("manifest paths may not target .thesa metadata".to_string());
}
safe.push(value);
}
Component::CurDir => {
return Err("current-directory components are not allowed".to_string());
}
Component::ParentDir => {
return Err("parent-directory components are not allowed".to_string());
}
Component::RootDir | Component::Prefix(_) => {
return Err("root or prefix components are not allowed".to_string());
}
}
}
if safe.as_os_str().is_empty() {
return Err("path has no normal components".to_string());
}
Ok(safe)
}
fn print_verify_report(report: &VerifyReport) {
println!("Archive: {}", report.archive);
println!("Status: {}", if report.ok { "OK" } else { "FAILED" });
println!("Files: {} verified", report.files_verified);
println!("Bytes: {} verified", report.bytes_verified);
println!(
"Manifest: {}",
if report.manifest_valid {
"valid"
} else {
"invalid"
}
);
println!(
"Checksums: {}",
if report.checksums_valid {
"valid"
} else {
"invalid"
}
);
println!(
"Path safety: {}",
if report.path_safety_valid {
"valid"
} else {
"invalid"
}
);
println!("Warnings: {}", report.warnings.len());
for warning in &report.warnings {
println!(" - warning: {warning}");
}
println!("Errors: {}", report.errors.len());
for error in &report.errors {
println!(" - error: {error}");
}
}
fn with_auth(
mut request: reqwest::blocking::RequestBuilder,
token: Option<&str>,
) -> reqwest::blocking::RequestBuilder {
request = request.header("Accept", "application/vnd.github+json");
if let Some(token) = token {
request = request.bearer_auth(token);
}
request
}
fn parse_api_error(response: Response) -> GitArchiverError {
let status = response.status();
let body = response
.text()
.unwrap_or_else(|_| "<unable to read body>".to_string());
GitArchiverError::ApiError {
status: status.as_u16(),
body,
}
}
fn list_repos_for_owner(
client: &Client,
owner: &str,
token: Option<&str>,
) -> Result<Vec<RepoItem>> {
list_repos_with_scope(client, "users", owner, token)
}
fn list_repos_for_org(client: &Client, org: &str, token: Option<&str>) -> Result<Vec<RepoItem>> {
list_repos_with_scope(client, "orgs", org, token)
}
fn list_repos_with_scope(
client: &Client,
scope: &str,
entity: &str,
token: Option<&str>,
) -> Result<Vec<RepoItem>> {
let mut page = 1usize;
let mut repos = Vec::new();
loop {
let url = format!(
"{}/{}/{}/repos?type=public&per_page={}&page={}",
GITHUB_API, scope, entity, DEFAULT_PAGE_SIZE, page
);
let response = with_auth(client.get(&url), token).send()?;
let status = response.status();
if status == StatusCode::NOT_FOUND {
return Err(GitArchiverError::NotFound(entity.to_string()));
}
if !status.is_success() {
return Err(parse_api_error(response));
}
let batch: Vec<RepoResponse> = response.json()?;
let count = batch.len();
repos.extend(batch.into_iter().filter_map(|repo| {
if repo.private {
return None;
}
Some(RepoItem {
name: repo.name,
full_name: repo.full_name,
clone_url: repo.clone_url,
})
}));
if count < DEFAULT_PAGE_SIZE {
break;
}
page = page.saturating_add(1);
}
Ok(repos)
}
fn fetch_single_repo(
client: &Client,
owner: &str,
repo: &str,
token: Option<&str>,
) -> Result<RepoItem> {
let url = format!("{}/repos/{}/{}", GITHUB_API, owner, repo);
let response = with_auth(client.get(&url), token).send()?;
if response.status() == StatusCode::NOT_FOUND {
return Err(GitArchiverError::NotFound(format!("{owner}/{repo}")));
}
if !response.status().is_success() {
return Err(parse_api_error(response));
}
let data: RepoResponse = response.json()?;
if data.private {
return Err(GitArchiverError::PrivateRepo(data.full_name));
}
Ok(RepoItem {
name: data.name,
full_name: data.full_name,
clone_url: data.clone_url,
})
}
fn clone_repository(repo: &RepoItem, output: &Path, args: &Cli) -> Result<CloneStatus> {
match prepare_clone(repo, output, args)? {
CloneAction::Skipped => Ok(CloneStatus::Skipped),
CloneAction::Running(mut child) => {
let status = child.wait()?;
if status.success() {
return Ok(CloneStatus::Cloned);
}
let code = status
.code()
.map_or_else(|| "signal/unknown".to_string(), |code| code.to_string());
Err(GitArchiverError::CloneFailed {
repo: repo.full_name.clone(),
message: format!("exit code {code}"),
})
}
}
}
fn clone_repository_with_progress(
repo: &RepoItem,
output: &Path,
args: &Cli,
stdout: &mut io::Stdout,
spinner: &Loader,
position: usize,
total: usize,
) -> Result<CloneStatus> {
let mut child = match prepare_clone(repo, output, args)? {
CloneAction::Skipped => {
write!(stdout, "\r\x1b[2K")?;
println!("[skip] {} ({}/{}) exists", repo.full_name, position, total);
return Ok(CloneStatus::Skipped);
}
CloneAction::Running(child) => child,
};
let mut tick = 0usize;
loop {
if let Some(status) = child.try_wait()? {
if status.success() {
write!(stdout, "\r\x1b[2K")?;
println!("[done] {} ({}/{})", repo.full_name, position, total);
return Ok(CloneStatus::Cloned);
}
let code = status
.code()
.map_or_else(|| "signal/unknown".to_string(), |code| code.to_string());
return Err(GitArchiverError::CloneFailed {
repo: repo.full_name.clone(),
message: format!("exit code {code}"),
});
}
let progress = LoaderProgress::from_counts(position as u64, total as u64);
let frame = spinner.line(tick, progress);
let status = format!("[{}/{}] cloning {}", position, total, repo.full_name);
write!(stdout, "\r\x1b[2K{} {}", frame, status)?;
stdout.flush()?;
tick = tick.saturating_add(1);
thread::sleep(Duration::from_millis(120));
}
}
fn prepare_clone(repo: &RepoItem, output: &Path, args: &Cli) -> Result<CloneAction> {
let destination = output.join(&repo.name);
if destination.exists() {
if args.skip_existing {
return Ok(CloneAction::Skipped);
}
if destination.is_dir() {
remove_dir_all(&destination)?;
} else {
remove_file(&destination)?;
}
}
let child = spawn_git_clone(repo, output, args)?;
Ok(CloneAction::Running(child))
}
fn spawn_git_clone(repo: &RepoItem, output: &Path, args: &Cli) -> Result<Child> {
let mut command_args = vec!["clone".to_string()];
if let Some(depth) = args.depth {
command_args.push("--depth".to_string());
command_args.push(depth.to_string());
}
command_args.push(repo.clone_url.clone());
command_args.push(repo.name.clone());
match Command::new("git")
.current_dir(output)
.args(&command_args)
.stdout(Stdio::null())
.stderr(Stdio::null())
.spawn()
{
Ok(child) => Ok(child),
Err(error) if error.kind() == io::ErrorKind::NotFound => Err(GitArchiverError::MissingGit),
Err(error) => Err(error.into()),
}
}
fn paint(text: impl AsRef<str>, color: AislingColor) -> String {
let color = fg(color);
format!("{color}{}\x1b[0m", text.as_ref())
}
fn paint_bold(text: impl AsRef<str>, color: AislingColor) -> String {
let color = fg(color);
format!("\x1b[1m{color}{}\x1b[0m", text.as_ref())
}
fn fg(color: AislingColor) -> String {
format!("\x1b[38;2;{};{};{}m", color.r, color.g, color.b)
}
fn thesa_gradient() -> Gradient {
Gradient::new(
vec![
AislingColor::rgb(0, 255, 210),
AislingColor::rgb(142, 92, 255),
AislingColor::rgb(255, 80, 210),
AislingColor::rgb(255, 196, 92),
],
36,
)
}
fn parse_indices(input: &str, max_len: usize) -> std::result::Result<Vec<usize>, String> {
let mut unique = Vec::new();
for token in input.split(',') {
let token = token.trim();
if token.is_empty() {
continue;
}
if let Some((start, end)) = token.split_once('-') {
let start = start
.trim()
.parse::<usize>()
.map_err(|_| "Invalid start of range".to_string())?;
let end = end
.trim()
.parse::<usize>()
.map_err(|_| "Invalid end of range".to_string())?;
if start == 0 || end == 0 || start > end {
return Err(
"Invalid range: indexes are 1-based and must be start <= end".to_string(),
);
}
let (start, end) = (start - 1, end - 1);
if end >= max_len {
return Err(format!(
"Range end {input} is above result count {max_len}",
input = end + 1,
));
}
for index in start..=end {
if unique.contains(&index) {
continue;
}
unique.push(index);
}
continue;
}
let index = token
.parse::<usize>()
.map_err(|_| format!("Invalid index '{token}'"))?;
if index == 0 {
return Err("Indexes are 1-based, use 1 or greater".to_string());
}
let index = index - 1;
if index >= max_len {
return Err(format!(
"Index {} is above result count {}",
index + 1,
max_len
));
}
if !unique.contains(&index) {
unique.push(index);
}
}
if unique.is_empty() {
return Err("No valid indexes provided".to_string());
}
Ok(unique)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parse_model_target_for_provider_hf() {
let hf_target = parse_model_target_for_provider("meta-llama/Llama-2-7b", ModelProvider::Hf)
.expect("valid hf target");
match hf_target {
ModelTarget::Model(model) => assert_eq!(model, "meta-llama/Llama-2-7b"),
ModelTarget::Org(_) => panic!("expected model target"),
ModelTarget::Top | ModelTarget::Latest => panic!("unexpected preset target"),
}
let hf_preset = parse_model_target_for_provider("top", ModelProvider::Hf)
.expect("valid hf preset target");
assert!(matches!(hf_preset, ModelTarget::Top));
let hf_latest = parse_model_target_for_provider("latest", ModelProvider::Hf)
.expect("valid hf latest target");
assert!(matches!(hf_latest, ModelTarget::Latest));
let hf_url = parse_model_target_for_provider(
"https://huggingface.co/openai-community/gpt2",
ModelProvider::Hf,
)
.expect("valid hf url target");
match hf_url {
ModelTarget::Model(model) => assert_eq!(model, "openai-community/gpt2"),
ModelTarget::Org(_) => panic!("expected model target from URL"),
ModelTarget::Top | ModelTarget::Latest => panic!("unexpected preset target"),
}
}
#[test]
fn parse_model_target_for_provider_civitai() {
let civitai_id = parse_model_target_for_provider("827184", ModelProvider::Civitai)
.expect("valid numeric id");
match civitai_id {
ModelTarget::Model(model) => assert_eq!(model, "827184"),
ModelTarget::Org(_) => panic!("expected numeric model target"),
ModelTarget::Top | ModelTarget::Latest => panic!("unexpected preset target"),
}
let civitai_preset = parse_model_target_for_provider("newest", ModelProvider::Civitai)
.expect("valid civitai preset target");
assert!(matches!(civitai_preset, ModelTarget::Latest));
let civitai_url = parse_model_target_for_provider(
"https://civitai.com/models/827184",
ModelProvider::Civitai,
)
.expect("valid civitai url");
match civitai_url {
ModelTarget::Model(model) => assert_eq!(model, "827184"),
ModelTarget::Org(_) => panic!("expected numeric model target from URL"),
ModelTarget::Top | ModelTarget::Latest => panic!("unexpected preset target"),
}
let civitai_keyword =
parse_model_target_for_provider("realistic-vision", ModelProvider::Civitai)
.expect("valid keyword target");
match civitai_keyword {
ModelTarget::Org(org) => assert_eq!(org, "realistic-vision"),
ModelTarget::Model(_) => panic!("expected keyword treated as org"),
ModelTarget::Top | ModelTarget::Latest => panic!("unexpected preset target"),
}
}
#[test]
fn parse_model_target_for_provider_ollama() {
let ollama_keyword = parse_model_target_for_provider("llama3", ModelProvider::Ollama)
.expect("valid ollama target");
match ollama_keyword {
ModelTarget::Org(org) => assert_eq!(org, "llama3"),
ModelTarget::Model(_) => panic!("expected org-style target for keyword"),
ModelTarget::Top | ModelTarget::Latest => panic!("unexpected preset target"),
}
let ollama_preset = parse_model_target_for_provider("trending", ModelProvider::Ollama)
.expect("valid ollama preset target");
assert!(matches!(ollama_preset, ModelTarget::Top));
}
#[test]
fn format_model_size_and_truncate_to_width() {
assert_eq!(format_model_size(None), "n/a");
assert_eq!(format_model_size(Some(512)), "512 B");
assert_eq!(format_model_size(Some(2048)), "2.0KB");
assert_eq!(format_model_size(Some(4_000_000)), "3.8MB");
assert_eq!(truncate_to_width("hello", 0), "");
assert_eq!(truncate_to_width("hello", 10), "hello");
assert_eq!(truncate_to_width("hello", 3), "hel");
assert_eq!(truncate_to_width("hello", 4), "h...");
assert_eq!(truncate_to_width("hello", 5), "hello");
}
#[test]
fn parse_site_url_accepts_http_and_https_only() {
assert!(parse_site_url("https://example.com").is_ok());
assert!(parse_site_url("http://127.0.0.1:8765/").is_ok());
assert!(parse_site_url("file:///tmp/site.html").is_err());
assert!(parse_site_url("not a url").is_err());
}
#[test]
fn parse_site_headers_accepts_name_value_pairs() {
let headers = parse_site_headers(&[
"Authorization: Bearer token".to_string(),
"X-Archive: thesa".to_string(),
])
.expect("valid headers");
assert_eq!(
headers,
vec![
("Authorization".to_string(), "Bearer token".to_string()),
("X-Archive".to_string(), "thesa".to_string()),
]
);
assert!(parse_site_headers(&["missing-colon".to_string()]).is_err());
}
#[test]
fn load_site_seed_urls_reads_additional_seed_file() {
let root = test_temp_dir("site_seed_file");
fs::create_dir_all(&root).expect("create fixture dir");
let input = root.join("seeds.txt");
fs::write(
&input,
"# docs\nhttps://example.com/docs\n\nhttp://127.0.0.1:8765/page\n",
)
.expect("write seed file");
let seeds = load_site_seed_urls(Some("https://example.com"), Some(&input))
.expect("load site seed urls");
assert_eq!(seeds.len(), 3);
assert_eq!(seeds[0].as_str(), "https://example.com/");
assert_eq!(seeds[1].as_str(), "https://example.com/docs");
assert_eq!(seeds[2].as_str(), "http://127.0.0.1:8765/page");
fs::remove_dir_all(root).expect("cleanup temp dir");
}
#[test]
fn archive_url_cli_dry_run_before_subcommand_parses_as_archive_command() {
let cli = Cli::try_parse_from([
"thesa",
"--dry-run",
"archive",
"url",
"https://example.com",
])
.expect("parse archive url dry-run command");
assert!(cli.dry_run);
let Some(Commands::Archive { dry_run, source }) = cli.command else {
panic!("expected archive command");
};
assert!(!dry_run);
let ArchiveCommands::Url(args) = source else {
panic!("expected url archive source");
};
assert_eq!(args.url.as_deref(), Some("https://example.com"));
assert!(!args.dry_run);
}
#[test]
fn archive_url_cli_dry_run_after_subcommand_parses_as_archive_command() {
let cli = Cli::try_parse_from([
"thesa",
"archive",
"url",
"https://example.com",
"--dry-run",
])
.expect("parse archive url command-level dry-run");
assert!(!cli.dry_run);
let Some(Commands::Archive { dry_run, source }) = cli.command else {
panic!("expected archive command");
};
assert!(!dry_run);
let ArchiveCommands::Url(args) = source else {
panic!("expected url archive source");
};
assert_eq!(args.url.as_deref(), Some("https://example.com"));
assert!(args.dry_run);
}
#[test]
fn archive_url_cli_dry_run_after_archive_parses_as_archive_command() {
let cli = Cli::try_parse_from([
"thesa",
"archive",
"--dry-run",
"url",
"https://example.com",
])
.expect("parse archive-level dry-run");
assert!(!cli.dry_run);
let Some(Commands::Archive { dry_run, source }) = cli.command else {
panic!("expected archive command");
};
assert!(dry_run);
let ArchiveCommands::Url(args) = source else {
panic!("expected url archive source");
};
assert_eq!(args.url.as_deref(), Some("https://example.com"));
assert!(!args.dry_run);
}
#[test]
fn archive_url_cli_parses_siteforge_extended_options() {
let cli = Cli::try_parse_from([
"thesa",
"archive",
"url",
"https://example.com",
"--input",
"seeds.txt",
"-j",
"8",
"--max-page-size-bytes",
"1024",
"--max-asset-size-bytes",
"2048",
"--max-total-archive-size-bytes",
"4096",
"--browser-command",
"chromium --headless {url}",
"--browser-profile",
"profile",
"--render-wait-ms",
"250",
"--header",
"X-Test: true",
"--cookie",
"session=test",
"--user-agent",
"thesa-test",
"--timeout-secs",
"7",
"--retry-count",
"3",
"--ocr",
])
.expect("parse extended siteforge options");
let Some(Commands::Archive { source, .. }) = cli.command else {
panic!("expected archive command");
};
let ArchiveCommands::Url(args) = source else {
panic!("expected url archive source");
};
assert_eq!(args.url.as_deref(), Some("https://example.com"));
assert_eq!(args.input, Some(PathBuf::from("seeds.txt")));
assert_eq!(args.concurrency, Some(8));
assert_eq!(args.max_page_size_bytes, Some(1024));
assert_eq!(args.max_asset_size_bytes, Some(2048));
assert_eq!(args.max_total_archive_size_bytes, Some(4096));
assert_eq!(
args.browser_command.as_deref(),
Some("chromium --headless {url}")
);
assert_eq!(args.browser_profile, Some(PathBuf::from("profile")));
assert_eq!(args.render_wait_ms, Some(250));
assert_eq!(args.headers, vec!["X-Test: true".to_string()]);
assert_eq!(args.cookie.as_deref(), Some("session=test"));
assert_eq!(args.user_agent.as_deref(), Some("thesa-test"));
assert_eq!(args.timeout_secs, Some(7));
assert_eq!(args.retry_count, Some(3));
assert!(args.ocr_enabled);
}
#[test]
fn archive_url_cli_allows_input_without_url() {
let cli = Cli::try_parse_from([
"thesa",
"archive",
"url",
"--input",
"seeds.txt",
"--dry-run",
])
.expect("parse input-only siteforge command");
let Some(Commands::Archive { source, .. }) = cli.command else {
panic!("expected archive command");
};
let ArchiveCommands::Url(args) = source else {
panic!("expected url archive source");
};
assert_eq!(args.url, None);
assert_eq!(args.input, Some(PathBuf::from("seeds.txt")));
assert!(args.dry_run);
}
#[test]
fn archive_cli_parses_siteforge_management_commands() {
let list = Cli::try_parse_from(["thesa", "archive", "list", "--archive-root", "./sites"])
.expect("parse siteforge list");
let Some(Commands::Archive {
source: ArchiveCommands::List(args),
..
}) = list.command
else {
panic!("expected archive list command");
};
assert_eq!(args.archive_root, PathBuf::from("./sites"));
let search = Cli::try_parse_from([
"thesa",
"archive",
"search",
"install",
"--archive-id",
"docs",
"--limit",
"5",
])
.expect("parse siteforge search");
let Some(Commands::Archive {
source: ArchiveCommands::Search(args),
..
}) = search.command
else {
panic!("expected archive search command");
};
assert_eq!(args.query, "install");
assert_eq!(args.archive_id.as_deref(), Some("docs"));
assert_eq!(args.limit, 5);
let export = Cli::try_parse_from([
"thesa",
"archive",
"export",
"docs",
"--format",
"agent-tar",
"--output",
"./exports/docs-agent",
])
.expect("parse siteforge export");
let Some(Commands::Archive {
source: ArchiveCommands::Export(args),
..
}) = export.command
else {
panic!("expected archive export command");
};
assert_eq!(args.archive_id, "docs");
assert_eq!(args.format, SiteforgeExportFormat::AgentTar);
assert_eq!(args.output, Some(PathBuf::from("./exports/docs-agent")));
let pack =
Cli::try_parse_from(["thesa", "archive", "pack", "docs", "--max-tokens", "80000"])
.expect("parse siteforge pack");
let Some(Commands::Archive {
source: ArchiveCommands::Pack(args),
..
}) = pack.command
else {
panic!("expected archive pack command");
};
assert_eq!(args.archive_id, "docs");
assert_eq!(args.max_tokens, Some(80_000));
for subcommand in ["inspect", "verify", "resume", "diagnostics"] {
let cli = Cli::try_parse_from(["thesa", "archive", subcommand, "docs"])
.expect("parse siteforge archive id command");
let Some(Commands::Archive { source, .. }) = cli.command else {
panic!("expected archive command");
};
match (subcommand, source) {
("inspect", ArchiveCommands::Inspect(args))
| ("verify", ArchiveCommands::Verify(args))
| ("resume", ArchiveCommands::Resume(args)) => assert_eq!(args.archive_id, "docs"),
("diagnostics", ArchiveCommands::Diagnostics(args)) => {
assert_eq!(args.archive_id.as_deref(), Some("docs"));
}
_ => panic!("unexpected command variant"),
}
}
}
#[test]
fn archive_refresh_cli_parses_root_filter_and_dry_run() {
let cli = Cli::try_parse_from([
"thesa",
"archive",
"refresh",
"--archive-root",
"./archives/repos",
"--filter",
"octo",
"--dry-run",
"--concurrency",
"2",
])
.expect("refresh command should parse");
match cli.command.expect("archive command") {
Commands::Archive {
source: ArchiveCommands::Refresh(args),
..
} => {
assert_eq!(args.archive_root, PathBuf::from("./archives/repos"));
assert_eq!(args.filter.as_deref(), Some("octo"));
assert!(args.dry_run);
assert_eq!(args.concurrency, 2);
}
_ => panic!("expected archive refresh command"),
}
}
#[test]
fn archive_refresh_aliases_parse() {
for alias in ["update", "pull"] {
let cli = Cli::try_parse_from(["thesa", "archive", alias, "--dry-run"])
.expect("refresh alias should parse");
assert!(matches!(
cli.command,
Some(Commands::Archive {
source: ArchiveCommands::Refresh(_),
..
})
));
}
}
#[test]
fn parses_github_remote_slugs() {
assert_eq!(
repoforge::parse_github_remote_slug("https://github.com/octocat/Hello-World.git"),
Some("octocat/Hello-World".to_string())
);
assert_eq!(
repoforge::parse_github_remote_slug("git@github.com:Tknott95/GitArchiver.git"),
Some("Tknott95/GitArchiver".to_string())
);
assert_eq!(
repoforge::parse_github_remote_slug("ssh://git@github.com/owner/repo"),
Some("owner/repo".to_string())
);
assert_eq!(
repoforge::parse_github_remote_slug("https://example.com/owner/repo"),
None
);
}
#[test]
fn discovers_git_archives_from_archive_tree() {
let root = test_temp_dir("discover_git_archives");
let repo = root.join("octocat").join("Hello-World");
let skipped = root.join("sites").join("docs").join(THESA_METADATA_DIR);
fs::create_dir_all(repo.join(".git")).expect("create fake git worktree marker");
fs::create_dir_all(skipped).expect("create skipped metadata dir");
let archives = repoforge::discover_git_archives(&root, None).expect("discover archives");
assert_eq!(archives.len(), 1);
assert_eq!(archives[0].path, repo);
assert_eq!(archives[0].slug, "octocat/Hello-World");
let filtered =
repoforge::discover_git_archives(&root, Some("hello")).expect("filter archives");
assert_eq!(filtered.len(), 1);
let none =
repoforge::discover_git_archives(&root, Some("nomatch")).expect("filter archives");
assert!(none.is_empty());
fs::remove_dir_all(root).expect("cleanup temp dir");
}
#[test]
fn site_tui_options_cycle_safe_presets() {
let mut options = SiteTuiOptions::default();
assert!(options.full_site);
assert_eq!(options.max_depth, 2);
assert_eq!(options.max_pages, 100);
assert_eq!(options.archive_profile, ArchiveOutputProfile::Full);
assert!(!options.render_js);
assert!(!options.render_js_auto);
assert_eq!(next_site_depth(1), 2);
assert_eq!(next_site_depth(2), 4);
assert_eq!(next_site_depth(4), 8);
assert_eq!(next_site_depth(8), 1);
assert_eq!(next_site_pages(25), 100);
assert_eq!(next_site_pages(100), 500);
assert_eq!(next_site_pages(500), 1000);
assert_eq!(next_site_pages(1000), 25);
apply_site_option_row(&mut options, 0);
assert!(!options.full_site);
apply_site_option_row(&mut options, 1);
assert_eq!(options.max_depth, 4);
apply_site_option_row(&mut options, 2);
assert_eq!(options.max_pages, 500);
apply_site_option_row(&mut options, 3);
assert!(!options.render_js);
assert!(options.render_js_auto);
apply_site_option_row(&mut options, 3);
assert!(options.render_js);
assert!(!options.render_js_auto);
apply_site_option_row(&mut options, 4);
assert_eq!(options.archive_profile, ArchiveOutputProfile::Agent);
apply_site_option_row(&mut options, 4);
assert_eq!(options.archive_profile, ArchiveOutputProfile::Lean);
}
#[test]
fn parses_site_archive_profile_and_auto_render_flags() {
let cli = Cli::try_parse_from([
"thesa",
"archive",
"site",
"https://example.com",
"--archive-profile",
"lean",
"--render-js-auto",
])
.expect("site archive args should parse");
match cli.command.expect("archive command") {
Commands::Archive {
source: ArchiveCommands::Site(args),
..
} => {
assert_eq!(args.archive_profile, Some(SiteArchiveProfile::Lean));
assert!(args.render_js_auto);
assert!(!args.render_js);
}
_ => panic!("expected site archive command"),
}
}
#[test]
fn visible_list_item_hit_tracks_render_offset() {
let inner = Rect::new(4, 3, 20, 4);
assert_eq!(visible_list_item_at(inner, 4, 3, 10, 0), Some(0));
assert_eq!(visible_list_item_at(inner, 4, 6, 10, 0), Some(3));
assert_eq!(visible_list_item_at(inner, 4, 3, 10, 6), Some(3));
assert_eq!(visible_list_item_at(inner, 4, 6, 10, 6), Some(6));
assert_eq!(visible_list_item_at(inner, 4, 7, 10, 6), None);
}
#[test]
fn list_render_offset_matches_scrin_list_state_behavior() {
assert_eq!(list_render_offset(0, 5, 4), 0);
assert_eq!(list_render_offset(10, 0, 4), 0);
assert_eq!(list_render_offset(10, 3, 4), 0);
assert_eq!(list_render_offset(10, 4, 4), 1);
assert_eq!(list_render_offset(10, 9, 4), 6);
}
#[test]
fn archive_manifest_hashes_files_and_skips_sidecar_metadata() {
let root = test_temp_dir("manifest_hashes_files");
fs::create_dir_all(root.join("src")).expect("create nested fixture dir");
fs::write(root.join("README.md"), b"hello thesa\n").expect("write readme");
fs::write(root.join("src/lib.rs"), b"pub fn archived() {}\n").expect("write source");
fs::create_dir_all(root.join(THESA_METADATA_DIR)).expect("create metadata dir");
fs::write(root.join(THESA_METADATA_DIR).join("old"), b"ignore me").expect("write sidecar");
let manifest = build_archive_manifest(&root, "github", "owner/repo", true)
.expect("manifest should build");
assert_eq!(manifest.format_version, THESA_FORMAT_VERSION);
assert_eq!(manifest.tool.name, "thesa");
assert_eq!(manifest.source.platform, "github");
assert_eq!(manifest.source.target, "owner/repo");
assert!(manifest.archive.complete);
assert_eq!(manifest.archive.file_count, 2);
assert_eq!(manifest.files[0].path, "README.md");
assert_eq!(manifest.files[1].path, "src/lib.rs");
assert!(manifest.files[0].modified_unix_nanos.is_some());
assert!(manifest.files[1].modified_unix_nanos.is_some());
assert_eq!(
manifest.files[0].blake3,
blake3::hash(b"hello thesa\n").to_hex().to_string()
);
assert!(
manifest
.files
.iter()
.all(|file| !file.path.starts_with(THESA_METADATA_DIR))
);
fs::remove_dir_all(root).expect("cleanup temp dir");
}
#[test]
fn write_archive_manifest_creates_manifest_and_checksum_sidecars() {
let root = test_temp_dir("manifest_writes_sidecars");
fs::create_dir_all(&root).expect("create fixture dir");
fs::write(root.join("artifact.txt"), b"artifact bytes").expect("write artifact");
let manifest = write_archive_manifest(&root, "model:hf", "openai-community/gpt2", false)
.expect("write manifest");
assert!(!manifest.archive.complete);
let manifest_path = root.join(THESA_METADATA_DIR).join("manifest.json");
let checksums_path = root.join(THESA_METADATA_DIR).join("checksums.blake3");
assert!(manifest_path.exists());
assert!(checksums_path.exists());
let manifest_text = fs::read_to_string(manifest_path).expect("read manifest sidecar");
assert!(!manifest_text.contains('\n'));
let parsed: ArchiveManifest =
serde_json::from_str(&manifest_text).expect("manifest sidecar should parse");
assert_eq!(parsed.source.platform, "model:hf");
assert_eq!(parsed.archive.file_count, 1);
assert!(parsed.files[0].modified_unix_nanos.is_some());
let checksums = fs::read_to_string(checksums_path).expect("read checksums sidecar");
assert!(checksums.contains("artifact.txt"));
assert!(checksums.contains(&blake3::hash(b"artifact bytes").to_hex().to_string()));
fs::remove_dir_all(root).expect("cleanup temp dir");
}
#[test]
fn verify_archive_accepts_valid_manifest_and_checksums() {
let root = test_temp_dir("verify_valid_archive");
fs::create_dir_all(&root).expect("create fixture dir");
fs::write(root.join("artifact.txt"), b"artifact bytes").expect("write artifact");
write_archive_manifest(&root, "github", "owner/repo", true).expect("write manifest");
let report = verify_archive(&root);
assert!(
report.ok,
"unexpected verification errors: {:?}",
report.errors
);
assert_eq!(report.files_verified, 1);
assert_eq!(report.bytes_verified, "artifact bytes".len() as u64);
assert!(report.manifest_valid);
assert!(report.checksums_valid);
assert!(report.path_safety_valid);
fs::remove_dir_all(root).expect("cleanup temp dir");
}
#[test]
fn verify_archive_rejects_corrupted_file() {
let root = test_temp_dir("verify_rejects_corruption");
fs::create_dir_all(&root).expect("create fixture dir");
fs::write(root.join("artifact.txt"), b"artifact bytes").expect("write artifact");
write_archive_manifest(&root, "github", "owner/repo", true).expect("write manifest");
fs::write(root.join("artifact.txt"), b"changed bytes").expect("corrupt artifact");
let report = verify_archive(&root);
assert!(!report.ok);
assert!(report
.errors
.iter()
.any(|error| error.contains("size mismatch") || error.contains("checksum mismatch")));
fs::remove_dir_all(root).expect("cleanup temp dir");
}
#[test]
fn verify_archive_rejects_unsafe_manifest_path_before_file_access() {
let root = test_temp_dir("verify_rejects_unsafe_path");
fs::create_dir_all(&root).expect("create fixture dir");
fs::write(root.join("artifact.txt"), b"artifact bytes").expect("write artifact");
let mut manifest =
write_archive_manifest(&root, "github", "owner/repo", true).expect("write manifest");
manifest.files[0].path = "../artifact.txt".to_string();
let manifest_json = serde_json::to_string_pretty(&manifest).expect("serialize manifest");
fs::write(
root.join(THESA_METADATA_DIR).join("manifest.json"),
manifest_json,
)
.expect("overwrite manifest with unsafe path");
let report = verify_archive(&root);
assert!(!report.ok);
assert!(!report.path_safety_valid);
assert!(
report
.errors
.iter()
.any(|error| error.contains("unsafe manifest path"))
);
fs::remove_dir_all(root).expect("cleanup temp dir");
}
fn test_temp_dir(name: &str) -> PathBuf {
let nonce = SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("system clock")
.as_nanos();
std::env::temp_dir().join(format!("thesa-{name}-{nonce}"))
}
}