use std::collections::BTreeSet;
use std::fs::{self, remove_dir_all, remove_file};
use std::io::{self, IsTerminal, Write};
use std::path::{Path, PathBuf};
use std::process::{Child, Command, Stdio};
use std::sync::mpsc;
use std::thread;
use std::time::{Duration, Instant};
use aisling::{
Color as AislingColor, EffectKind, Gradient, GradientDirection, Loader, LoaderConfig,
LoaderKind, LoaderProgress,
};
use clap::{Parser, ValueEnum};
use crossterm::event::{self, Event, KeyCode, KeyEventKind};
use reqwest::StatusCode;
use reqwest::blocking::{Client, Response};
use scrin::{
Color as TuiColor, Frame, Rect, Terminal, TerminalOptions,
core::buffer::Buffer,
effects::{EffectPlayer, LoaderPlayer},
layout::{Constraint, Direction, Layout},
style::{Modifier, Style},
widgets::{
Alignment, Block, Clear, Line, List, ListItem, ListState, Paragraph, Span, Text, Widget,
WrapMode, block::BorderStyle,
},
};
use serde::Deserialize;
use thiserror::Error;
const GITHUB_API: &str = "https://api.github.com";
const HF_BASE: &str = "https://huggingface.co";
const HF_MIRROR_BASE: &str = "https://hf-mirror.com";
const OLLAMA_LOCAL: &str = "http://localhost:11434";
const CIVITAI_API: &str = "https://civitai.com/api/v1";
const DEFAULT_PAGE_SIZE: usize = 100;
const BATCH_SIZE_CHOICES: [usize; 5] = [1, 2, 3, 6, 8];
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum TuiScreen {
Mode,
Provider,
Target,
HfToken,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
enum Mode {
Repos,
Models,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
enum ModelProvider {
Hf,
Ollama,
Civitai,
}
impl ModelProvider {
fn label(self) -> &'static str {
match self {
ModelProvider::Hf => "Hugging Face",
ModelProvider::Ollama => "Ollama",
ModelProvider::Civitai => "CivitAI",
}
}
fn short_label(self) -> &'static str {
match self {
ModelProvider::Hf => "HF",
ModelProvider::Ollama => "OLM",
ModelProvider::Civitai => "CIV",
}
}
}
impl std::fmt::Display for ModelProvider {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
ModelProvider::Hf => write!(f, "hf"),
ModelProvider::Ollama => write!(f, "ollama"),
ModelProvider::Civitai => write!(f, "civitai"),
}
}
}
impl std::fmt::Display for Mode {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Mode::Repos => write!(f, "repos"),
Mode::Models => write!(f, "models"),
}
}
}
#[derive(Parser, Clone)]
#[command(
name = "thesa",
version,
about = "A compendium to archive GitHub repositories and models"
)]
struct Cli {
target: Option<String>,
#[arg(short = 'o', long = "output", default_value = "./archives")]
output: PathBuf,
#[arg(short = 't', long = "token")]
token: Option<String>,
#[arg(long = "hf-token")]
hf_token: Option<String>,
#[arg(long = "civitai-token")]
civitai_token: Option<String>,
#[arg(long = "depth")]
depth: Option<u32>,
#[arg(short = 'c', long = "concurrency", default_value_t = 4)]
concurrency: usize,
#[arg(long = "skip-existing")]
skip_existing: bool,
#[arg(long = "filter")]
filter: Option<String>,
#[arg(long = "dry-run")]
dry_run: bool,
#[arg(short = 'T', long = "tui")]
tui: bool,
#[arg(long = "mode", value_enum, default_value_t = Mode::Repos)]
mode: Mode,
#[arg(long = "provider", value_enum, default_value_t = ModelProvider::Hf)]
provider: ModelProvider,
#[arg(long = "hf-mirror")]
hf_mirror: bool,
}
#[derive(Debug, Clone)]
enum Target {
Owner(String),
Repo { owner: String, repo: String },
}
#[derive(Debug, Clone)]
struct RepoItem {
name: String,
full_name: String,
clone_url: String,
}
#[derive(Debug, Deserialize)]
struct RepoResponse {
name: String,
#[serde(rename = "full_name")]
full_name: String,
#[serde(rename = "clone_url")]
clone_url: String,
private: bool,
}
#[derive(Debug, Clone)]
#[allow(dead_code)]
struct ModelItem {
id: String,
downloads: u64,
pipeline_tag: Option<String>,
last_modified: Option<String>,
provider: ModelProvider,
download_url: Option<String>,
size: Option<u64>,
}
#[derive(Debug, Deserialize)]
struct HfModelResponse {
#[serde(rename = "modelId", default)]
model_id: Option<String>,
#[serde(default)]
id: String,
#[serde(default)]
downloads: u64,
#[serde(rename = "pipeline_tag", default)]
pipeline_tag: Option<String>,
#[serde(rename = "lastModified", default)]
last_modified: Option<String>,
}
#[derive(Debug, Deserialize)]
struct OllamaModelResponse {
models: Vec<OllamaModel>,
}
#[derive(Debug, Deserialize)]
struct OllamaModel {
name: String,
#[serde(default)]
size: u64,
#[serde(default)]
modified_at: String,
}
#[derive(Debug, Deserialize)]
struct CivitaiModelResponse {
items: Vec<CivitaiModel>,
}
#[derive(Debug, Deserialize)]
#[allow(dead_code)]
struct CivitaiModel {
id: u64,
name: String,
#[serde(default)]
downloads: u64,
#[serde(rename = "type", default)]
model_type: Option<String>,
#[serde(default)]
description: Option<String>,
model_versions: Option<Vec<CivitaiModelVersion>>,
}
#[derive(Debug, Deserialize)]
#[allow(dead_code)]
struct CivitaiModelVersion {
id: u64,
name: Option<String>,
#[serde(default)]
download_url: Option<String>,
#[serde(default)]
size: Option<u64>,
}
#[derive(Debug, Clone)]
enum CloneStatus {
Cloned,
Skipped,
}
#[derive(Debug, Clone, Copy)]
enum BatchState {
Pending,
Running,
Done,
Skipped,
Failed,
}
#[derive(Debug)]
enum CloneAction {
Skipped,
Running(Child),
}
#[derive(Debug, Default)]
struct CloneSummary {
cloned: usize,
skipped: usize,
failed: Vec<String>,
}
#[derive(Error, Debug)]
#[allow(dead_code)]
enum GitArchiverError {
#[error("invalid target '{0}'")]
InvalidTarget(String),
#[error("GitHub API returned {status}: {body}")]
ApiError { status: u16, body: String },
#[error("GitHub entity '{0}' was not found")]
NotFound(String),
#[error("repo '{0}' is private, only public repos can be cloned")]
PrivateRepo(String),
#[error("output path '{0}' is not a directory")]
OutputNotDirectory(String),
#[error("'git' executable not found in PATH")]
MissingGit,
#[error("clone failed for '{repo}': {message}")]
CloneFailed { repo: String, message: String },
#[error(
"Hugging Face API returned {status}: {body}\n hint: set HF_TOKEN env or pass --hf-token to authenticate (https://huggingface.co/settings/tokens)"
)]
HfApiError { status: u16, body: String },
#[error("Hugging Face model '{0}' was not found")]
HfNotFound(String),
#[error("Hugging Face download failed for '{model}': {message}")]
HfDownloadFailed { model: String, message: String },
#[error("Ollama API returned {status}: {body}")]
OllamaApiError { status: u16, body: String },
#[error("Ollama model '{0}' was not found")]
OllamaNotFound(String),
#[error("Ollama download failed for '{model}': {message}")]
OllamaDownloadFailed { model: String, message: String },
#[error("CivitAI API returned {status}: {body}")]
CivitaiApiError { status: u16, body: String },
#[error("CivitAI model '{0}' was not found")]
CivitaiNotFound(String),
#[error("CivitAI download failed for '{model}': {message}")]
CivitaiDownloadFailed { model: String, message: String },
#[error("'ollama' executable not found in PATH")]
MissingOllama,
#[error("{0}")]
Message(String),
#[error("I/O error: {0}")]
Io(#[from] io::Error),
#[error("request error: {0}")]
Request(#[from] reqwest::Error),
}
type Result<T> = std::result::Result<T, GitArchiverError>;
struct TerminalSession {
terminal: Terminal,
}
impl TerminalSession {
fn new() -> Result<Self> {
let mut terminal = Terminal::init_with(TerminalOptions {
mouse_capture: true,
..TerminalOptions::default()
})?;
terminal.clear();
Ok(Self { terminal })
}
}
impl Drop for TerminalSession {
fn drop(&mut self) {
let _ = self.terminal.restore();
}
}
fn draw_full_frame<F>(terminal: &mut Terminal, f: F) -> io::Result<()>
where
F: FnOnce(&mut Frame<'_>),
{
terminal.draw(f)?;
let frame = terminal.back_buffer().to_ansi_string();
let mut stdout = io::stdout();
write!(stdout, "\x1b[H{frame}")?;
stdout.flush()?;
Ok(())
}
trait FrameRenderExt {
fn render_widget<W: Widget>(&mut self, widget: W, area: Rect);
fn render_block_widget<'item, W: Widget>(&mut self, area: Rect, widget: W, block: Block<'item>);
fn render_block_stateful_list<'item>(
&mut self,
area: Rect,
list: &List<'item>,
state: &mut ListState,
block: Block<'item>,
);
}
impl FrameRenderExt for Frame<'_> {
fn render_widget<W: Widget>(&mut self, widget: W, area: Rect) {
widget.render(self.buffer(), area);
}
fn render_block_widget<'item, W: Widget>(
&mut self,
area: Rect,
widget: W,
block: Block<'item>,
) {
block.render(self.buffer(), area);
let inner = block.inner(area);
if inner.width > 0 && inner.height > 0 {
self.buffer()
.fill(inner, ' ', theme_white(), Some(theme_surface()));
widget.render(self.buffer(), inner);
normalize_bg(self.buffer(), inner, theme_surface());
}
}
fn render_block_stateful_list<'b>(
&mut self,
area: Rect,
list: &List<'b>,
state: &mut ListState,
block: Block<'b>,
) {
block.render(self.buffer(), area);
let inner = block.inner(area);
if inner.width > 0 && inner.height > 0 {
self.buffer()
.fill(inner, ' ', theme_white(), Some(theme_surface()));
list.render_stateful(self.buffer(), inner, state);
normalize_bg(self.buffer(), inner, theme_surface());
}
}
}
#[derive(Debug, Default)]
struct UiState {
show_details: bool,
abort_requested: bool,
panel_lines: usize,
}
fn main() {
if let Err(err) = run() {
eprintln!("error: {err}");
std::process::exit(1);
}
}
fn run() -> Result<()> {
let mut args = Cli::parse();
if args.token.is_none() {
args.token = std::env::var("GITHUB_TOKEN").ok();
}
if args.hf_token.is_none() {
args.hf_token = std::env::var("HF_TOKEN").ok();
}
if args.civitai_token.is_none() {
args.civitai_token = std::env::var("CIVITAI_TOKEN").ok();
}
let interactive = args.tui || args.target.is_none();
if args.tui && !io::stdout().is_terminal() {
return Err(GitArchiverError::Message(
"--tui requires an interactive terminal".to_string(),
));
}
let mut terminal_session = if interactive && io::stdout().is_terminal() {
Some(TerminalSession::new()?)
} else {
None
};
if interactive {
let mut screen = TuiScreen::Mode;
loop {
match screen {
TuiScreen::Mode => {
if let Some(session) = terminal_session.as_mut() {
match prompt_mode_tui(&mut session.terminal)? {
Some(mode) => {
args.mode = mode;
if mode == Mode::Models {
screen = TuiScreen::Provider;
} else {
screen = TuiScreen::Target;
}
}
None => return Ok(()),
}
} else {
args.mode = prompt_mode()?;
if args.mode == Mode::Models {
screen = TuiScreen::Provider;
} else {
screen = TuiScreen::Target;
}
}
}
TuiScreen::Provider => {
if let Some(session) = terminal_session.as_mut() {
match prompt_model_provider_tui(&mut session.terminal, args.provider)? {
Some(provider) => {
args.provider = provider;
screen = TuiScreen::Target;
}
None => screen = TuiScreen::Mode,
}
} else {
args.provider = prompt_model_provider(args.provider)?;
screen = TuiScreen::Target;
}
}
TuiScreen::Target => {
if args.target.is_some() {
screen = TuiScreen::HfToken;
continue;
}
if let Some(session) = terminal_session.as_mut() {
match prompt_target_tui_inner(
&mut session.terminal,
args.mode,
Some(args.provider),
)? {
Some(target) => {
args.target = Some(target);
screen = TuiScreen::HfToken;
}
None => {
if args.mode == Mode::Models {
screen = TuiScreen::Provider;
} else {
screen = TuiScreen::Mode;
}
}
}
} else {
args.target = Some(prompt_target(args.mode, Some(args.provider))?);
screen = TuiScreen::HfToken;
}
}
TuiScreen::HfToken => {
if args.mode == Mode::Models
&& args.hf_token.is_none()
&& args.provider == ModelProvider::Hf
{
if let Some(session) = terminal_session.as_mut() {
match prompt_hf_token_tui(&mut session.terminal)? {
Some(token) => args.hf_token = Some(token),
None => {
screen = TuiScreen::Target;
continue;
}
}
} else if io::stdout().is_terminal() {
args.hf_token = prompt_hf_token()?;
}
}
break;
}
}
}
}
let client = Client::builder()
.user_agent("thesa")
.build()
.map_err(GitArchiverError::Request)?;
match args.mode {
Mode::Repos => run_repos_mode(&mut args, &client, &mut terminal_session),
Mode::Models => run_models_mode(&mut args, &client, &mut terminal_session),
}
}
fn run_repos_mode(
args: &mut Cli,
client: &Client,
terminal_session: &mut Option<TerminalSession>,
) -> Result<()> {
let target = parse_target(args.target.as_deref().unwrap_or_default())?;
if args.output == PathBuf::from("./archives") {
args.output = default_output_path(&target);
}
let output = prepare_output_dir(&args.output)?;
let mut repos = if let Some(session) = terminal_session.as_mut() {
fetch_repositories_tui(
&mut session.terminal,
client,
&target,
args.token.as_deref(),
)?
} else {
fetch_repositories_for_target(client, &target, args.token.as_deref())?
};
let interactive = args.tui || args.target.is_some();
if interactive {
repos = match &target {
Target::Owner(_) if terminal_session.is_some() => {
let session = terminal_session.as_mut().expect("checked above");
pick_repositories_tui_inner(&mut session.terminal, &repos, &mut args.concurrency)?
}
Target::Owner(_) => pick_repositories_interactively(&repos)?,
Target::Repo { .. } => repos,
};
} else if let Some(pattern) = args.filter.as_deref() {
repos.retain(|repo| repo.name.contains(pattern));
}
if repos.is_empty() {
*terminal_session = None;
println!("No repositories matched your request.");
return Ok(());
}
if args.dry_run {
*terminal_session = None;
println!("Dry run: {} repositories would be processed", repos.len());
for repo in &repos {
println!(" - {}", repo.full_name);
}
return Ok(());
}
let mut ui_state = UiState {
show_details: true,
..UiState::default()
};
let summary = if let Some(session) = terminal_session.as_mut() {
clone_repositories_scrin_inner(&mut session.terminal, &repos, &output, args, &mut ui_state)?
} else if io::stdout().is_terminal() {
clone_repositories_scrin(&repos, &output, args, &mut ui_state)?
} else {
let (ui_tx, ui_rx) = mpsc::channel::<char>();
let _ui_thread = thread::spawn(move || {
let stdin = io::stdin();
loop {
let mut line = String::new();
if stdin.read_line(&mut line).is_err() || line.is_empty() {
break;
}
let mut chars = line.chars().filter(|c| !c.is_whitespace());
if let Some(command) = chars.next() {
if ui_tx.send(command).is_err() {
break;
}
}
}
});
if args.concurrency > 1 {
clone_repositories_batched(&repos, &output, args, &ui_rx, &mut ui_state)?
} else {
clone_repositories_serial(&repos, &output, args, &ui_rx, &mut ui_state)?
}
};
*terminal_session = None;
if ui_state.abort_requested {
println!("Aborted by user input");
}
println!(
"\nCompleted: {} cloned, {} skipped, {} failed",
summary.cloned,
summary.skipped,
summary.failed.len()
);
if !summary.failed.is_empty() {
eprintln!("Failures:");
for item in summary.failed {
eprintln!(" - {item}");
}
return Err(GitArchiverError::Message(
"some repositories failed".to_string(),
));
}
Ok(())
}
fn run_models_mode(
args: &mut Cli,
client: &Client,
terminal_session: &mut Option<TerminalSession>,
) -> Result<()> {
let target =
parse_model_target_for_provider(args.target.as_deref().unwrap_or_default(), args.provider)?;
if args.output == PathBuf::from("./archives") {
args.output = PathBuf::from("./archives").join("models");
}
let output = prepare_output_dir(&args.output)?;
let models = if let Some(session) = terminal_session.as_mut() {
fetch_models_tui(&mut session.terminal, client, &target, args)?
} else {
fetch_models_for_target(client, &target, args)?
};
let picked = if let Some(session) = terminal_session.as_mut() {
pick_models_tui_inner(&mut session.terminal, &models, &mut args.concurrency)?
} else if io::stdout().is_terminal() {
pick_models_interactively(&models)?
} else {
models
};
if picked.is_empty() {
*terminal_session = None;
println!("No models matched your request.");
return Ok(());
}
if args.dry_run {
*terminal_session = None;
println!("Dry run: {} models would be downloaded", picked.len());
for model in &picked {
println!(
" - [{}] {} (downloads: {}, size: {})",
model.provider.short_label(),
model.id,
format_downloads(model.downloads),
format_model_size(model.size),
);
}
return Ok(());
}
let mut ui_state = UiState {
show_details: true,
..UiState::default()
};
let summary = if let Some(session) = terminal_session.as_mut() {
download_models_scrin_inner(&mut session.terminal, &picked, &output, args, &mut ui_state)?
} else if io::stdout().is_terminal() {
download_models_scrin(&picked, &output, args, &mut ui_state)?
} else {
download_models_serial(&picked, &output, args, &mut ui_state)?
};
*terminal_session = None;
if ui_state.abort_requested {
println!("Aborted by user input");
}
println!(
"\nCompleted: {} downloaded, {} skipped, {} failed",
summary.cloned,
summary.skipped,
summary.failed.len()
);
if !summary.failed.is_empty() {
eprintln!("Failures:");
for item in summary.failed {
eprintln!(" - {item}");
}
return Err(GitArchiverError::Message(
"some model downloads failed".to_string(),
));
}
Ok(())
}
#[derive(Debug, Clone)]
enum ModelTarget {
Org(String),
Model(String),
Top,
Latest,
}
fn parse_model_target(input: &str) -> Result<ModelTarget> {
let cleaned = input.trim();
if cleaned.is_empty() {
return Err(GitArchiverError::InvalidTarget(
"empty model target".to_string(),
));
}
if let Some(path) = cleaned.strip_prefix("https://huggingface.co/") {
return parse_model_path(path);
}
if let Some(path) = cleaned.strip_prefix("http://huggingface.co/") {
return parse_model_path(path);
}
let parts: Vec<&str> = cleaned.split('/').filter(|p| !p.is_empty()).collect();
match parts.len() {
0 => Err(GitArchiverError::InvalidTarget(cleaned.to_string())),
1 => Ok(ModelTarget::Org(parts[0].to_string())),
_ => Ok(ModelTarget::Model(parts[0].to_string() + "/" + parts[1])),
}
}
fn parse_model_target_for_provider(input: &str, provider: ModelProvider) -> Result<ModelTarget> {
let normalized = input.trim();
let canonical = normalized.to_lowercase();
match canonical.as_str() {
"top" | "trending" | "popular" => return Ok(ModelTarget::Top),
"latest" | "newest" | "new" => return Ok(ModelTarget::Latest),
_ => {}
}
if normalized.is_empty() {
return Err(GitArchiverError::InvalidTarget(
"empty model target".to_string(),
));
}
if let ModelProvider::Civitai = provider {
if let Some(path) = normalized
.strip_prefix("https://civitai.com/models/")
.or_else(|| normalized.strip_prefix("http://civitai.com/models/"))
{
let id = path
.split(|c| c == '/' || c == '?' || c == '#')
.next()
.unwrap_or("")
.trim();
if !id.is_empty() && id.chars().all(|c| c.is_ascii_digit()) {
return Ok(ModelTarget::Model(id.to_string()));
}
}
if normalized.chars().all(|c| c.is_ascii_digit()) {
return Ok(ModelTarget::Model(normalized.to_string()));
}
}
let model_target = parse_model_target(normalized)?;
match provider {
ModelProvider::Civitai => match model_target {
ModelTarget::Model(model) if model.chars().all(|c| c.is_ascii_digit()) => {
Ok(ModelTarget::Model(model))
}
other => Ok(other),
},
_ => Ok(model_target),
}
}
fn parse_model_path(path: &str) -> Result<ModelTarget> {
let parts: Vec<&str> = path
.trim_matches('/')
.split('/')
.filter(|p| !p.is_empty())
.collect();
match parts.len() {
0 => Err(GitArchiverError::InvalidTarget(path.to_string())),
1 => Ok(ModelTarget::Org(parts[0].to_string())),
_ => Ok(ModelTarget::Model(parts[0].to_string() + "/" + parts[1])),
}
}
fn hf_auth_header(token: Option<&str>) -> Option<String> {
token.map(|t| format!("Bearer {t}"))
}
fn hf_api_base(mirror: bool) -> &'static str {
if mirror { HF_MIRROR_BASE } else { HF_BASE }
}
fn list_models_for_org(
client: &Client,
org: &str,
token: Option<&str>,
mirror: bool,
) -> Result<Vec<ModelItem>> {
list_hf_models_with_sort(client, Some(org), "downloads", -1, token, mirror)
}
fn list_hf_top_models(
client: &Client,
token: Option<&str>,
mirror: bool,
) -> Result<Vec<ModelItem>> {
list_hf_models_with_sort(client, None, "downloads", -1, token, mirror)
}
fn list_hf_latest_models(
client: &Client,
token: Option<&str>,
mirror: bool,
) -> Result<Vec<ModelItem>> {
list_hf_models_with_sort(client, None, "lastModified", -1, token, mirror)
}
fn list_hf_models_with_sort(
client: &Client,
org: Option<&str>,
sort: &str,
direction: i8,
token: Option<&str>,
mirror: bool,
) -> Result<Vec<ModelItem>> {
let base = hf_api_base(mirror);
let mut page = 1usize;
let mut models = Vec::new();
loop {
let mut url = format!(
"{}/api/models?sort={}&direction={}&limit={}&page={}",
base,
urlencoding::encode(sort),
direction,
DEFAULT_PAGE_SIZE,
page
);
if let Some(org) = org {
url.push_str("&author=");
url.push_str(&urlencoding::encode(org));
}
let mut request = client.get(&url);
if let Some(auth) = hf_auth_header(token) {
request = request.header("Authorization", auth);
}
let response = request.send()?;
let status = response.status();
if status == StatusCode::NOT_FOUND {
if let Some(org) = org {
return Err(GitArchiverError::HfNotFound(org.to_string()));
}
return Err(GitArchiverError::HfApiError {
status: status.as_u16(),
body: "empty response from Hugging Face listing".to_string(),
});
}
if !status.is_success() {
return Err(parse_hf_api_error(response));
}
let batch: Vec<HfModelResponse> = response.json()?;
let count = batch.len();
models.extend(batch.into_iter().map(|m| ModelItem {
id: if let Some(ref mid) = m.model_id {
mid.clone()
} else {
m.id
},
downloads: m.downloads,
pipeline_tag: m.pipeline_tag,
last_modified: m.last_modified,
provider: ModelProvider::Hf,
download_url: None,
size: None,
}));
if count < DEFAULT_PAGE_SIZE {
break;
}
page = page.saturating_add(1);
}
Ok(models)
}
fn fetch_single_model(
client: &Client,
model_id: &str,
token: Option<&str>,
mirror: bool,
) -> Result<ModelItem> {
let base = hf_api_base(mirror);
let url = format!("{}/api/models/{model_id}", base);
let mut request = client.get(&url);
if let Some(auth) = hf_auth_header(token) {
request = request.header("Authorization", auth);
}
let response = request.send()?;
if response.status() == StatusCode::NOT_FOUND {
return Err(GitArchiverError::HfNotFound(model_id.to_string()));
}
if !response.status().is_success() {
return Err(parse_hf_api_error(response));
}
let data: HfModelResponse = response.json()?;
Ok(ModelItem {
id: if let Some(ref mid) = data.model_id {
mid.clone()
} else {
data.id
},
downloads: data.downloads,
pipeline_tag: data.pipeline_tag,
last_modified: data.last_modified,
provider: ModelProvider::Hf,
download_url: None,
size: None,
})
}
fn fetch_models_for_target(
client: &Client,
target: &ModelTarget,
args: &Cli,
) -> Result<Vec<ModelItem>> {
match args.provider {
ModelProvider::Hf => match target {
ModelTarget::Org(org) => {
list_models_for_org(client, org, args.hf_token.as_deref(), args.hf_mirror)
}
ModelTarget::Model(id) => Ok(vec![fetch_single_model(
client,
id,
args.hf_token.as_deref(),
args.hf_mirror,
)?]),
ModelTarget::Top => {
list_hf_top_models(client, args.hf_token.as_deref(), args.hf_mirror)
}
ModelTarget::Latest => {
list_hf_latest_models(client, args.hf_token.as_deref(), args.hf_mirror)
}
},
ModelProvider::Ollama => match target {
ModelTarget::Org(org) => list_ollama_models(client, org),
ModelTarget::Model(id) => Ok(vec![fetch_single_ollama_model(client, id)?]),
ModelTarget::Top => list_ollama_top_models(client),
ModelTarget::Latest => list_ollama_latest_models(client),
},
ModelProvider::Civitai => match target {
ModelTarget::Org(org) if org.chars().all(|c| c.is_ascii_digit()) => {
if let Ok(model) =
fetch_single_civitai_model(client, &org, args.civitai_token.as_deref())
{
return Ok(vec![model]);
}
list_civitai_models(
client,
Some(&org),
"Most+Downloaded",
args.civitai_token.as_deref(),
)
}
ModelTarget::Org(org) => list_civitai_models(
client,
Some(org),
"Most+Downloaded",
args.civitai_token.as_deref(),
),
ModelTarget::Model(id) => Ok(vec![fetch_single_civitai_model(
client,
id,
args.civitai_token.as_deref(),
)?]),
ModelTarget::Top => list_civitai_models(
client,
None,
"Most+Downloaded",
args.civitai_token.as_deref(),
),
ModelTarget::Latest => {
list_civitai_models(client, None, "Newest", args.civitai_token.as_deref())
}
},
}
}
fn list_ollama_models(client: &Client, org: &str) -> Result<Vec<ModelItem>> {
list_ollama_models_by_query(client, Some(org), true)
}
fn list_ollama_top_models(client: &Client) -> Result<Vec<ModelItem>> {
let mut models = list_ollama_models_by_query(client, None, false)?;
models.sort_by(|a, b| b.size.cmp(&a.size));
Ok(models)
}
fn list_ollama_latest_models(client: &Client) -> Result<Vec<ModelItem>> {
let mut models = list_ollama_models_by_query(client, None, false)?;
models.sort_by(|a, b| {
let b_modified = b.last_modified.as_deref().unwrap_or("");
let a_modified = a.last_modified.as_deref().unwrap_or("");
b_modified.cmp(a_modified)
});
Ok(models)
}
fn list_ollama_models_by_query(
client: &Client,
query: Option<&str>,
fallback_on_empty: bool,
) -> Result<Vec<ModelItem>> {
let local_url = format!("{}/api/tags", OLLAMA_LOCAL);
let query = query.map(str::to_lowercase).unwrap_or_default();
if let Ok(resp) = client
.get(&local_url)
.timeout(Duration::from_secs(2))
.send()
{
if resp.status().is_success() {
if let Ok(data) = resp.json::<OllamaModelResponse>() {
let models = data
.models
.into_iter()
.filter(|m| query.is_empty() || m.name.to_lowercase().contains(&query))
.map(|m| ModelItem {
id: m.name.clone(),
downloads: 0,
pipeline_tag: None,
last_modified: Some(m.modified_at),
provider: ModelProvider::Ollama,
download_url: None,
size: Some(m.size),
})
.collect::<Vec<_>>();
if !models.is_empty() {
return Ok(models);
}
}
}
}
if fallback_on_empty && !query.is_empty() {
return Ok(vec![ModelItem {
id: query.to_string(),
downloads: 0,
pipeline_tag: None,
last_modified: None,
provider: ModelProvider::Ollama,
download_url: None,
size: None,
}]);
}
Ok(Vec::new())
}
fn fetch_single_ollama_model(_client: &Client, model_id: &str) -> Result<ModelItem> {
Ok(ModelItem {
id: model_id.to_string(),
downloads: 0,
pipeline_tag: None,
last_modified: None,
provider: ModelProvider::Ollama,
download_url: None,
size: None,
})
}
fn list_civitai_models(
client: &Client,
query: Option<&str>,
sort: &str,
token: Option<&str>,
) -> Result<Vec<ModelItem>> {
let mut url = format!(
"{}/models?limit={}&sort={}",
CIVITAI_API, DEFAULT_PAGE_SIZE, sort
);
if let Some(query) = query {
if !query.is_empty() {
url.push_str("&query=");
url.push_str(&urlencoding::encode(query));
}
}
let mut request = client.get(&url);
if let Some(token) = token {
request = request.header("Authorization", format!("Bearer {token}"));
}
let response = request.send()?;
let status = response.status();
if !status.is_success() {
let body = response
.text()
.unwrap_or_else(|_| "<unable to read body>".to_string());
return Err(GitArchiverError::CivitaiApiError {
status: status.as_u16(),
body,
});
}
let data: CivitaiModelResponse = response.json()?;
let models = data
.items
.into_iter()
.map(|m| {
let (download_url, size) = m
.model_versions
.as_ref()
.and_then(|v| v.first())
.map(|v| (v.download_url.clone(), v.size))
.unwrap_or((None, None));
ModelItem {
id: m.name.clone(),
downloads: m.downloads,
pipeline_tag: m.model_type,
last_modified: None,
provider: ModelProvider::Civitai,
download_url,
size,
}
})
.collect::<Vec<_>>();
if models.is_empty() {
return Err(GitArchiverError::CivitaiNotFound(
query.unwrap_or("civitai top models").to_string(),
));
}
Ok(models)
}
fn fetch_single_civitai_model(
client: &Client,
model_id: &str,
token: Option<&str>,
) -> Result<ModelItem> {
let numeric_id: u64 = model_id.parse().map_err(|_| {
GitArchiverError::CivitaiNotFound(format!(
"{model_id} (CivitAI requires numeric model ID, e.g. 827184)"
))
})?;
let url = format!("{}/models/{}", CIVITAI_API, numeric_id);
let mut request = client.get(&url);
if let Some(token) = token {
request = request.header("Authorization", format!("Bearer {token}"));
}
let response = request.send()?;
let status = response.status();
if status == StatusCode::NOT_FOUND {
return Err(GitArchiverError::CivitaiNotFound(model_id.to_string()));
}
if !status.is_success() {
let body = response
.text()
.unwrap_or_else(|_| "<unable to read body>".to_string());
return Err(GitArchiverError::CivitaiApiError {
status: status.as_u16(),
body,
});
}
let m: CivitaiModel = response.json()?;
let (download_url, size) = m
.model_versions
.as_ref()
.and_then(|v| v.first())
.map(|v| (v.download_url.clone(), v.size))
.unwrap_or((None, None));
Ok(ModelItem {
id: m.name.clone(),
downloads: m.downloads,
pipeline_tag: m.model_type,
last_modified: None,
provider: ModelProvider::Civitai,
download_url,
size,
})
}
fn parse_hf_api_error(response: Response) -> GitArchiverError {
let status = response.status();
let body = response
.text()
.unwrap_or_else(|_| "<unable to read body>".to_string());
GitArchiverError::HfApiError {
status: status.as_u16(),
body,
}
}
fn download_model(model: &ModelItem, output: &Path, args: &Cli) -> Result<CloneStatus> {
match model.provider {
ModelProvider::Hf => download_hf_model(model, output, args),
ModelProvider::Ollama => download_ollama_model(model, output, args),
ModelProvider::Civitai => download_civitai_model(model, output, args),
}
}
fn download_hf_model(model: &ModelItem, output: &Path, args: &Cli) -> Result<CloneStatus> {
let destination = output.join(&model.id.replace('/', "_"));
if destination.exists() {
if args.skip_existing {
return Ok(CloneStatus::Skipped);
}
if destination.is_dir() {
remove_dir_all(&destination)?;
} else {
remove_file(&destination)?;
}
}
let child = spawn_hf_clone(model, output, args)?;
let status = child.wait_with_output()?;
if status.status.success() {
Ok(CloneStatus::Cloned)
} else {
let code = status
.status
.code()
.map_or_else(|| "signal/unknown".to_string(), |c| c.to_string());
Err(GitArchiverError::HfDownloadFailed {
model: model.id.clone(),
message: format!("exit code {code}"),
})
}
}
fn download_ollama_model(model: &ModelItem, output: &Path, args: &Cli) -> Result<CloneStatus> {
let destination = output.join(&model.id.replace('/', "_").replace(':', "_"));
if destination.exists() {
if args.skip_existing {
return Ok(CloneStatus::Skipped);
}
if destination.is_dir() {
remove_dir_all(&destination)?;
} else {
remove_file(&destination)?;
}
}
fs::create_dir_all(&destination)?;
let command_args = vec!["pull".to_string(), model.id.clone()];
let child = match Command::new("ollama")
.current_dir(&destination)
.args(&command_args)
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
{
Ok(child) => child,
Err(error) if error.kind() == io::ErrorKind::NotFound => {
return Err(GitArchiverError::MissingOllama);
}
Err(error) => return Err(error.into()),
};
let status = child.wait_with_output()?;
if status.status.success() {
Ok(CloneStatus::Cloned)
} else {
let code = status
.status
.code()
.map_or_else(|| "signal/unknown".to_string(), |c| c.to_string());
Err(GitArchiverError::OllamaDownloadFailed {
model: model.id.clone(),
message: format!("exit code {code}"),
})
}
}
fn download_civitai_model(model: &ModelItem, output: &Path, args: &Cli) -> Result<CloneStatus> {
let dir_name = model.id.replace('/', "_");
let destination = output.join(&dir_name);
if destination.exists() {
if args.skip_existing {
return Ok(CloneStatus::Skipped);
}
if destination.is_dir() {
remove_dir_all(&destination)?;
} else {
remove_file(&destination)?;
}
}
let url =
model
.download_url
.as_deref()
.ok_or_else(|| GitArchiverError::CivitaiDownloadFailed {
model: model.id.clone(),
message: "no download URL available".to_string(),
})?;
let mut request = client_get().get(url);
if let Some(token) = args.civitai_token.as_deref() {
request = request.header("Authorization", format!("Bearer {token}"));
}
let response = request.send()?;
let status = response.status();
if !status.is_success() {
let body = response
.text()
.unwrap_or_else(|_| "<unable to read body>".to_string());
return Err(GitArchiverError::CivitaiDownloadFailed {
model: model.id.clone(),
message: format!("HTTP {status}: {body}"),
});
}
let bytes = response.bytes()?;
fs::create_dir_all(&destination)?;
let filename = url.rsplit('/').next().unwrap_or("model.safetensors");
let filepath = destination.join(filename);
fs::write(&filepath, &bytes)?;
Ok(CloneStatus::Cloned)
}
fn client_get() -> Client {
Client::builder()
.user_agent("thesa")
.build()
.unwrap_or_else(|_| Client::new())
}
fn spawn_hf_clone(model: &ModelItem, output: &Path, args: &Cli) -> Result<Child> {
let url = format!("https://huggingface.co/{}", model.id);
let dir_name = model.id.replace('/', "_");
let mut command_args = vec!["clone".to_string()];
if let Some(depth) = args.depth {
command_args.push("--depth".to_string());
command_args.push(depth.to_string());
}
command_args.push(url);
command_args.push(dir_name);
match Command::new("git")
.current_dir(output)
.args(&command_args)
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
{
Ok(child) => Ok(child),
Err(error) if error.kind() == io::ErrorKind::NotFound => Err(GitArchiverError::MissingGit),
Err(error) => Err(error.into()),
}
}
fn prompt_mode_tui(terminal: &mut Terminal) -> Result<Option<Mode>> {
let mut cursor = 0usize;
let mut frame_index = 0usize;
loop {
draw_full_frame(terminal, |frame| {
draw_mode_selector_tui(frame, cursor, frame_index);
})?;
frame_index = frame_index.wrapping_add(1);
if !event::poll(Duration::from_millis(50))? {
continue;
}
let Event::Key(key) = event::read()? else {
continue;
};
if key.kind != KeyEventKind::Press {
continue;
}
match key.code {
KeyCode::Esc => return Ok(None),
KeyCode::Up | KeyCode::Left => cursor = cursor.saturating_sub(1),
KeyCode::Down | KeyCode::Right => {
if cursor + 1 < 2 {
cursor += 1;
}
}
KeyCode::Enter => {
return Ok(Some(if cursor == 0 {
Mode::Repos
} else {
Mode::Models
}));
}
KeyCode::Char('r') => return Ok(Some(Mode::Repos)),
KeyCode::Char('m') => return Ok(Some(Mode::Models)),
_ => {}
}
}
}
fn prompt_model_provider_tui(
terminal: &mut Terminal,
current: ModelProvider,
) -> Result<Option<ModelProvider>> {
let mut cursor = model_provider_index(current);
let mut frame_index = 0usize;
loop {
draw_full_frame(terminal, |frame| {
draw_model_provider_selector_tui(frame, cursor, frame_index);
})?;
frame_index = frame_index.wrapping_add(1);
if !event::poll(Duration::from_millis(50))? {
continue;
}
let Event::Key(key) = event::read()? else {
continue;
};
if key.kind != KeyEventKind::Press {
continue;
}
match key.code {
KeyCode::Esc => return Ok(None),
KeyCode::Up | KeyCode::Left => cursor = cursor.saturating_sub(1),
KeyCode::Down | KeyCode::Right => {
if cursor + 1 < 3 {
cursor += 1;
}
}
KeyCode::Enter => return Ok(Some(model_provider_from_index(cursor))),
KeyCode::Char('h') | KeyCode::Char('1') => return Ok(Some(ModelProvider::Hf)),
KeyCode::Char('o') | KeyCode::Char('2') => return Ok(Some(ModelProvider::Ollama)),
KeyCode::Char('c') | KeyCode::Char('3') => return Ok(Some(ModelProvider::Civitai)),
_ => {}
}
}
}
fn model_provider_index(provider: ModelProvider) -> usize {
match provider {
ModelProvider::Hf => 0,
ModelProvider::Ollama => 1,
ModelProvider::Civitai => 2,
}
}
fn model_provider_from_index(index: usize) -> ModelProvider {
match index {
1 => ModelProvider::Ollama,
2 => ModelProvider::Civitai,
_ => ModelProvider::Hf,
}
}
fn prompt_target_tui_inner(
terminal: &mut Terminal,
mode: Mode,
provider: Option<ModelProvider>,
) -> Result<Option<String>> {
let mut input = String::new();
let mut frame_index = 0usize;
let mut status = match mode {
Mode::Repos => "enter GitHub owner or repo name".to_string(),
Mode::Models => match provider.unwrap_or(ModelProvider::Hf) {
ModelProvider::Hf => "enter Hugging Face org, org/model, top, or latest".to_string(),
ModelProvider::Ollama => "enter Ollama model name, top, or latest".to_string(),
ModelProvider::Civitai => "enter CivitAI model id, keyword, top, or latest".to_string(),
},
};
loop {
draw_full_frame(terminal, |frame| {
draw_target_tui(
frame,
&input,
&status,
frame_index,
mode,
if matches!(mode, Mode::Models) {
provider
} else {
None
},
);
})?;
frame_index = frame_index.wrapping_add(1);
if !event::poll(Duration::from_millis(50))? {
continue;
}
let Event::Key(key) = event::read()? else {
continue;
};
if key.kind != KeyEventKind::Press {
continue;
}
match key.code {
KeyCode::Esc if input.is_empty() => return Ok(None),
KeyCode::Esc => {
input.clear();
status = match mode {
Mode::Repos => "enter GitHub owner or repo name".to_string(),
Mode::Models => match provider.unwrap_or(ModelProvider::Hf) {
ModelProvider::Hf => {
"enter Hugging Face org, org/model, top, or latest".to_string()
}
ModelProvider::Ollama => {
"enter Ollama model name, top, or latest".to_string()
}
ModelProvider::Civitai => {
"enter CivitAI model id, keyword, top, or latest".to_string()
}
},
};
}
KeyCode::Enter => {
let target = input.trim().to_string();
if target.is_empty() {
status = "name is required".to_string();
continue;
}
let parse_result = match mode {
Mode::Repos => parse_target(&target).map(|_| target.clone()),
Mode::Models => parse_model_target_for_provider(
&target,
provider.unwrap_or(ModelProvider::Hf),
)
.map(|_| target.clone()),
};
if parse_result.is_ok() {
return Ok(Some(target));
}
status = match mode {
Mode::Repos => "try octocat or octocat/Spoon-Knife".to_string(),
Mode::Models => match provider.unwrap_or(ModelProvider::Hf) {
ModelProvider::Hf => "try meta-llama or meta-llama/Llama-2-7b".to_string(),
ModelProvider::Ollama => "try llama3, latest, or qwen2.5:7b".to_string(),
ModelProvider::Civitai => {
"try 827184 or top, realistic-vision-v6".to_string()
}
},
};
}
KeyCode::Backspace => {
input.pop();
}
KeyCode::Char(ch) => {
input.push(ch);
}
_ => {}
}
}
}
fn pick_models_tui_inner(
terminal: &mut Terminal,
models: &[ModelItem],
batch_size: &mut usize,
) -> Result<Vec<ModelItem>> {
if models.is_empty() {
return Ok(vec![]);
}
let mut query = String::new();
let mut selected = BTreeSet::new();
let mut cursor = 0usize;
let mut frame_index = 0usize;
let mut batch_overlay = false;
let mut status =
"Space checks models; Enter downloads checked models, or visible if none checked"
.to_string();
loop {
let filtered = filtered_model_indices(models, &query);
if cursor >= filtered.len() {
cursor = filtered.len().saturating_sub(1);
}
let page_step = picker_page_step(terminal.size().height);
draw_full_frame(terminal, |frame| {
draw_model_picker_tui(
frame,
models,
&filtered,
&selected,
cursor,
&query,
&status,
frame_index,
*batch_size,
batch_overlay,
);
})?;
frame_index = frame_index.wrapping_add(1);
if !event::poll(Duration::from_millis(50))? {
continue;
}
let Event::Key(key) = event::read()? else {
continue;
};
if key.kind != KeyEventKind::Press {
continue;
}
if batch_overlay {
match key.code {
KeyCode::Esc | KeyCode::Char('b') | KeyCode::Enter => {
batch_overlay = false;
status = format!("batch size set to {}; Enter downloads models", *batch_size);
}
KeyCode::Char(ch) => {
if let Some(value) = batch_size_from_key(ch) {
*batch_size = value;
batch_overlay = false;
status = format!("batch size set to {value}; Enter downloads models");
}
}
KeyCode::Left | KeyCode::Up => {
*batch_size = previous_batch_size(*batch_size);
status = format!("batch size set to {}; Enter accepts", *batch_size);
}
KeyCode::Right | KeyCode::Down => {
*batch_size = next_batch_size(*batch_size);
status = format!("batch size set to {}; Enter accepts", *batch_size);
}
_ => {}
}
continue;
}
match key.code {
KeyCode::Esc => return Err(GitArchiverError::Message("aborted by user".to_string())),
KeyCode::Enter => {
let picked = if selected.is_empty() {
filtered
} else {
selected.iter().copied().collect::<Vec<_>>()
};
if picked.is_empty() {
status = "no models selected or visible".to_string();
continue;
}
return Ok(picked
.into_iter()
.map(|index| models[index].clone())
.collect());
}
KeyCode::Up => cursor = cursor.saturating_sub(1),
KeyCode::Down => {
if cursor + 1 < filtered.len() {
cursor += 1;
}
}
KeyCode::PageUp => {
let step = page_step.min(filtered.len().max(1));
cursor = cursor.saturating_sub(step);
status = if filtered.is_empty() {
"no models to move to".to_string()
} else {
format!("moved up by {step}")
};
}
KeyCode::PageDown => {
if filtered.is_empty() {
status = "no models to move to".to_string();
} else {
let step = page_step.min(filtered.len());
cursor = (cursor + step).min(filtered.len() - 1);
status = format!("moved down by {step}");
}
}
KeyCode::Home => {
if !filtered.is_empty() {
cursor = 0;
status = "jumped to first model".to_string();
} else {
status = "no models to move to".to_string();
}
}
KeyCode::End => {
if !filtered.is_empty() {
cursor = filtered.len() - 1;
status = "jumped to last model".to_string();
} else {
status = "no models to move to".to_string();
}
}
KeyCode::Char(' ') => {
if let Some(model_index) = filtered.get(cursor).copied() {
if !selected.insert(model_index) {
selected.remove(&model_index);
}
let checked = selected.len();
status = if checked == 0 {
"nothing checked; Enter downloads visible models".to_string()
} else {
format!("{checked} checked; Enter downloads checked models only")
};
}
}
KeyCode::Char('a') => {
for model_index in &filtered {
selected.insert(*model_index);
}
status = format!("checked {} visible models", filtered.len());
}
KeyCode::Char('c') => {
selected.clear();
query.clear();
cursor = 0;
status = "cleared filter and selection".to_string();
}
KeyCode::Char('b') => {
batch_overlay = true;
status = "choose batch size: 1 2 3 6 8".to_string();
}
KeyCode::Backspace => {
query.pop();
cursor = 0;
}
KeyCode::Char(ch) => {
query.push(ch);
cursor = 0;
}
_ => {}
}
}
}
fn filtered_model_indices(models: &[ModelItem], query: &str) -> Vec<usize> {
if query.trim().is_empty() {
return (0..models.len()).collect();
}
let query = query.to_lowercase();
models
.iter()
.enumerate()
.filter(|(_, m)| m.id.to_lowercase().contains(&query))
.map(|(index, _)| index)
.collect()
}
fn download_models_scrin(
models: &[ModelItem],
output: &Path,
args: &Cli,
ui_state: &mut UiState,
) -> Result<CloneSummary> {
let mut session = TerminalSession::new()?;
download_models_scrin_inner(&mut session.terminal, models, output, args, ui_state)
}
fn download_models_scrin_inner(
terminal: &mut Terminal,
models: &[ModelItem],
output: &Path,
args: &Cli,
ui_state: &mut UiState,
) -> Result<CloneSummary> {
let batch_size = args.concurrency.max(1);
let total = models.len();
let total_batches = total.div_ceil(batch_size);
let mut summary = CloneSummary::default();
let mut effect_kind = EffectKind::Waves;
let started_at = Instant::now();
let mut tick = 0usize;
let mut batch_index = 1;
while batch_index <= total {
let batch_end = (batch_index + batch_size - 1).min(total);
let batch = &models[batch_index - 1..batch_end];
let names: Vec<String> = batch.iter().map(|m| m.id.clone()).collect();
let mut state = vec![BatchState::Pending; batch.len()];
let (tx, rx) = mpsc::channel::<(usize, BatchEvent)>();
let mut handles = Vec::new();
for (offset, model) in batch.iter().enumerate() {
let model = model.clone();
let output = output.to_path_buf();
let args = args.clone();
let tx = tx.clone();
handles.push((
model.id.clone(),
thread::spawn(move || {
let started = tx.send((offset, BatchEvent::Started));
if started.is_err() {
return;
}
let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
download_model(&model, &output, &args)
}));
let event = match outcome {
Ok(Ok(CloneStatus::Cloned)) => BatchEvent::Result(BatchResult::Done),
Ok(Ok(CloneStatus::Skipped)) => BatchEvent::Result(BatchResult::Skipped),
Ok(Err(err)) => BatchEvent::Result(BatchResult::Error(err.to_string())),
Err(_) => BatchEvent::Result(BatchResult::Error(
"worker thread panicked while downloading".to_string(),
)),
};
let _ = tx.send((offset, event));
}),
));
}
drop(tx);
let mut completed = 0;
while completed < batch.len() {
while event::poll(Duration::from_millis(0))? {
if let Event::Key(key) = event::read()? {
match key.code {
KeyCode::Char('q') | KeyCode::Esc => ui_state.abort_requested = true,
KeyCode::Char('t') => ui_state.show_details = !ui_state.show_details,
KeyCode::Char('m') => effect_kind = EffectKind::Waves,
KeyCode::Char('b') => effect_kind = EffectKind::Burn,
KeyCode::Char('w') => effect_kind = EffectKind::Wipe,
_ => {}
}
}
}
while let Ok((offset, event)) = rx.try_recv() {
match event {
BatchEvent::Started => state[offset] = BatchState::Running,
BatchEvent::Result(BatchResult::Done) => {
state[offset] = BatchState::Done;
summary.cloned += 1;
completed += 1;
}
BatchEvent::Result(BatchResult::Skipped) => {
state[offset] = BatchState::Skipped;
summary.skipped += 1;
completed += 1;
}
BatchEvent::Result(BatchResult::Error(message)) => {
state[offset] = BatchState::Failed;
summary
.failed
.push(format!("{}: {}", names[offset], message));
completed += 1;
}
}
}
draw_full_frame(terminal, |frame| {
render_download_dashboard(
frame,
&names,
&state,
&summary,
batch_index,
batch_size,
total,
total_batches,
tick,
started_at.elapsed(),
effect_kind,
ui_state,
)
})?;
tick = tick.wrapping_add(1);
if completed < batch.len() {
thread::sleep(Duration::from_millis(80));
}
}
for (model_name, handle) in handles {
if handle.join().is_err() {
summary
.failed
.push(format!("{model_name}: worker thread panicked"));
}
}
if ui_state.abort_requested {
break;
}
batch_index = batch_end + 1;
}
Ok(summary)
}
fn download_models_serial(
models: &[ModelItem],
output: &Path,
args: &Cli,
ui_state: &mut UiState,
) -> Result<CloneSummary> {
let mut summary = CloneSummary::default();
let total = models.len();
for (index, model) in models.iter().enumerate() {
if ui_state.abort_requested {
break;
}
match download_model(model, output, args) {
Ok(CloneStatus::Cloned) => {
println!("[done] {} ({}/{})", model.id, index + 1, total);
summary.cloned += 1;
}
Ok(CloneStatus::Skipped) => {
println!("[skip] {} ({}/{})", model.id, index + 1, total);
summary.skipped += 1;
}
Err(err) => {
eprintln!("[fail] {} ({}/{}): {err}", model.id, index + 1, total);
summary.failed.push(format!("{}: {err}", model.id));
}
}
}
Ok(summary)
}
fn pick_repositories_interactively(repos: &[RepoItem]) -> Result<Vec<RepoItem>> {
if repos.is_empty() {
return Ok(vec![]);
}
let snapshot: Vec<RepoItem> = repos.to_vec();
loop {
println!("Found {} public repositories", snapshot.len());
let query = prompt_line("Search public repos (blank = all, q = quit): ")?;
if query.eq_ignore_ascii_case("q") {
return Err(GitArchiverError::Message("aborted by user".to_string()));
}
let filtered = if query.is_empty() {
snapshot.clone()
} else {
let normalized_query = query.to_lowercase();
snapshot
.iter()
.filter(|repo| repo.name.to_lowercase().contains(&normalized_query))
.cloned()
.collect::<Vec<_>>()
};
if filtered.is_empty() {
println!("No matches for '{query}'.");
continue;
}
for (index, repo) in filtered.iter().enumerate() {
println!("{:>3}. {}", index + 1, repo.full_name);
}
let selection = prompt_line("Clone [A]ll, specific indexes (1,3-5) or [q] to quit [a]: ")?;
if selection.is_empty() || selection.eq_ignore_ascii_case("a") {
return Ok(filtered);
}
if selection.eq_ignore_ascii_case("q") {
return Err(GitArchiverError::Message("aborted by user".to_string()));
}
match parse_indices(&selection, filtered.len()) {
Ok(indices) => {
let selected = indices
.into_iter()
.map(|idx| filtered[idx].clone())
.collect::<Vec<_>>();
return Ok(selected);
}
Err(message) => {
println!("{message}");
}
}
}
}
fn pick_models_interactively(models: &[ModelItem]) -> Result<Vec<ModelItem>> {
if models.is_empty() {
return Ok(vec![]);
}
let snapshot = models.to_vec();
loop {
println!("Found {} models", snapshot.len());
let query = prompt_line("Search models (blank = all, q = quit): ")?;
if query.eq_ignore_ascii_case("q") {
return Err(GitArchiverError::Message("aborted by user".to_string()));
}
let filtered = if query.is_empty() {
snapshot.clone()
} else {
let q = query.to_lowercase();
snapshot
.iter()
.filter(|m| m.id.to_lowercase().contains(&q))
.cloned()
.collect::<Vec<_>>()
};
if filtered.is_empty() {
println!("No matches for '{query}'.");
continue;
}
for (index, model) in filtered.iter().enumerate() {
println!("{:>3}. {} [dl: {}]", index + 1, model.id, model.downloads);
}
let selection =
prompt_line("Download [A]ll, specific indexes (1,3-5) or [q] to quit [a]: ")?;
if selection.is_empty() || selection.eq_ignore_ascii_case("a") {
return Ok(filtered);
}
if selection.eq_ignore_ascii_case("q") {
return Err(GitArchiverError::Message("aborted by user".to_string()));
}
match parse_indices(&selection, filtered.len()) {
Ok(indices) => {
return Ok(indices
.into_iter()
.map(|idx| filtered[idx].clone())
.collect());
}
Err(message) => {
println!("{message}");
}
}
}
}
fn draw_hf_token_tui(
frame: &mut Frame<'_>,
input: &str,
status: &str,
tick: usize,
show_plain: bool,
) {
let area = frame.area();
paint_surface(frame, area);
if area.width < 24 || area.height < 8 {
let line = if show_plain {
format!("HF token> {input}\n{status}")
} else {
format!("HF token> {}\n{status}", "*".repeat(input.len()))
};
frame.render_widget(
Paragraph::from_text(Text::new(vec![Line::styled(
&line,
Style::default().fg(theme_white()),
)])),
area,
);
return;
}
let layout = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(header_height(area)),
Constraint::Min(5),
Constraint::Length(2),
])
.split(area);
draw_thesa_header(frame, layout[0], tick, status, "hf auth");
let display_input = if show_plain {
input.to_string()
} else {
"*".repeat(input.len())
};
let body = Paragraph::from_text(Text::new(vec![
Line::new(vec![
Span::styled("HF token> ", prompt_style()),
Span::styled(&display_input, Style::default().fg(theme_white())),
]),
Line::raw(""),
Line::new(vec![
Span::styled("hint ", Style::default().fg(theme_dim())),
Span::styled(
"https://huggingface.co/settings/tokens",
Style::default().fg(theme_blue()),
),
]),
Line::raw(""),
Line::styled(
"Press Enter with blank to skip (will rate-limit).",
Style::default().fg(theme_dim()),
),
]))
.wrap(WrapMode::Character { trim: false });
frame.render_block_widget(
layout[1],
body,
knott_block(" Hugging Face Token ", theme_blue()),
);
draw_thesa_bottom_bar(frame, layout[2], status, &["Enter accept", "Esc back"]);
let input_area = panel_inner(layout[1]);
let cursor_x = input_area
.x
.saturating_add(10)
.saturating_add(input.chars().count() as u16)
.min(input_area.x + input_area.width.saturating_sub(1));
frame.set_cursor_position(cursor_x, input_area.y);
}
fn draw_mode_selector_tui(frame: &mut Frame<'_>, cursor: usize, tick: usize) {
let area = frame.area();
paint_surface(frame, area);
if area.width < 28 || area.height < 10 {
let line = format!("select mode: repos={} models={}", cursor == 0, cursor == 1);
frame.render_widget(
Paragraph::from_text(Text::new(vec![Line::styled(
&line,
Style::default().fg(theme_white()),
)])),
area,
);
return;
}
let layout = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(header_height(area)),
Constraint::Min(8),
Constraint::Length(2),
])
.split(area);
draw_thesa_header(frame, layout[0], tick, "select archive mode", "setup");
let modes = [
(
"repos",
"GitHub repositories (owner/org/repo)",
"r",
theme_blue(),
),
(
"models",
"Hugging Face, Ollama, CivitAI models",
"m",
theme_green(),
),
];
let items = modes
.iter()
.enumerate()
.map(|(i, (name, desc, key, color))| {
let selected = cursor == i;
let marker = if selected { ">>" } else { " " };
let style = if selected {
selected_style(tick)
} else {
Style::default().fg(theme_white())
};
ListItem::with_line(Line::new(vec![
Span::styled(
&format!(" {marker} "),
style.add_modifier(if selected {
Modifier::BOLD
} else {
Modifier::empty()
}),
),
Span::styled(
&format!("[{key}]"),
Style::default().fg(*color).add_modifier(Modifier::BOLD),
),
Span::raw(" "),
Span::styled(&name.to_uppercase(), style.add_modifier(Modifier::BOLD)),
Span::raw(" "),
Span::styled(desc, Style::default().fg(theme_dim())),
]))
})
.collect::<Vec<_>>();
let list = List::new(&items)
.with_highlight_style(selected_style(tick))
.with_highlight_symbol(">>");
let mut state = ListState::default();
state.select(Some(cursor));
frame.render_block_stateful_list(
layout[1],
&list,
&mut state,
knott_block(" Archive Mode ", theme_blue()),
);
draw_thesa_bottom_bar(
frame,
layout[2],
"Up/Down or r/m to select, Enter to confirm, Esc back/quit",
&[
"Up/Down move",
"r repos",
"m models",
"Enter select",
"Esc back",
],
);
}
fn draw_model_provider_selector_tui(frame: &mut Frame<'_>, cursor: usize, tick: usize) {
let area = frame.area();
paint_surface(frame, area);
if area.width < 24 || area.height < 8 {
frame.render_widget(
Paragraph::from_text(Text::new(vec![Line::styled(
"Model provider\nhf\nollama\ncivitai",
Style::default().fg(theme_white()),
)])),
area,
);
return;
}
let layout = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(header_height(area)),
Constraint::Min(8),
Constraint::Length(3),
])
.split(area);
draw_thesa_header(
frame,
layout[0],
tick,
"select model provider before choosing target",
"model setup",
);
let providers = [
(
"hugging face",
"remote model hub: orgs, org/model, top, latest",
"h/1",
ModelProvider::Hf,
),
(
"ollama",
"local Ollama library: installed names, top, latest",
"o/2",
ModelProvider::Ollama,
),
(
"civitai",
"CivitAI models: numeric IDs, keyword search, top, latest",
"c/3",
ModelProvider::Civitai,
),
];
let items = providers
.iter()
.enumerate()
.map(|(i, (name, desc, key, provider))| {
let selected = cursor == i;
let marker = if selected { ">>" } else { " " };
let style = if selected {
selected_style(tick)
} else {
Style::default().fg(theme_white())
};
ListItem::with_line(Line::new(vec![
Span::styled(&format!(" {marker} "), style),
Span::styled(
&format!("[{key}]"),
Style::default()
.fg(model_provider_color(*provider))
.add_modifier(Modifier::BOLD),
),
Span::raw(" "),
Span::styled(&name.to_uppercase(), style.add_modifier(Modifier::BOLD)),
Span::raw(" "),
Span::styled(desc, Style::default().fg(theme_dim())),
]))
})
.collect::<Vec<_>>();
let list = List::new(&items)
.with_highlight_style(selected_style(tick))
.with_highlight_symbol(">>");
let mut state = ListState::default();
state.select(Some(cursor));
frame.render_block_stateful_list(
layout[1],
&list,
&mut state,
knott_block(" Provider ", theme_green()),
);
draw_thesa_bottom_bar(
frame,
layout[2],
"Choose where model archives come from",
&[
"Up/Down move",
"h/o/c choose",
"1/2/3 choose",
"Enter continue",
"Esc back",
],
);
}
fn draw_target_tui(
frame: &mut Frame<'_>,
input: &str,
status: &str,
tick: usize,
mode: Mode,
provider: Option<ModelProvider>,
) {
let area = frame.area();
paint_surface(frame, area);
let prompt_label = match mode {
Mode::Repos => "repo/owner> ",
Mode::Models => "model target> ",
};
if area.width < 24 || area.height < 8 {
let line = format!("{prompt_label}{input}\n{status}");
frame.render_widget(
Paragraph::from_text(Text::new(vec![Line::styled(
&line,
Style::default().fg(theme_white()),
)])),
area,
);
return;
}
let layout = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(header_height(area)),
Constraint::Min(5),
Constraint::Length(2),
])
.split(area);
let model_provider = provider.unwrap_or(ModelProvider::Hf);
let mode_label = match mode {
Mode::Repos => "github repos",
Mode::Models => match model_provider {
ModelProvider::Hf => "huggingface models",
ModelProvider::Ollama => "ollama models",
ModelProvider::Civitai => "civitai models",
},
};
draw_thesa_header(frame, layout[0], tick, status, mode_label);
let examples = match mode {
Mode::Repos => vec![
Span::styled("examples ", Style::default().fg(theme_dim())),
Span::styled("octocat", Style::default().fg(theme_blue())),
Span::raw(" "),
Span::styled("octocat/Spoon-Knife", Style::default().fg(theme_green())),
Span::raw(" "),
Span::styled(
"https://github.com/owner/repo",
Style::default().fg(theme_red()),
),
],
Mode::Models => match model_provider {
ModelProvider::Hf => vec![
Span::styled("examples ", Style::default().fg(theme_dim())),
Span::styled("meta-llama", Style::default().fg(theme_blue())),
Span::raw(" "),
Span::styled("meta-llama/Llama-2-7b", Style::default().fg(theme_green())),
Span::raw(" "),
Span::styled("top", Style::default().fg(theme_red())),
Span::raw(" "),
Span::styled(
"https://huggingface.co/openai-community/gpt2",
Style::default().fg(theme_red()),
),
],
ModelProvider::Ollama => vec![
Span::styled("examples ", Style::default().fg(theme_dim())),
Span::styled("llama3:8b", Style::default().fg(theme_blue())),
Span::raw(" "),
Span::styled("qwen2.5:7b", Style::default().fg(theme_green())),
Span::raw(" "),
Span::styled("latest", Style::default().fg(theme_red())),
Span::raw(" "),
Span::styled("phi3:3.8b", Style::default().fg(theme_red())),
],
ModelProvider::Civitai => vec![
Span::styled("examples ", Style::default().fg(theme_dim())),
Span::styled("827184", Style::default().fg(theme_blue())),
Span::raw(" "),
Span::styled("realistic-vision-v6", Style::default().fg(theme_green())),
Span::raw(" "),
Span::styled("top", Style::default().fg(theme_red())),
Span::raw(" "),
Span::styled(
"https://civitai.com/models/827184",
Style::default().fg(theme_red()),
),
],
},
};
let body = Paragraph::from_text(Text::new(vec![
Line::new(vec![
Span::styled(prompt_label, prompt_style()),
Span::styled(input, Style::default().fg(theme_white())),
]),
Line::raw(""),
Line::new(examples),
]))
.wrap(WrapMode::Character { trim: false });
frame.render_block_widget(layout[1], body, knott_block(" Target ", theme_blue()));
draw_thesa_bottom_bar(frame, layout[2], status, &["Enter accept", "Esc back"]);
let input_area = panel_inner(layout[1]);
let cursor_x = input_area
.x
.saturating_add(prompt_label.len() as u16)
.saturating_add(input.chars().count() as u16)
.min(input_area.x + input_area.width.saturating_sub(1));
frame.set_cursor_position(cursor_x, input_area.y);
}
#[allow(clippy::too_many_arguments)]
fn draw_model_picker_tui(
frame: &mut Frame<'_>,
models: &[ModelItem],
filtered: &[usize],
selected: &BTreeSet<usize>,
cursor: usize,
query: &str,
status: &str,
tick: usize,
batch_size: usize,
batch_overlay: bool,
) {
let area = frame.area();
paint_surface(frame, area);
if area.width < 28 || area.height < 8 {
let line = format!(
"models {} visible / {} total\nfilter> {}\n{}",
filtered.len(),
models.len(),
query,
status,
);
frame.render_widget(
Paragraph::from_text(Text::new(vec![Line::styled(
&line,
Style::default().fg(theme_white()),
)])),
area,
);
return;
}
let layout = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(header_height(area)),
Constraint::Min(8),
Constraint::Length(2),
])
.split(area);
draw_thesa_header(frame, layout[0], tick, status, "model picker");
let body = Layout::default()
.direction(Direction::Horizontal)
.constraints(if layout[1].width >= 104 {
[Constraint::Percentage(72), Constraint::Percentage(28)]
} else {
[Constraint::Percentage(100), Constraint::Percentage(0)]
})
.split(layout[1]);
let clone_scope = if selected.is_empty() {
format!("visible: {}", filtered.len())
} else {
format!("checked only: {}", selected.len())
};
let max_name_width = if body[0].width > 56 {
(body[0].width as usize - 52).max(20)
} else {
(body[0].width as usize - 30).max(10)
};
let provider_counts = format!(
"hf:{} ollama:{} civitai:{}",
models
.iter()
.filter(|m| m.provider == ModelProvider::Hf)
.count(),
models
.iter()
.filter(|m| m.provider == ModelProvider::Ollama)
.count(),
models
.iter()
.filter(|m| m.provider == ModelProvider::Civitai)
.count(),
);
let items = filtered
.iter()
.map(|model_index| {
let model = &models[*model_index];
let marker = if selected.contains(model_index) {
"[x]"
} else {
"[ ]"
};
let tag = model.pipeline_tag.as_deref().unwrap_or("-");
let dl = format_downloads(model.downloads);
let size = format_model_size(model.size);
let name = truncate_to_width(&model.id, max_name_width);
let tag = truncate_to_width(tag, 10);
ListItem::with_line(Line::new(vec![
Span::styled(
marker,
Style::default()
.fg(theme_green())
.add_modifier(Modifier::BOLD),
),
Span::raw(" "),
Span::styled(
&format!("[{}]", model.provider.short_label()),
Style::default()
.fg(model_provider_color(model.provider))
.add_modifier(Modifier::BOLD),
),
Span::raw(" "),
Span::styled(
&format!("{name:<max_name_width$}"),
Style::default().fg(theme_white()),
),
Span::styled(&format!(" {dl:>7} "), Style::default().fg(theme_blue())),
Span::styled(&format!(" {size:>9} "), Style::default().fg(theme_dim())),
Span::styled(&format!(" [{tag}]"), Style::default().fg(theme_dim())),
]))
})
.collect::<Vec<_>>();
let mut state = ListState::default();
if !filtered.is_empty() {
state.select(Some(cursor));
}
let list_title = format!(
" Models | filter: {} | {} | batch: {} | {} ",
if query.is_empty() { "all" } else { query },
clone_scope,
batch_size,
provider_counts,
);
let list = List::new(&items)
.with_highlight_style(selected_style(tick))
.with_highlight_symbol("> ");
frame.render_block_stateful_list(
body[0],
&list,
&mut state,
knott_block(&list_title, theme_blue()),
);
if body[1].width > 0 {
let side = Layout::default()
.direction(Direction::Vertical)
.constraints([Constraint::Percentage(65), Constraint::Percentage(35)])
.split(body[1]);
let mut sidebar_lines = vec![
section_line("Controls"),
help_pair_line("type", "filter models"),
help_pair_line("Space", "check/uncheck model"),
help_pair_line("a", "check visible"),
help_pair_line("b", "batch size overlay"),
help_pair_line("c", "clear"),
help_pair_line("PgUp/PgDown", "jump pages"),
help_pair_line("Home/End", "first/last"),
help_pair_line("Enter", "checked only, or visible"),
Line::raw(""),
section_line("Selection"),
help_pair_line("checked", &selected.len().to_string()),
help_pair_line("filtered", &filtered.len().to_string()),
];
if let Some(model) = filtered.get(cursor).and_then(|i| models.get(*i)) {
sidebar_lines.push(Line::raw(""));
sidebar_lines.push(section_line("Active"));
sidebar_lines.push(help_pair_line("provider", model.provider.label()));
sidebar_lines.push(Line::new(vec![
Span::styled(
"name ",
Style::default()
.fg(theme_blue())
.add_modifier(Modifier::BOLD),
),
Span::styled(
&truncate_to_width(&model.id, (body[1].width as usize).saturating_sub(16)),
Style::default().fg(theme_white()),
),
]));
sidebar_lines.push(help_pair_line(
"tag",
model.pipeline_tag.as_deref().unwrap_or("-"),
));
sidebar_lines.push(help_pair_line(
"downloads",
&format_downloads(model.downloads),
));
sidebar_lines.push(help_pair_line("size", &format_model_size(model.size)));
sidebar_lines.push(help_pair_line(
"updated",
model.last_modified.as_deref().unwrap_or("n/a"),
));
} else {
sidebar_lines.push(Line::raw(""));
sidebar_lines.push(section_line("Active"));
sidebar_lines.push(Line::raw("no model selected"));
}
let sidebar = Paragraph::from_text(Text::new(sidebar_lines))
.wrap(WrapMode::Character { trim: false });
frame.render_block_widget(
side[0],
sidebar,
knott_block(" controls + selection ", theme_dim()),
);
render_aisling_panel(
frame,
side[1],
" scrin aisling burn ",
EffectKind::Burn,
tick,
"models",
);
}
draw_thesa_bottom_bar(
frame,
layout[2],
status,
&[
"Up/Down move",
"PgUp/PgDown jump",
"Home/End jump",
"Space check",
"b batch",
"Enter download",
"Esc quit",
],
);
if batch_overlay {
draw_batch_overlay(frame, area, batch_size);
}
}
fn picker_page_step(terminal_height: u16) -> usize {
let header_rows = if terminal_height >= 30 { 5 } else { 3 };
let list_rows = terminal_height.saturating_sub(header_rows + 2);
(list_rows.max(1) as usize).max(5) / 2
}
fn render_download_dashboard(
frame: &mut Frame,
names: &[String],
state: &[BatchState],
summary: &CloneSummary,
batch_index: usize,
batch_size: usize,
total_models: usize,
total_batches: usize,
tick: usize,
elapsed: Duration,
effect_kind: EffectKind,
ui_state: &UiState,
) {
let area = frame.area();
paint_surface(frame, area);
let overall_done = summary.cloned + summary.skipped + summary.failed.len();
let status = if ui_state.abort_requested {
"stopping after current batch"
} else {
"downloading models"
};
if area.width < 36 || area.height < 10 {
let loader = Loader::with_config(
LoaderKind::Tqdm,
LoaderConfig::default()
.with_size(area.width.saturating_sub(4) as usize, 1)
.with_label("thesa")
.with_unit("model")
.with_fraction(true),
);
let line = loader.line(
tick,
LoaderProgress::from_counts(overall_done as u64, total_models as u64),
);
let panel = Paragraph::from_text(Text::new(vec![
Line::styled(
"thesa",
Style::default()
.fg(theme_white())
.add_modifier(Modifier::BOLD),
),
Line::styled(&line, Style::default().fg(theme_white())),
Line::styled(status, Style::default().fg(theme_dim())),
]));
frame.render_block_widget(area, panel, knott_block(" thesa models ", theme_blue()));
return;
}
let root = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(header_height(area)),
Constraint::Min(8),
Constraint::Length(2),
])
.split(area);
draw_thesa_header(frame, root[0], tick, status, "model download");
let body = Layout::default()
.direction(Direction::Horizontal)
.constraints(if root[1].width >= 104 {
[Constraint::Percentage(68), Constraint::Percentage(32)]
} else {
[Constraint::Percentage(100), Constraint::Percentage(0)]
})
.split(root[1]);
let left = Layout::default()
.direction(Direction::Vertical)
.constraints([Constraint::Length(5), Constraint::Min(5)])
.split(body[0]);
let current_batch = ((batch_index - 1) / batch_size) + 1;
render_command_progress_panel(
frame,
left[0],
" command progress ",
CommandProgress {
verb: "download models",
unit: "model",
done_label: "downloaded",
done: summary.cloned,
skipped: summary.skipped,
failed: summary.failed.len(),
total: total_models,
batch_index: current_batch,
batch_size,
total_batches,
elapsed,
},
tick,
);
let model_loader = Loader::with_config(
LoaderKind::Tqdm,
LoaderConfig::default()
.with_size(left[1].width.saturating_sub(34).max(10) as usize, 1)
.with_label("model")
.with_unit("step")
.with_fraction(true),
);
let mut rows = Vec::new();
if ui_state.show_details {
for (index, (name, model_state)) in names.iter().zip(state.iter()).enumerate() {
rows.push(repo_loader_row(
&model_loader,
tick + index,
name,
*model_state,
));
}
} else {
rows.push(Line::styled(
"details hidden; press t to show concurrent model loaders",
Style::default().fg(theme_dim()),
));
}
let panel = Paragraph::from_text(Text::new(rows)).wrap(WrapMode::Character { trim: false });
frame.render_block_widget(
left[1],
panel,
knott_block(" Concurrent Downloads | Aisling Tqdm ", theme_blue()),
);
if body[1].width > 0 {
let (pending, running, done, skipped, failed) = batch_state_counts(state);
let side_layout = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(8),
Constraint::Length(8),
Constraint::Min(4),
])
.split(body[1]);
render_command_info_panel(
frame,
side_layout[0],
" telemetry ",
vec![
section_line("Batch State"),
help_pair_line("running", &running.to_string()),
help_pair_line("pending", &pending.to_string()),
help_pair_line("done", &done.to_string()),
help_pair_line("skipped", &skipped.to_string()),
help_pair_line("failed", &failed.to_string()),
],
theme_blue(),
);
let side_lines = vec![
section_line("Controls"),
help_pair_line("q / Esc", "stop after current batch"),
help_pair_line("t", "toggle model rows"),
help_pair_line("m", "motion effect"),
help_pair_line("b", "burn effect"),
help_pair_line("w", "wipe effect"),
];
render_command_info_panel(
frame,
side_layout[1],
" command keys ",
side_lines,
theme_dim(),
);
render_aisling_panel(
frame,
side_layout[2],
&format!(" scrin aisling {} ", effect_kind.name()),
effect_kind,
tick,
"models\ndownload",
);
}
draw_thesa_bottom_bar(
frame,
root[2],
status,
&["q stop", "t details", "m motion", "b burn", "w wipe"],
);
}
fn format_downloads(n: u64) -> String {
if n >= 1_000_000 {
format!("{:.1}M", n as f64 / 1_000_000.0)
} else if n >= 1_000 {
format!("{:.1}K", n as f64 / 1_000.0)
} else {
format!("{n}")
}
}
fn format_model_size(size: Option<u64>) -> String {
match size {
None => "n/a".to_string(),
Some(bytes) => {
let gb = 1024f64 * 1024f64 * 1024f64;
let mb = 1024f64 * 1024f64;
let kb = 1024f64;
if bytes as f64 >= gb {
format!("{:.1}GB", bytes as f64 / gb)
} else if bytes as f64 >= mb {
format!("{:.1}MB", bytes as f64 / mb)
} else if bytes as f64 >= kb {
format!("{:.1}KB", bytes as f64 / kb)
} else {
format!("{bytes} B")
}
}
}
}
fn truncate_to_width(value: &str, width: usize) -> String {
if width == 0 {
return String::new();
}
let chars: Vec<char> = value.chars().collect();
if chars.len() <= width {
value.to_string()
} else if width <= 3 {
chars.iter().take(width).copied().collect()
} else {
let mut out: String = chars.iter().take(width - 3).copied().collect();
out.push_str("...");
out
}
}
fn model_provider_color(provider: ModelProvider) -> TuiColor {
match provider {
ModelProvider::Hf => TuiColor::rgb(130, 190, 255),
ModelProvider::Ollama => TuiColor::rgb(130, 255, 130),
ModelProvider::Civitai => TuiColor::rgb(255, 200, 130),
}
}
fn fetch_repositories_for_target(
client: &Client,
target: &Target,
token: Option<&str>,
) -> Result<Vec<RepoItem>> {
match target {
Target::Owner(owner) => match list_repos_for_owner(client, owner, token) {
Ok(items) => Ok(items),
Err(GitArchiverError::NotFound(_)) => list_repos_for_org(client, owner, token),
Err(other) => Err(other),
},
Target::Repo { owner, repo } => Ok(vec![fetch_single_repo(client, owner, repo, token)?]),
}
}
fn fetch_repositories_tui(
terminal: &mut Terminal,
client: &Client,
target: &Target,
token: Option<&str>,
) -> Result<Vec<RepoItem>> {
let client = client.clone();
let target = target.clone();
let token = token.map(str::to_string);
let loading_label = match &target {
Target::Owner(owner) => format!("loading public repos for {owner}"),
Target::Repo { owner, repo } => format!("loading repo {owner}/{repo}"),
};
let (tx, rx) = mpsc::channel();
thread::spawn(move || {
let result = fetch_repositories_for_target(&client, &target, token.as_deref());
let _ = tx.send(result);
});
let mut tick = 0usize;
loop {
if let Ok(result) = rx.try_recv() {
return result;
}
draw_full_frame(terminal, |frame| {
draw_loading_tui(frame, &loading_label, tick, Mode::Repos, None)
})?;
tick = tick.wrapping_add(1);
if event::poll(Duration::from_millis(80))? {
let Event::Key(key) = event::read()? else {
continue;
};
if key.kind == KeyEventKind::Press && matches!(key.code, KeyCode::Esc) {
return Err(GitArchiverError::Message("aborted by user".to_string()));
}
}
}
}
fn fetch_models_tui(
terminal: &mut Terminal,
client: &Client,
target: &ModelTarget,
args: &Cli,
) -> Result<Vec<ModelItem>> {
let client = client.clone();
let target = target.clone();
let args_clone = args.clone();
let loading_label = match &target {
ModelTarget::Org(org) => format!("loading models for {org} from {}", args.provider.label()),
ModelTarget::Model(id) => format!("loading model {id} from {}", args.provider.label()),
ModelTarget::Top => format!("loading top models from {}", args.provider.label()),
ModelTarget::Latest => {
format!("loading latest models from {}", args.provider.label())
}
};
let (tx, rx) = mpsc::channel();
thread::spawn(move || {
let result = fetch_models_for_target(&client, &target, &args_clone);
let _ = tx.send(result);
});
let mut tick = 0usize;
loop {
if let Ok(result) = rx.try_recv() {
return result;
}
draw_full_frame(terminal, |frame| {
draw_loading_tui(
frame,
&loading_label,
tick,
Mode::Models,
Some(args.provider),
)
})?;
tick = tick.wrapping_add(1);
if event::poll(Duration::from_millis(80))? {
let Event::Key(key) = event::read()? else {
continue;
};
if key.kind == KeyEventKind::Press && matches!(key.code, KeyCode::Esc) {
return Err(GitArchiverError::Message("aborted by user".to_string()));
}
}
}
}
fn draw_loading_tui(
frame: &mut Frame<'_>,
label: &str,
tick: usize,
mode: Mode,
provider: Option<ModelProvider>,
) {
let area = frame.area();
paint_surface(frame, area);
if area.width < 30 || area.height < 8 {
let message = format!("thesa\n{label}");
frame.render_widget(
Paragraph::from_text(Text::new(vec![Line::styled(
&message,
Style::default().fg(theme_white()),
)])),
area,
);
return;
}
let layout = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(header_height(area)),
Constraint::Min(5),
Constraint::Length(2),
])
.split(area);
let mode_label = match mode {
Mode::Repos => "github",
Mode::Models => match provider.unwrap_or(ModelProvider::Hf) {
ModelProvider::Hf => "huggingface",
ModelProvider::Ollama => "ollama",
ModelProvider::Civitai => "civitai",
},
};
draw_thesa_header(frame, layout[0], tick, label, mode_label);
let loader = Loader::with_config(
LoaderKind::Tqdm,
LoaderConfig::default()
.with_size(layout[1].width.saturating_sub(4).max(12) as usize, 1)
.with_label(mode_label)
.with_unit(match mode {
Mode::Repos => "repo",
Mode::Models => "model",
})
.with_fraction(false),
);
let loader_line = loader.line(
tick,
LoaderProgress::from_counts((tick % 97) as u64 + 1, 100),
);
let body = Layout::default()
.direction(Direction::Horizontal)
.constraints(if layout[1].width >= 74 {
[Constraint::Percentage(55), Constraint::Percentage(45)]
} else {
[Constraint::Percentage(100), Constraint::Percentage(0)]
})
.split(layout[1]);
let panel = Paragraph::from_text(Text::new(vec![
section_line("Loading"),
Line::styled(label, Style::default().fg(theme_white())),
Line::raw(&loader_line),
]))
.wrap(WrapMode::Character { trim: false });
frame.render_block_widget(
body[0],
panel,
knott_block(" Aisling Loader ", theme_blue()),
);
if body[1].width > 0 {
render_aisling_panel(
frame,
body[1],
" scrin aisling wipe ",
EffectKind::Wipe,
tick,
label,
);
}
draw_thesa_bottom_bar(frame, layout[2], label, &["Esc cancel"]);
}
fn pick_repositories_tui_inner(
terminal: &mut Terminal,
repos: &[RepoItem],
batch_size: &mut usize,
) -> Result<Vec<RepoItem>> {
if repos.is_empty() {
return Ok(vec![]);
}
let mut query = String::new();
let mut selected = BTreeSet::new();
let mut cursor = 0usize;
let mut frame_index = 0usize;
let mut batch_overlay = false;
let mut status =
"Space checks repos; Enter clones checked repos, or visible if none checked".to_string();
loop {
let filtered = filtered_repo_indices(repos, &query);
if cursor >= filtered.len() {
cursor = filtered.len().saturating_sub(1);
}
let page_step = picker_page_step(terminal.size().height);
draw_full_frame(terminal, |frame| {
draw_repo_picker_tui(
frame,
repos,
&filtered,
&selected,
cursor,
&query,
&status,
frame_index,
*batch_size,
batch_overlay,
);
})?;
frame_index = frame_index.wrapping_add(1);
if !event::poll(Duration::from_millis(50))? {
continue;
}
let Event::Key(key) = event::read()? else {
continue;
};
if key.kind != KeyEventKind::Press {
continue;
}
if batch_overlay {
match key.code {
KeyCode::Esc | KeyCode::Char('b') | KeyCode::Enter => {
batch_overlay = false;
status = format!("batch size set to {}; Enter clones repos", *batch_size);
}
KeyCode::Char(ch) => {
if let Some(value) = batch_size_from_key(ch) {
*batch_size = value;
batch_overlay = false;
status = format!("batch size set to {value}; Enter clones repos");
}
}
KeyCode::Left | KeyCode::Up => {
*batch_size = previous_batch_size(*batch_size);
status = format!("batch size set to {}; Enter accepts", *batch_size);
}
KeyCode::Right | KeyCode::Down => {
*batch_size = next_batch_size(*batch_size);
status = format!("batch size set to {}; Enter accepts", *batch_size);
}
_ => {}
}
continue;
}
match key.code {
KeyCode::Esc => return Err(GitArchiverError::Message("aborted by user".to_string())),
KeyCode::Enter => {
let picked = if selected.is_empty() {
filtered
} else {
selected.iter().copied().collect::<Vec<_>>()
};
if picked.is_empty() {
status = "no repositories selected or visible".to_string();
continue;
}
return Ok(picked
.into_iter()
.map(|index| repos[index].clone())
.collect());
}
KeyCode::Up => cursor = cursor.saturating_sub(1),
KeyCode::Down => {
if cursor + 1 < filtered.len() {
cursor += 1;
}
}
KeyCode::PageUp => {
let step = page_step.min(filtered.len().max(1));
cursor = cursor.saturating_sub(step);
status = if filtered.is_empty() {
"no repositories to move to".to_string()
} else {
format!("moved up by {step}")
};
}
KeyCode::PageDown => {
if filtered.is_empty() {
status = "no repositories to move to".to_string();
} else {
let step = page_step.min(filtered.len());
cursor = (cursor + step).min(filtered.len() - 1);
status = format!("moved down by {step}");
}
}
KeyCode::Home => {
if !filtered.is_empty() {
cursor = 0;
status = "jumped to first repo".to_string();
} else {
status = "no repositories to move to".to_string();
}
}
KeyCode::End => {
if !filtered.is_empty() {
cursor = filtered.len() - 1;
status = "jumped to last repo".to_string();
} else {
status = "no repositories to move to".to_string();
}
}
KeyCode::Char(' ') => {
if let Some(repo_index) = filtered.get(cursor).copied() {
if !selected.insert(repo_index) {
selected.remove(&repo_index);
}
let checked = selected.len();
status = if checked == 0 {
"nothing checked; Enter clones visible repos".to_string()
} else {
format!("{checked} checked; Enter clones checked repos only")
};
}
}
KeyCode::Char('a') => {
for repo_index in &filtered {
selected.insert(*repo_index);
}
status = format!("checked {} visible repos", filtered.len());
}
KeyCode::Char('c') => {
selected.clear();
query.clear();
cursor = 0;
status = "cleared filter and selection".to_string();
}
KeyCode::Char('b') => {
batch_overlay = true;
status = "choose batch size: 1 2 3 6 8".to_string();
}
KeyCode::Backspace => {
query.pop();
cursor = 0;
}
KeyCode::Char(ch) => {
query.push(ch);
cursor = 0;
}
_ => {}
}
}
}
fn filtered_repo_indices(repos: &[RepoItem], query: &str) -> Vec<usize> {
if query.trim().is_empty() {
return (0..repos.len()).collect();
}
let query = query.to_lowercase();
repos
.iter()
.enumerate()
.filter(|(_, repo)| {
repo.name.to_lowercase().contains(&query)
|| repo.full_name.to_lowercase().contains(&query)
})
.map(|(index, _)| index)
.collect()
}
#[allow(clippy::too_many_arguments)]
fn draw_repo_picker_tui(
frame: &mut Frame<'_>,
repos: &[RepoItem],
filtered: &[usize],
selected: &BTreeSet<usize>,
cursor: usize,
query: &str,
status: &str,
tick: usize,
batch_size: usize,
batch_overlay: bool,
) {
let area = frame.area();
paint_surface(frame, area);
if area.width < 28 || area.height < 8 {
let line = format!(
"repos {} visible / {} total\nfilter> {}\n{}",
filtered.len(),
repos.len(),
query,
status,
);
frame.render_widget(
Paragraph::from_text(Text::new(vec![Line::styled(
&line,
Style::default().fg(theme_white()),
)])),
area,
);
return;
}
let layout = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(header_height(area)),
Constraint::Min(8),
Constraint::Length(2),
])
.split(area);
draw_thesa_header(frame, layout[0], tick, status, "repo picker");
let body = Layout::default()
.direction(Direction::Horizontal)
.constraints(if layout[1].width >= 96 {
[Constraint::Percentage(72), Constraint::Percentage(28)]
} else {
[Constraint::Percentage(100), Constraint::Percentage(0)]
})
.split(layout[1]);
let clone_scope = if selected.is_empty() {
format!("visible: {}", filtered.len())
} else {
format!("checked only: {}", selected.len())
};
let items = filtered
.iter()
.map(|repo_index| {
let repo = &repos[*repo_index];
let marker = if selected.contains(repo_index) {
"[x]"
} else {
"[ ]"
};
ListItem::with_line(Line::new(vec![
Span::styled(
marker,
Style::default()
.fg(theme_green())
.add_modifier(Modifier::BOLD),
),
Span::raw(" "),
Span::styled(&repo.full_name, Style::default().fg(theme_white())),
]))
})
.collect::<Vec<_>>();
let mut state = ListState::default();
if !filtered.is_empty() {
state.select(Some(cursor));
}
let list_title = format!(
" Repositories | filter: {} | {} | batch: {} ",
if query.is_empty() { "all" } else { query },
clone_scope,
batch_size,
);
let list = List::new(&items)
.with_highlight_style(selected_style(tick))
.with_highlight_symbol("> ");
frame.render_block_stateful_list(
body[0],
&list,
&mut state,
knott_block(&list_title, theme_blue()),
);
if body[1].width > 0 {
let side = Layout::default()
.direction(Direction::Vertical)
.constraints([Constraint::Percentage(62), Constraint::Percentage(38)])
.split(body[1]);
let sidebar = Paragraph::from_text(Text::new(vec![
section_line("Controls"),
help_pair_line("type", "filter public repos"),
help_pair_line("Space", "check/uncheck repo"),
help_pair_line("a", "check visible"),
help_pair_line("b", "batch size overlay"),
help_pair_line("c", "clear"),
help_pair_line("PgUp/PgDown", "jump pages"),
help_pair_line("Home/End", "first/last"),
help_pair_line("Enter", "checked only, or visible"),
]));
frame.render_block_widget(side[0], sidebar, knott_block(" controls ", theme_dim()));
render_aisling_panel(
frame,
side[1],
" scrin aisling waves ",
EffectKind::Waves,
tick,
"repos",
);
}
draw_thesa_bottom_bar(
frame,
layout[2],
status,
&[
"Up/Down move",
"PgUp/PgDown jump",
"Home/End jump",
"Space check",
"b batch",
"Enter clone",
"Esc quit",
],
);
if batch_overlay {
draw_batch_overlay(frame, area, batch_size);
}
}
fn clone_repositories_scrin(
repos: &[RepoItem],
output: &Path,
args: &Cli,
ui_state: &mut UiState,
) -> Result<CloneSummary> {
let mut session = TerminalSession::new()?;
clone_repositories_scrin_inner(&mut session.terminal, repos, output, args, ui_state)
}
fn clone_repositories_scrin_inner(
terminal: &mut Terminal,
repos: &[RepoItem],
output: &Path,
args: &Cli,
ui_state: &mut UiState,
) -> Result<CloneSummary> {
let batch_size = args.concurrency.max(1);
let total = repos.len();
let total_batches = total.div_ceil(batch_size);
let mut summary = CloneSummary::default();
let mut effect_kind = EffectKind::Waves;
let started_at = Instant::now();
let mut tick = 0usize;
let mut batch_index = 1;
while batch_index <= total {
let batch_end = (batch_index + batch_size - 1).min(total);
let batch = &repos[batch_index - 1..batch_end];
let names = batch
.iter()
.map(|repo| repo.full_name.clone())
.collect::<Vec<_>>();
let mut state = vec![BatchState::Pending; batch.len()];
let (tx, rx) = mpsc::channel::<(usize, BatchEvent)>();
let mut handles = Vec::new();
for (offset, repo) in batch.iter().enumerate() {
let repo = repo.clone();
let output = output.to_path_buf();
let args = args.clone();
let tx = tx.clone();
handles.push((
repo.full_name.clone(),
thread::spawn(move || {
let started = tx.send((offset, BatchEvent::Started));
if started.is_err() {
return;
}
let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
clone_repository(&repo, &output, &args)
}));
let event = match outcome {
Ok(Ok(CloneStatus::Cloned)) => BatchEvent::Result(BatchResult::Done),
Ok(Ok(CloneStatus::Skipped)) => BatchEvent::Result(BatchResult::Skipped),
Ok(Err(err)) => BatchEvent::Result(BatchResult::Error(err.to_string())),
Err(_) => BatchEvent::Result(BatchResult::Error(
"worker thread panicked while cloning".to_string(),
)),
};
let _ = tx.send((offset, event));
}),
));
}
drop(tx);
let mut completed = 0;
while completed < batch.len() {
while event::poll(Duration::from_millis(0))? {
if let Event::Key(key) = event::read()? {
match key.code {
KeyCode::Char('q') | KeyCode::Esc => ui_state.abort_requested = true,
KeyCode::Char('t') => ui_state.show_details = !ui_state.show_details,
KeyCode::Char('m') => effect_kind = EffectKind::Waves,
KeyCode::Char('b') => effect_kind = EffectKind::Burn,
KeyCode::Char('w') => effect_kind = EffectKind::Wipe,
_ => {}
}
}
}
while let Ok((offset, event)) = rx.try_recv() {
match event {
BatchEvent::Started => state[offset] = BatchState::Running,
BatchEvent::Result(BatchResult::Done) => {
state[offset] = BatchState::Done;
summary.cloned += 1;
completed += 1;
}
BatchEvent::Result(BatchResult::Skipped) => {
state[offset] = BatchState::Skipped;
summary.skipped += 1;
completed += 1;
}
BatchEvent::Result(BatchResult::Error(message)) => {
state[offset] = BatchState::Failed;
summary
.failed
.push(format!("{}: {}", names[offset], message));
completed += 1;
}
}
}
draw_full_frame(terminal, |frame| {
render_clone_dashboard(
frame,
&names,
&state,
&summary,
batch_index,
batch_size,
total,
total_batches,
tick,
started_at.elapsed(),
effect_kind,
ui_state,
)
})?;
tick = tick.wrapping_add(1);
if completed < batch.len() {
thread::sleep(Duration::from_millis(80));
}
}
for (repo_name, handle) in handles {
if handle.join().is_err() {
summary
.failed
.push(format!("{repo_name}: worker thread panicked"));
}
}
draw_full_frame(terminal, |frame| {
render_clone_dashboard(
frame,
&names,
&state,
&summary,
batch_index,
batch_size,
total,
total_batches,
tick,
started_at.elapsed(),
effect_kind,
ui_state,
)
})?;
if ui_state.abort_requested {
break;
}
batch_index = batch_end + 1;
}
Ok(summary)
}
#[allow(clippy::too_many_arguments)]
fn render_clone_dashboard(
frame: &mut Frame,
names: &[String],
state: &[BatchState],
summary: &CloneSummary,
batch_index: usize,
batch_size: usize,
total_repos: usize,
total_batches: usize,
tick: usize,
elapsed: Duration,
effect_kind: EffectKind,
ui_state: &UiState,
) {
let area = frame.area();
paint_surface(frame, area);
let overall_done = summary.cloned + summary.skipped + summary.failed.len();
let status = if ui_state.abort_requested {
"stopping after current batch"
} else {
"cloning"
};
if area.width < 36 || area.height < 10 {
let loader = Loader::with_config(
LoaderKind::Tqdm,
LoaderConfig::default()
.with_size(area.width.saturating_sub(4) as usize, 1)
.with_label("thesa")
.with_unit("repo")
.with_fraction(true),
);
let line = loader.line(
tick,
LoaderProgress::from_counts(overall_done as u64, total_repos as u64),
);
let panel = Paragraph::from_text(Text::new(vec![
Line::styled(
"thesa",
Style::default()
.fg(theme_white())
.add_modifier(Modifier::BOLD),
),
Line::styled(&line, Style::default().fg(theme_white())),
Line::styled(status, Style::default().fg(theme_dim())),
]));
frame.render_block_widget(area, panel, knott_block(" thesa ", theme_blue()));
return;
}
let root = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(header_height(area)),
Constraint::Min(8),
Constraint::Length(2),
])
.split(area);
draw_thesa_header(frame, root[0], tick, status, "clone dashboard");
let body = Layout::default()
.direction(Direction::Horizontal)
.constraints(if root[1].width >= 104 {
[Constraint::Percentage(68), Constraint::Percentage(32)]
} else {
[Constraint::Percentage(100), Constraint::Percentage(0)]
})
.split(root[1]);
let left = Layout::default()
.direction(Direction::Vertical)
.constraints([Constraint::Length(5), Constraint::Min(5)])
.split(body[0]);
let current_batch = ((batch_index - 1) / batch_size) + 1;
render_command_progress_panel(
frame,
left[0],
" command progress ",
CommandProgress {
verb: "clone repositories",
unit: "repo",
done_label: "cloned",
done: summary.cloned,
skipped: summary.skipped,
failed: summary.failed.len(),
total: total_repos,
batch_index: current_batch,
batch_size,
total_batches,
elapsed,
},
tick,
);
let repo_loader = Loader::with_config(
LoaderKind::Tqdm,
LoaderConfig::default()
.with_size(left[1].width.saturating_sub(34).max(10) as usize, 1)
.with_label("repo")
.with_unit("step")
.with_fraction(true),
);
let mut rows = Vec::new();
if ui_state.show_details {
for (index, (name, repo_state)) in names.iter().zip(state.iter()).enumerate() {
rows.push(repo_loader_row(
&repo_loader,
tick + index,
name,
*repo_state,
));
}
} else {
rows.push(Line::styled(
"details hidden; press t to show concurrent repo loaders",
Style::default().fg(theme_dim()),
));
}
let repos = Paragraph::from_text(Text::new(rows)).wrap(WrapMode::Character { trim: false });
frame.render_block_widget(
left[1],
repos,
knott_block(" Concurrent Pulls | Aisling Tqdm ", theme_blue()),
);
if body[1].width > 0 {
let (pending, running, done, skipped, failed) = batch_state_counts(state);
let side_layout = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(8),
Constraint::Length(8),
Constraint::Min(4),
])
.split(body[1]);
render_command_info_panel(
frame,
side_layout[0],
" telemetry ",
vec![
section_line("Batch State"),
help_pair_line("running", &running.to_string()),
help_pair_line("pending", &pending.to_string()),
help_pair_line("done", &done.to_string()),
help_pair_line("skipped", &skipped.to_string()),
help_pair_line("failed", &failed.to_string()),
],
theme_blue(),
);
let side_lines = vec![
section_line("Controls"),
help_pair_line("q / Esc", "stop after current batch"),
help_pair_line("t", "toggle repo rows"),
help_pair_line("m", "motion effect"),
help_pair_line("b", "burn effect"),
help_pair_line("w", "wipe effect"),
];
render_command_info_panel(
frame,
side_layout[1],
" command keys ",
side_lines,
theme_dim(),
);
render_aisling_panel(
frame,
side_layout[2],
&format!(" scrin aisling {} ", effect_kind.name()),
effect_kind,
tick,
"repos\nclone",
);
}
draw_thesa_bottom_bar(
frame,
root[2],
status,
&["q stop", "t details", "m motion", "b burn", "w wipe"],
);
}
fn batch_size_from_key(ch: char) -> Option<usize> {
match ch {
'1' => Some(1),
'2' => Some(2),
'3' => Some(3),
'6' => Some(6),
'8' => Some(8),
_ => None,
}
}
fn batch_state_counts(state: &[BatchState]) -> (usize, usize, usize, usize, usize) {
let mut pending = 0;
let mut running = 0;
let mut done = 0;
let mut skipped = 0;
let mut failed = 0;
for item in state {
match item {
BatchState::Pending => pending += 1,
BatchState::Running => running += 1,
BatchState::Done => done += 1,
BatchState::Skipped => skipped += 1,
BatchState::Failed => failed += 1,
}
}
(pending, running, done, skipped, failed)
}
fn previous_batch_size(current: usize) -> usize {
let index = BATCH_SIZE_CHOICES
.iter()
.position(|value| *value == current)
.unwrap_or(0);
BATCH_SIZE_CHOICES[index.saturating_sub(1)]
}
fn next_batch_size(current: usize) -> usize {
let index = BATCH_SIZE_CHOICES
.iter()
.position(|value| *value == current)
.unwrap_or(0);
BATCH_SIZE_CHOICES[(index + 1).min(BATCH_SIZE_CHOICES.len() - 1)]
}
fn draw_batch_overlay(frame: &mut Frame<'_>, area: Rect, batch_size: usize) {
let width = area.width.min(58).max(30.min(area.width));
let height = area.height.min(11).max(7.min(area.height));
let x = area.x + area.width.saturating_sub(width) / 2;
let y = area.y + area.height.saturating_sub(height) / 2;
let overlay = Rect::new(x, y, width, height);
frame.render_widget(Clear::new(), overlay);
let mut choice_spans = Vec::new();
for value in BATCH_SIZE_CHOICES {
if !choice_spans.is_empty() {
choice_spans.push(Span::raw(" "));
}
let style = if value == batch_size {
Style::default()
.bg(TuiColor::WHITE)
.fg(TuiColor::BLACK)
.add_modifier(Modifier::BOLD)
} else {
Style::default()
.fg(theme_blue())
.add_modifier(Modifier::BOLD)
};
choice_spans.push(Span::styled(&format!(" {value} "), style));
}
let panel = Paragraph::from_text(Text::new(vec![
section_line("Batch Size"),
Line::styled(
"How many items to process at once",
Style::default().fg(theme_dim()),
),
Line::raw(""),
Line::new(choice_spans),
Line::raw(""),
help_pair_line("1/2/3/6/8", "select and close"),
help_pair_line("Left/Right", "cycle choices"),
help_pair_line("b / Enter", "close overlay"),
]))
.wrap(WrapMode::Character { trim: false });
frame.render_block_widget(overlay, panel, knott_block(" Batch Options ", theme_red()));
}
fn draw_thesa_header(frame: &mut Frame<'_>, area: Rect, tick: usize, status: &str, mode: &str) {
if area.height == 0 {
return;
}
if area.height >= 5 && area.width >= 88 {
let deck = Layout::default()
.direction(Direction::Horizontal)
.constraints([Constraint::Percentage(62), Constraint::Percentage(38)])
.split(area);
draw_thesa_header_text(frame, deck[0], tick, status, mode);
render_aisling_panel(
frame,
deck[1],
" scrin aisling matrix ",
EffectKind::Matrix,
tick,
"thesa",
);
return;
}
draw_thesa_header_text(frame, area, tick, status, mode);
}
fn draw_thesa_header_text(
frame: &mut Frame<'_>,
area: Rect,
tick: usize,
status: &str,
mode: &str,
) {
let accent = animated_accent(tick);
let mut primary = Vec::new();
primary.push(Span::styled(
" thesa ",
Style::default()
.bg(accent)
.fg(TuiColor::BLACK)
.add_modifier(Modifier::BOLD),
));
append_status_chip(&mut primary, "mode", mode, theme_blue());
append_status_chip(&mut primary, "state", status, theme_green());
append_status_chip(&mut primary, "effect", "aisling", theme_red());
append_status_chip(&mut primary, "frame", tick.to_string(), theme_dim());
let mut lines = vec![Line::new(primary)];
if area.height >= 5 {
lines.push(Line::raw(""));
lines.push(Line::new(vec![
Span::styled("archive ", Style::default().fg(theme_dim())),
Span::styled("public repos", Style::default().fg(theme_blue())),
Span::styled(" + ", Style::default().fg(theme_dim())),
Span::styled("models", Style::default().fg(theme_green())),
Span::styled(" with live scrin panes", Style::default().fg(theme_dim())),
]));
}
let header = Paragraph::from_text(Text::new(lines)).alignment(Alignment::Left);
frame.render_block_widget(area, header, knott_block(" thesa command deck ", accent));
}
fn draw_thesa_bottom_bar(frame: &mut Frame<'_>, area: Rect, status: &str, keys: &[&str]) {
if area.height == 0 {
return;
}
frame
.buffer()
.fill(area, ' ', theme_white(), Some(theme_surface()));
let mut status_spans = vec![
Span::styled("status: ", Style::default().fg(theme_dim())),
Span::styled(status, Style::default().fg(theme_white())),
];
append_status_chip(&mut status_spans, "ui", "scrin", theme_blue());
append_status_chip(&mut status_spans, "fx", "aisling", theme_green());
let key_line = Line::new(
keys.iter()
.flat_map(|key| {
let (head, detail) = key.split_once(' ').unwrap_or((key, ""));
let detail_spans = if detail.is_empty() {
vec![Span::raw(" ")]
} else {
vec![Span::raw(&format!(" {detail} "))]
};
let mut spans = Vec::new();
spans.push(Span::styled(head, key_style()));
spans.extend(detail_spans);
spans
})
.collect::<Vec<_>>(),
);
let lines = if area.height <= 1 {
vec![Line::new(status_spans)]
} else {
vec![Line::new(status_spans), key_line]
};
frame.render_widget(
Paragraph::from_text(Text::new(lines)).with_style(Style::default().fg(theme_white())),
area,
);
normalize_bg(frame.buffer(), area, theme_surface());
}
fn knott_block<'a>(title: &'a str, color: TuiColor) -> Block<'a> {
Block::new(title)
.with_borders(BorderStyle::Rounded)
.with_border_color(color)
.with_bg(theme_surface())
}
fn panel_inner(area: Rect) -> Rect {
knott_block("", theme_dim()).inner(area)
}
fn theme_white() -> TuiColor {
TuiColor::rgb(201, 209, 217)
}
fn theme_red() -> TuiColor {
TuiColor::rgb(248, 81, 73)
}
fn theme_blue() -> TuiColor {
TuiColor::rgb(88, 166, 255)
}
fn theme_green() -> TuiColor {
TuiColor::rgb(63, 185, 80)
}
fn theme_dim() -> TuiColor {
TuiColor::rgb(110, 118, 129)
}
fn theme_bg() -> TuiColor {
TuiColor::rgb(13, 17, 23)
}
fn theme_surface() -> TuiColor {
TuiColor::rgb(22, 27, 34)
}
fn prompt_style() -> Style {
Style::default()
.fg(theme_blue())
.add_modifier(Modifier::BOLD)
}
fn key_style() -> Style {
Style::default()
.bg(theme_blue())
.fg(TuiColor::BLACK)
.add_modifier(Modifier::BOLD)
}
fn selected_style(tick: usize) -> Style {
Style::default()
.bg(animated_accent(tick))
.fg(TuiColor::BLACK)
.add_modifier(Modifier::BOLD)
}
fn header_height(area: Rect) -> u16 {
if area.height >= 30 { 5 } else { 3 }
}
fn animated_accent(tick: usize) -> TuiColor {
let _ = tick;
theme_blue()
}
fn section_line(label: &str) -> Line {
Line::new(vec![Span::styled(
label,
Style::default()
.fg(theme_white())
.add_modifier(Modifier::BOLD),
)])
}
fn help_pair_line(head: &str, detail: &str) -> Line {
Line::new(vec![
Span::styled(
&format!("{head:<12}"),
Style::default()
.fg(theme_blue())
.add_modifier(Modifier::BOLD),
),
Span::styled(detail, Style::default().fg(theme_dim())),
])
}
fn append_status_chip(
spans: &mut Vec<Span>,
label: &str,
value: impl Into<String>,
color: TuiColor,
) {
if !spans.is_empty() {
spans.push(Span::raw(" "));
}
spans.push(Span::styled("[", Style::default().fg(theme_dim())));
spans.push(Span::styled(label, Style::default().fg(theme_dim())));
spans.push(Span::raw(" "));
let value = value.into();
spans.push(Span::styled(
&value,
Style::default().fg(color).add_modifier(Modifier::BOLD),
));
spans.push(Span::styled("]", Style::default().fg(theme_dim())));
}
fn paint_surface(frame: &mut Frame<'_>, area: Rect) {
if area.width == 0 || area.height == 0 {
return;
}
frame
.buffer()
.fill(area, ' ', theme_white(), Some(theme_bg()));
}
fn normalize_bg(buffer: &mut Buffer, area: Rect, bg: TuiColor) {
for y in area.y as usize..area.bottom() as usize {
for x in area.x as usize..area.right() as usize {
let Some(cell) = buffer.get(x, y).copied() else {
continue;
};
if cell.bg.is_none() {
buffer.set(
x,
y,
scrin::core::buffer::Cell {
bg: Some(bg),
..cell
},
);
}
}
}
}
fn render_aisling_panel(
frame: &mut Frame<'_>,
area: Rect,
title: &str,
kind: EffectKind,
tick: usize,
text: &str,
) {
if area.width < 4 || area.height < 4 {
return;
}
let block = knott_block(title, animated_accent(tick)).with_borders(BorderStyle::Thick);
block.render(frame.buffer(), area);
let inner = block.inner(area);
if inner.width == 0 || inner.height == 0 {
return;
}
frame
.buffer()
.fill(inner, ' ', theme_white(), Some(theme_surface()));
render_aisling_effect(frame, inner, kind, tick, text);
normalize_bg(frame.buffer(), inner, theme_surface());
}
fn render_aisling_effect(
frame: &mut Frame<'_>,
area: Rect,
kind: EffectKind,
tick: usize,
text: &str,
) {
if area.width == 0 || area.height == 0 {
return;
}
let mut player = EffectPlayer::new(kind, text)
.with_size(area.width as usize, area.height as usize)
.with_duration(96)
.with_seed(17)
.with_gradient_colors(
vec![
theme_blue(),
theme_green(),
theme_red(),
TuiColor::rgb(163, 113, 247),
],
45.0,
)
.with_accent(theme_blue());
player.set_frame(tick % player.total_frames().max(1));
player.render_to_buffer(frame.buffer(), area);
}
struct CommandProgress<'a> {
verb: &'a str,
unit: &'a str,
done_label: &'a str,
done: usize,
skipped: usize,
failed: usize,
total: usize,
batch_index: usize,
batch_size: usize,
total_batches: usize,
elapsed: Duration,
}
fn render_command_progress_panel(
frame: &mut Frame<'_>,
area: Rect,
title: &str,
data: CommandProgress<'_>,
tick: usize,
) {
if area.width < 4 || area.height < 4 {
return;
}
let block = knott_block(title, theme_green()).with_borders(BorderStyle::Thick);
block.render(frame.buffer(), area);
let inner = block.inner(area);
if inner.width == 0 || inner.height == 0 {
return;
}
frame
.buffer()
.fill(inner, ' ', theme_white(), Some(theme_surface()));
let completed = data.done + data.skipped + data.failed;
let progress_area = Rect::new(
inner.x,
inner.bottom().saturating_sub(1),
inner.width,
1.min(inner.height),
);
let text_height = inner.height.saturating_sub(progress_area.height);
let text_area = Rect::new(inner.x, inner.y, inner.width, text_height);
let mut chips = Vec::new();
append_status_chip(
&mut chips,
data.done_label,
data.done.to_string(),
theme_green(),
);
append_status_chip(
&mut chips,
"skipped",
data.skipped.to_string(),
TuiColor::YELLOW,
);
append_status_chip(&mut chips, "failed", data.failed.to_string(), theme_red());
append_status_chip(
&mut chips,
"elapsed",
format!("{}s", data.elapsed.as_secs()),
theme_dim(),
);
let text = Paragraph::from_text(Text::new(vec![
Line::new(chips),
Line::new(vec![
Span::styled("command ", Style::default().fg(theme_dim())),
Span::styled(data.verb, Style::default().fg(theme_white())),
]),
Line::styled(
&format!(
"batch {}/{} concurrency {} total {} {}",
data.batch_index, data.total_batches, data.batch_size, data.total, data.unit
),
Style::default().fg(theme_dim()),
),
]));
text.render(frame.buffer(), text_area);
normalize_bg(frame.buffer(), text_area, theme_surface());
let loader = LoaderPlayer::new(LoaderKind::Bar)
.with_size(progress_area.width as usize, 1)
.with_label("thesa".to_string())
.with_unit(data.unit)
.with_fraction(true)
.with_accent(theme_blue())
.with_gradient_colors(vec![theme_blue(), theme_green()], 0.0);
loader.render(
tick,
LoaderProgress::from_counts(completed as u64, data.total.max(1) as u64),
frame.buffer(),
progress_area,
);
normalize_bg(frame.buffer(), progress_area, theme_surface());
}
fn render_command_info_panel(
frame: &mut Frame<'_>,
area: Rect,
title: &str,
lines: Vec<Line>,
color: TuiColor,
) {
let panel = Paragraph::from_text(Text::new(lines)).wrap(WrapMode::Character { trim: false });
frame.render_block_widget(area, panel, knott_block(title, color));
}
fn repo_loader_row(loader: &Loader, tick: usize, name: &str, state: BatchState) -> Line {
let done = matches!(
state,
BatchState::Done | BatchState::Skipped | BatchState::Failed
);
let loader_line = loader.line(tick, LoaderProgress::from_counts(done as u64, 1));
let style = match state {
BatchState::Pending => Style::default().fg(TuiColor::DARK_GRAY),
BatchState::Running => Style::default().fg(TuiColor::MAGENTA),
BatchState::Done => Style::default().fg(TuiColor::CYAN),
BatchState::Skipped => Style::default().fg(TuiColor::YELLOW),
BatchState::Failed => Style::default().fg(TuiColor::RED),
};
let state_label = format!("{:<7}", state.label());
Line::new(vec![
Span::styled(&state_label, style.add_modifier(Modifier::BOLD)),
Span::raw(" "),
Span::raw(&loader_line),
Span::raw(" "),
Span::styled(name, style),
])
}
fn clone_repositories_serial(
repos: &[RepoItem],
output: &Path,
args: &Cli,
ui_rx: &mpsc::Receiver<char>,
ui_state: &mut UiState,
) -> Result<CloneSummary> {
let spinner = Loader::with_config(
LoaderKind::Tqdm,
LoaderConfig::default()
.with_size(54, 1)
.with_label("thesa")
.with_unit("repo")
.with_fraction(true),
);
let mut stdout = io::stdout();
let mut summary = CloneSummary::default();
let total = repos.len();
for (index, repo) in repos.iter().enumerate() {
if ui_state.abort_requested {
break;
}
apply_ui_events(ui_rx, ui_state);
match clone_repository_with_progress(
repo,
output,
args,
&mut stdout,
&spinner,
index + 1,
total,
) {
Ok(CloneStatus::Cloned) => summary.cloned += 1,
Ok(CloneStatus::Skipped) => summary.skipped += 1,
Err(err) => summary.failed.push(format!("{}: {}", repo.full_name, err)),
}
}
Ok(summary)
}
fn clone_repositories_batched(
repos: &[RepoItem],
output: &Path,
args: &Cli,
ui_rx: &mpsc::Receiver<char>,
ui_state: &mut UiState,
) -> Result<CloneSummary> {
let batch_size = args.concurrency.max(1);
let total = repos.len();
let mut summary = CloneSummary::default();
let mut batch_index = 1;
let total_batches = total.div_ceil(batch_size);
let mut stop_after_batch = false;
while batch_index <= total {
let batch_end = (batch_index + batch_size - 1).min(total);
let batch = &repos[batch_index - 1..batch_end];
let batch_positions: Vec<String> =
batch.iter().map(|repo| repo.full_name.clone()).collect();
let (tx, rx) = mpsc::channel::<(usize, BatchEvent)>();
println!(
"Starting batch {}/{} ({})...",
(batch_index - 1) / batch_size + 1,
total_batches,
batch.len()
);
let mut state = vec![BatchState::Pending; batch.len()];
apply_ui_events(ui_rx, ui_state);
if ui_state.abort_requested {
break;
}
let mut handles = Vec::new();
for (offset, repo) in batch.iter().enumerate() {
let repo = repo.clone();
let output = output.to_path_buf();
let args = args.clone();
let position = batch_index + offset;
let tx = tx.clone();
handles.push((
position,
repo.full_name.clone(),
thread::spawn(move || {
let started = tx.send((offset, BatchEvent::Started));
if started.is_err() {
return;
}
let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
clone_repository(&repo, &output, &args)
}));
let event = match outcome {
Ok(Ok(CloneStatus::Cloned)) => BatchEvent::Result(BatchResult::Done),
Ok(Ok(CloneStatus::Skipped)) => BatchEvent::Result(BatchResult::Skipped),
Ok(Err(err)) => BatchEvent::Result(BatchResult::Error(err.to_string())),
Err(_) => BatchEvent::Result(BatchResult::Error(
"worker thread panicked while cloning".to_string(),
)),
};
let _ = tx.send((offset, event));
}),
));
}
drop(tx);
let mut completed = 0;
while completed < batch.len() {
let mut progress_updated = false;
apply_ui_events(ui_rx, ui_state);
if ui_state.abort_requested {
stop_after_batch = true;
}
while let Ok((offset, event)) = rx.try_recv() {
match event {
BatchEvent::Started => {
state[offset] = BatchState::Running;
progress_updated = true;
}
BatchEvent::Result(BatchResult::Done) => {
state[offset] = BatchState::Done;
summary.cloned += 1;
completed += 1;
progress_updated = true;
}
BatchEvent::Result(BatchResult::Skipped) => {
state[offset] = BatchState::Skipped;
summary.skipped += 1;
completed += 1;
progress_updated = true;
}
BatchEvent::Result(BatchResult::Error(message)) => {
state[offset] = BatchState::Failed;
summary
.failed
.push(format!("{}: {}", batch_positions[offset], message));
completed += 1;
progress_updated = true;
}
}
}
if progress_updated {
render_batch_state(
&mut io::stdout(),
(batch_index - 1) / batch_size + 1,
total_batches,
batch_positions.as_slice(),
state.as_slice(),
ui_state.show_details,
&mut ui_state.panel_lines,
batch.len(),
total,
)?;
}
if completed < batch.len() {
thread::sleep(Duration::from_millis(100));
}
}
for (position, repo_name, handle) in handles {
if handle.join().is_err() {
summary.failed.push(format!(
"{repo_name} ({position}/{total}): worker thread panicked"
));
}
}
if stop_after_batch {
break;
}
batch_index = batch_end + 1;
}
Ok(summary)
}
fn apply_ui_events(ui_rx: &mpsc::Receiver<char>, ui_state: &mut UiState) {
while let Ok(command) = ui_rx.try_recv() {
match command {
't' | 'T' => {
ui_state.show_details = !ui_state.show_details;
}
'q' | 'Q' => {
ui_state.abort_requested = true;
}
_ => {}
}
}
}
fn render_batch_state(
stdout: &mut io::Stdout,
batch_number: usize,
total_batches: usize,
names: &[String],
state: &[BatchState],
show_details: bool,
panel_lines: &mut usize,
batch_size: usize,
total_repos: usize,
) -> io::Result<()> {
if !stdout.is_terminal() {
return Ok(());
}
let mut done = 0;
let mut skipped = 0;
let mut failed = 0;
let mut running = 0;
let mut pending = 0;
for entry in state {
match entry {
BatchState::Done => done += 1,
BatchState::Skipped => skipped += 1,
BatchState::Failed => failed += 1,
BatchState::Running => running += 1,
BatchState::Pending => pending += 1,
}
}
let completed = done + skipped + failed;
let loader = Loader::with_config(
LoaderKind::Pulse,
LoaderConfig::default()
.with_size(58, 1)
.with_label("thesa")
.with_unit("repo")
.with_fraction(true)
.with_gradient(thesa_gradient(), GradientDirection::Horizontal),
);
let loader_line = loader
.frame(
completed + running,
LoaderProgress::from_counts(completed as u64, batch_size as u64),
)
.to_ansi_string();
let cyan = AislingColor::rgb(0, 255, 210);
let violet = AislingColor::rgb(142, 92, 255);
let amber = AislingColor::rgb(255, 196, 92);
let rose = AislingColor::rgb(255, 80, 210);
let slate = AislingColor::rgb(132, 150, 168);
let mut lines = vec![
paint(
"+--------------------------------------------------------------------+",
violet,
),
format!(
"| {} {} | {} |",
paint_bold("THESA", cyan),
paint("public-repo collector", slate),
paint(
format!("batch {batch_number}/{total_batches} total {total_repos}"),
amber,
)
),
paint(
"+--------------------------------------------------------------------+",
violet,
),
format!("| {} |", loader_line.trim_end()),
format!(
"| {} {} {} {} {} {} {} {} {} {} |",
paint("done", cyan),
done,
paint("skip", amber),
skipped,
paint("fail", rose),
failed,
paint("run", violet),
running,
paint("wait", slate),
pending,
),
format!(
"| {} |",
paint(
"keys: t + Enter toggles detail, q + Enter stops after batch",
slate
)
),
paint(
"+--------------------------------------------------------------------+",
violet,
),
];
if show_details {
for (name, repo_state) in names.iter().zip(state.iter()) {
lines.push(format!(" {} {}", repo_state.icon(), name));
}
} else {
lines.push(paint(
" detail hidden; press t + Enter to show repos",
slate,
));
}
lines.push(String::new());
if *panel_lines > 0 {
write!(stdout, "\x1b[{}A", *panel_lines)?;
}
let written_lines = (*panel_lines).max(lines.len());
for index in 0..written_lines {
let line = lines.get(index).map(String::as_str).unwrap_or("");
write!(stdout, "\x1b[2K\r{line}\n")?;
}
*panel_lines = written_lines;
stdout.flush()?;
Ok(())
}
#[derive(Debug, Clone)]
enum BatchResult {
Done,
Skipped,
Error(String),
}
#[derive(Debug, Clone)]
enum BatchEvent {
Started,
Result(BatchResult),
}
impl BatchState {
fn label(self) -> &'static str {
match self {
BatchState::Pending => "queued",
BatchState::Running => "pulling",
BatchState::Done => "done",
BatchState::Skipped => "skip",
BatchState::Failed => "failed",
}
}
fn icon(self) -> &'static str {
match self {
BatchState::Pending => " [ ]",
BatchState::Running => "[-x]",
BatchState::Done => " [*]",
BatchState::Skipped => " [/]",
BatchState::Failed => " [!]",
}
}
}
fn parse_target(input: &str) -> Result<Target> {
let cleaned = input.trim().trim_end_matches(".git");
if cleaned.is_empty() {
return Err(GitArchiverError::InvalidTarget(input.to_string()));
}
if let Some(path) = cleaned.strip_prefix("https://github.com/") {
return parse_owner_or_repo_path(path);
}
if let Some(path) = cleaned.strip_prefix("http://github.com/") {
return parse_owner_or_repo_path(path);
}
if let Some(path) = cleaned.strip_prefix("git@github.com:") {
return parse_owner_or_repo_path(path);
}
parse_owner_or_repo_path(cleaned)
}
fn parse_owner_or_repo_path(input: &str) -> Result<Target> {
let parts: Vec<&str> = input
.trim_matches('/')
.split('/')
.filter(|part| !part.is_empty())
.collect();
if parts.is_empty() {
return Err(GitArchiverError::InvalidTarget(input.to_string()));
}
if parts.len() == 1 {
return Ok(Target::Owner(parts[0].to_string()));
}
Ok(Target::Repo {
owner: parts[0].to_string(),
repo: parts[1].to_string(),
})
}
fn default_output_path(target: &Target) -> PathBuf {
let slug = match target {
Target::Owner(owner) => owner.clone(),
Target::Repo { repo, .. } => repo.clone(),
};
PathBuf::from("./archives").join(slug)
}
fn prompt_target(mode: Mode, provider: Option<ModelProvider>) -> Result<String> {
loop {
let value = match mode {
Mode::Repos => prompt_line("GitHub target [owner or owner/repo]: ")?,
Mode::Models => match provider.unwrap_or(ModelProvider::Hf) {
ModelProvider::Hf => prompt_line("Model target [org, org/model, top, latest]: ")?,
ModelProvider::Ollama => {
prompt_line("Model target [name, keyword, top, latest]: ")?
}
ModelProvider::Civitai => prompt_line("Model target [id, keyword, top, latest]: ")?,
},
};
if value.is_empty() {
println!("Target is required.");
continue;
}
let ok = match mode {
Mode::Repos => parse_target(&value).is_ok(),
Mode::Models => {
parse_model_target_for_provider(&value, provider.unwrap_or(ModelProvider::Hf))
.is_ok()
}
};
if ok {
return Ok(value);
}
match mode {
Mode::Repos => println!(
"Invalid target. Examples: torvalds, torvalds/linux, https://github.com/torvalds/linux"
),
Mode::Models => match provider.unwrap_or(ModelProvider::Hf) {
ModelProvider::Hf => println!(
"Invalid target. Examples: meta-llama, meta-llama/Llama-2-7b, top, latest"
),
ModelProvider::Ollama => {
println!("Invalid target. Examples: llama3, llama3:8b, latest")
}
ModelProvider::Civitai => {
println!("Invalid target. Examples: 827184, realistic-vision-v6, top, latest")
}
},
}
}
}
fn prompt_line(prompt: &str) -> Result<String> {
print!("{prompt}");
io::stdout().flush()?;
let mut value = String::new();
io::stdin()
.read_line(&mut value)
.map_err(GitArchiverError::Io)?;
Ok(value.trim().to_string())
}
fn prompt_mode() -> Result<Mode> {
println!("Archive mode:");
println!(" 1) repos - GitHub repositories");
println!(" 2) models - Hugging Face, Ollama, or CivitAI models");
let value = prompt_line("Select [1/2 or r/m, default: repos]: ")?;
match value.as_str() {
"2" | "m" | "models" => Ok(Mode::Models),
_ => Ok(Mode::Repos),
}
}
fn prompt_model_provider(current: ModelProvider) -> Result<ModelProvider> {
println!("Model provider:");
println!(" 1) hf - Hugging Face models");
println!(" 2) ollama - local Ollama models");
println!(" 3) civitai - CivitAI models");
let value = prompt_line(&format!("Select [1/2/3 or h/o/c, default: {current}]: "))?;
match value.as_str() {
"2" | "o" | "ollama" => Ok(ModelProvider::Ollama),
"3" | "c" | "civitai" => Ok(ModelProvider::Civitai),
"1" | "h" | "hf" | "huggingface" | "hugging-face" => Ok(ModelProvider::Hf),
"" => Ok(current),
_ => {
println!("Invalid provider; keeping {current}.");
Ok(current)
}
}
}
fn prompt_hf_token() -> Result<Option<String>> {
println!("Hugging Face API requires a token for model access.");
println!("Get one at https://huggingface.co/settings/tokens");
let value = prompt_line("HF token (blank to skip, will rate-limit): ")?;
if value.is_empty() {
Ok(None)
} else {
Ok(Some(value))
}
}
fn prompt_hf_token_tui(terminal: &mut Terminal) -> Result<Option<String>> {
let mut input = String::new();
let mut frame_index = 0usize;
let status = "Hugging Face API requires a token for model access".to_string();
let mut show_plain = false;
loop {
draw_full_frame(terminal, |frame| {
draw_hf_token_tui(frame, &input, &status, frame_index, show_plain);
})?;
frame_index = frame_index.wrapping_add(1);
if !event::poll(Duration::from_millis(50))? {
continue;
}
let Event::Key(key) = event::read()? else {
continue;
};
if key.kind != KeyEventKind::Press {
continue;
}
match key.code {
KeyCode::Esc => return Ok(None),
KeyCode::Enter => {
if input.is_empty() {
return Ok(None);
}
return Ok(Some(input));
}
KeyCode::Backspace => {
input.pop();
}
KeyCode::Char(ch) => {
input.push(ch);
show_plain = true;
}
_ => {}
}
}
}
fn prepare_output_dir(output: &Path) -> Result<PathBuf> {
if output.exists() && !output.is_dir() {
return Err(GitArchiverError::OutputNotDirectory(
output.display().to_string(),
));
}
fs::create_dir_all(output)?;
Ok(output.to_path_buf())
}
fn with_auth(
mut request: reqwest::blocking::RequestBuilder,
token: Option<&str>,
) -> reqwest::blocking::RequestBuilder {
request = request.header("Accept", "application/vnd.github+json");
if let Some(token) = token {
request = request.bearer_auth(token);
}
request
}
fn parse_api_error(response: Response) -> GitArchiverError {
let status = response.status();
let body = response
.text()
.unwrap_or_else(|_| "<unable to read body>".to_string());
GitArchiverError::ApiError {
status: status.as_u16(),
body,
}
}
fn list_repos_for_owner(
client: &Client,
owner: &str,
token: Option<&str>,
) -> Result<Vec<RepoItem>> {
list_repos_with_scope(client, "users", owner, token)
}
fn list_repos_for_org(client: &Client, org: &str, token: Option<&str>) -> Result<Vec<RepoItem>> {
list_repos_with_scope(client, "orgs", org, token)
}
fn list_repos_with_scope(
client: &Client,
scope: &str,
entity: &str,
token: Option<&str>,
) -> Result<Vec<RepoItem>> {
let mut page = 1usize;
let mut repos = Vec::new();
loop {
let url = format!(
"{}/{}/{}/repos?type=public&per_page={}&page={}",
GITHUB_API, scope, entity, DEFAULT_PAGE_SIZE, page
);
let response = with_auth(client.get(&url), token).send()?;
let status = response.status();
if status == StatusCode::NOT_FOUND {
return Err(GitArchiverError::NotFound(entity.to_string()));
}
if !status.is_success() {
return Err(parse_api_error(response));
}
let batch: Vec<RepoResponse> = response.json()?;
let count = batch.len();
repos.extend(batch.into_iter().filter_map(|repo| {
if repo.private {
return None;
}
Some(RepoItem {
name: repo.name,
full_name: repo.full_name,
clone_url: repo.clone_url,
})
}));
if count < DEFAULT_PAGE_SIZE {
break;
}
page = page.saturating_add(1);
}
Ok(repos)
}
fn fetch_single_repo(
client: &Client,
owner: &str,
repo: &str,
token: Option<&str>,
) -> Result<RepoItem> {
let url = format!("{}/repos/{}/{}", GITHUB_API, owner, repo);
let response = with_auth(client.get(&url), token).send()?;
if response.status() == StatusCode::NOT_FOUND {
return Err(GitArchiverError::NotFound(format!("{owner}/{repo}")));
}
if !response.status().is_success() {
return Err(parse_api_error(response));
}
let data: RepoResponse = response.json()?;
if data.private {
return Err(GitArchiverError::PrivateRepo(data.full_name));
}
Ok(RepoItem {
name: data.name,
full_name: data.full_name,
clone_url: data.clone_url,
})
}
fn clone_repository(repo: &RepoItem, output: &Path, args: &Cli) -> Result<CloneStatus> {
match prepare_clone(repo, output, args)? {
CloneAction::Skipped => Ok(CloneStatus::Skipped),
CloneAction::Running(mut child) => {
let status = child.wait()?;
if status.success() {
return Ok(CloneStatus::Cloned);
}
let code = status
.code()
.map_or_else(|| "signal/unknown".to_string(), |code| code.to_string());
Err(GitArchiverError::CloneFailed {
repo: repo.full_name.clone(),
message: format!("exit code {code}"),
})
}
}
}
fn clone_repository_with_progress(
repo: &RepoItem,
output: &Path,
args: &Cli,
stdout: &mut io::Stdout,
spinner: &Loader,
position: usize,
total: usize,
) -> Result<CloneStatus> {
let mut child = match prepare_clone(repo, output, args)? {
CloneAction::Skipped => {
write!(stdout, "\r\x1b[2K")?;
println!("[skip] {} ({}/{}) exists", repo.full_name, position, total);
return Ok(CloneStatus::Skipped);
}
CloneAction::Running(child) => child,
};
let mut tick = 0usize;
loop {
if let Some(status) = child.try_wait()? {
if status.success() {
write!(stdout, "\r\x1b[2K")?;
println!("[done] {} ({}/{})", repo.full_name, position, total);
return Ok(CloneStatus::Cloned);
}
let code = status
.code()
.map_or_else(|| "signal/unknown".to_string(), |code| code.to_string());
return Err(GitArchiverError::CloneFailed {
repo: repo.full_name.clone(),
message: format!("exit code {code}"),
});
}
let progress = LoaderProgress::from_counts(position as u64, total as u64);
let frame = spinner.line(tick, progress);
let status = format!("[{}/{}] cloning {}", position, total, repo.full_name);
write!(stdout, "\r\x1b[2K{} {}", frame, status)?;
stdout.flush()?;
tick = tick.saturating_add(1);
thread::sleep(Duration::from_millis(120));
}
}
fn prepare_clone(repo: &RepoItem, output: &Path, args: &Cli) -> Result<CloneAction> {
let destination = output.join(&repo.name);
if destination.exists() {
if args.skip_existing {
return Ok(CloneAction::Skipped);
}
if destination.is_dir() {
remove_dir_all(&destination)?;
} else {
remove_file(&destination)?;
}
}
let child = spawn_git_clone(repo, output, args)?;
Ok(CloneAction::Running(child))
}
fn spawn_git_clone(repo: &RepoItem, output: &Path, args: &Cli) -> Result<Child> {
let mut command_args = vec!["clone".to_string()];
if let Some(depth) = args.depth {
command_args.push("--depth".to_string());
command_args.push(depth.to_string());
}
command_args.push(repo.clone_url.clone());
command_args.push(repo.name.clone());
match Command::new("git")
.current_dir(output)
.args(&command_args)
.stdout(Stdio::null())
.stderr(Stdio::null())
.spawn()
{
Ok(child) => Ok(child),
Err(error) if error.kind() == io::ErrorKind::NotFound => Err(GitArchiverError::MissingGit),
Err(error) => Err(error.into()),
}
}
fn paint(text: impl AsRef<str>, color: AislingColor) -> String {
let color = fg(color);
format!("{color}{}\x1b[0m", text.as_ref())
}
fn paint_bold(text: impl AsRef<str>, color: AislingColor) -> String {
let color = fg(color);
format!("\x1b[1m{color}{}\x1b[0m", text.as_ref())
}
fn fg(color: AislingColor) -> String {
format!("\x1b[38;2;{};{};{}m", color.r, color.g, color.b)
}
fn thesa_gradient() -> Gradient {
Gradient::new(
vec![
AislingColor::rgb(0, 255, 210),
AislingColor::rgb(142, 92, 255),
AislingColor::rgb(255, 80, 210),
AislingColor::rgb(255, 196, 92),
],
36,
)
}
fn parse_indices(input: &str, max_len: usize) -> std::result::Result<Vec<usize>, String> {
let mut unique = Vec::new();
for token in input.split(',') {
let token = token.trim();
if token.is_empty() {
continue;
}
if let Some((start, end)) = token.split_once('-') {
let start = start
.trim()
.parse::<usize>()
.map_err(|_| "Invalid start of range".to_string())?;
let end = end
.trim()
.parse::<usize>()
.map_err(|_| "Invalid end of range".to_string())?;
if start == 0 || end == 0 || start > end {
return Err(
"Invalid range: indexes are 1-based and must be start <= end".to_string(),
);
}
let (start, end) = (start - 1, end - 1);
if end >= max_len {
return Err(format!(
"Range end {input} is above result count {max_len}",
input = end + 1,
));
}
for index in start..=end {
if unique.contains(&index) {
continue;
}
unique.push(index);
}
continue;
}
let index = token
.parse::<usize>()
.map_err(|_| format!("Invalid index '{token}'"))?;
if index == 0 {
return Err("Indexes are 1-based, use 1 or greater".to_string());
}
let index = index - 1;
if index >= max_len {
return Err(format!(
"Index {} is above result count {}",
index + 1,
max_len
));
}
if !unique.contains(&index) {
unique.push(index);
}
}
if unique.is_empty() {
return Err("No valid indexes provided".to_string());
}
Ok(unique)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parse_model_target_for_provider_hf() {
let hf_target = parse_model_target_for_provider("meta-llama/Llama-2-7b", ModelProvider::Hf)
.expect("valid hf target");
match hf_target {
ModelTarget::Model(model) => assert_eq!(model, "meta-llama/Llama-2-7b"),
ModelTarget::Org(_) => panic!("expected model target"),
ModelTarget::Top | ModelTarget::Latest => panic!("unexpected preset target"),
}
let hf_preset = parse_model_target_for_provider("top", ModelProvider::Hf)
.expect("valid hf preset target");
assert!(matches!(hf_preset, ModelTarget::Top));
let hf_latest = parse_model_target_for_provider("latest", ModelProvider::Hf)
.expect("valid hf latest target");
assert!(matches!(hf_latest, ModelTarget::Latest));
let hf_url = parse_model_target_for_provider(
"https://huggingface.co/openai-community/gpt2",
ModelProvider::Hf,
)
.expect("valid hf url target");
match hf_url {
ModelTarget::Model(model) => assert_eq!(model, "openai-community/gpt2"),
ModelTarget::Org(_) => panic!("expected model target from URL"),
ModelTarget::Top | ModelTarget::Latest => panic!("unexpected preset target"),
}
}
#[test]
fn parse_model_target_for_provider_civitai() {
let civitai_id = parse_model_target_for_provider("827184", ModelProvider::Civitai)
.expect("valid numeric id");
match civitai_id {
ModelTarget::Model(model) => assert_eq!(model, "827184"),
ModelTarget::Org(_) => panic!("expected numeric model target"),
ModelTarget::Top | ModelTarget::Latest => panic!("unexpected preset target"),
}
let civitai_preset = parse_model_target_for_provider("newest", ModelProvider::Civitai)
.expect("valid civitai preset target");
assert!(matches!(civitai_preset, ModelTarget::Latest));
let civitai_url = parse_model_target_for_provider(
"https://civitai.com/models/827184",
ModelProvider::Civitai,
)
.expect("valid civitai url");
match civitai_url {
ModelTarget::Model(model) => assert_eq!(model, "827184"),
ModelTarget::Org(_) => panic!("expected numeric model target from URL"),
ModelTarget::Top | ModelTarget::Latest => panic!("unexpected preset target"),
}
let civitai_keyword =
parse_model_target_for_provider("realistic-vision", ModelProvider::Civitai)
.expect("valid keyword target");
match civitai_keyword {
ModelTarget::Org(org) => assert_eq!(org, "realistic-vision"),
ModelTarget::Model(_) => panic!("expected keyword treated as org"),
ModelTarget::Top | ModelTarget::Latest => panic!("unexpected preset target"),
}
}
#[test]
fn parse_model_target_for_provider_ollama() {
let ollama_keyword = parse_model_target_for_provider("llama3", ModelProvider::Ollama)
.expect("valid ollama target");
match ollama_keyword {
ModelTarget::Org(org) => assert_eq!(org, "llama3"),
ModelTarget::Model(_) => panic!("expected org-style target for keyword"),
ModelTarget::Top | ModelTarget::Latest => panic!("unexpected preset target"),
}
let ollama_preset = parse_model_target_for_provider("trending", ModelProvider::Ollama)
.expect("valid ollama preset target");
assert!(matches!(ollama_preset, ModelTarget::Top));
}
#[test]
fn format_model_size_and_truncate_to_width() {
assert_eq!(format_model_size(None), "n/a");
assert_eq!(format_model_size(Some(512)), "512 B");
assert_eq!(format_model_size(Some(2048)), "2.0KB");
assert_eq!(format_model_size(Some(4_000_000)), "3.8MB");
assert_eq!(truncate_to_width("hello", 0), "");
assert_eq!(truncate_to_width("hello", 10), "hello");
assert_eq!(truncate_to_width("hello", 3), "hel");
assert_eq!(truncate_to_width("hello", 4), "h...");
assert_eq!(truncate_to_width("hello", 5), "hello");
}
}