use std::collections::BTreeSet;
#[cfg(test)]
use std::fs;
use std::fs::{remove_dir_all, remove_file};
mod cli;
mod constants;
mod datasets;
mod domain;
mod error;
mod manifest;
mod models;
mod repos;
mod selection;
mod sites;
mod target;
mod terminal;
mod types;
use std::io::{self, IsTerminal, Write};
use std::path::{Path, PathBuf};
use std::process::{Child, Command, Stdio};
use std::sync::mpsc;
use std::thread;
use std::time::{Duration, Instant};
#[cfg(test)]
use std::time::{SystemTime, UNIX_EPOCH};
use aisling::{
Color as AislingColor, EffectKind, Gradient, GradientDirection, Loader, LoaderConfig,
LoaderKind, LoaderProgress,
};
use clap::Parser;
use cli::{ArchiveCommands, Cli, Commands};
#[cfg(test)]
use constants::THESA_FORMAT_VERSION;
use constants::{
BATCH_SIZE_CHOICES, DEFAULT_PAGE_SIZE, GITHUB_API, MODE_OPTION_COUNT, SITE_OPTION_COUNT,
THESA_METADATA_DIR,
};
use crossterm::event::{
self, Event, KeyCode, KeyEventKind, MouseButton, MouseEvent, MouseEventKind,
};
use datasets::{
parse_dataset_target_for_provider, run_baseforge_archive_command, run_datasets_mode,
};
use domain::{
BatchState, CloneAction, CloneStatus, CloneSummary, ModelItem, RepoItem, RepoResponse, UiState,
};
use error::{Result, ThesaError};
#[cfg(test)]
use manifest::{ArchiveManifest, build_archive_manifest, verify_archive};
use manifest::{prepare_output_dir, run_verify_command, write_archive_manifest};
use models::{
ModelTarget, download_model, fetch_models_for_target, parse_model_target_for_provider,
};
#[cfg(test)]
use repos::git_refresh_owner;
use repos::run_git_archive_refresh;
use reqwest::StatusCode;
use reqwest::blocking::{Client, Response};
use scrin::{
Color as TuiColor, Rect, Terminal,
core::buffer::Buffer,
effects::{EffectPlayer, LoaderPlayer},
layout::{Constraint, Direction, Layout},
style::{Modifier, Style},
widgets::{
Alignment, Block, Clear, Line, List, ListItem, ListState, Paragraph, Span, Text, Widget,
WrapMode, block::BorderStyle,
},
};
use selection::parse_indices;
use siteforge::ArchiveOutputProfile;
#[cfg(test)]
use siteforge::ExportFormat as SiteforgeExportFormat;
#[cfg(test)]
use sites::{load_site_seed_urls, parse_site_headers};
use sites::{
parse_site_url, print_site_archive_dry_run, run_site_archive, run_siteforge_diagnostics,
run_siteforge_export, run_siteforge_inspect, run_siteforge_list, run_siteforge_pack,
run_siteforge_resume, run_siteforge_search, run_siteforge_verify, run_sites_mode,
site_archive_profile_label, site_render_mode_label,
};
use target::{Target, default_output_path, parse_target};
use terminal::{FrameRenderExt, RenderFrame as Frame, TerminalSession, draw_full_frame};
#[cfg(test)]
use types::SiteArchiveProfile;
use types::{Mode, ModelProvider, SiteTuiOptions, TuiScreen};
fn main() {
if let Err(err) = run() {
eprintln!("error: {err}");
std::process::exit(1);
}
}
fn run() -> Result<()> {
let mut args = Cli::parse();
if let Some(command) = args.command.clone() {
return match command {
Commands::Archive { dry_run, source } => {
run_archive_command(source, args.dry_run || dry_run)
}
Commands::Verify { archive, json } => run_verify_command(&archive, json),
};
}
if args.token.is_none() {
args.token = std::env::var("GITHUB_TOKEN").ok();
}
if args.hf_token.is_none() {
args.hf_token = std::env::var("HF_TOKEN").ok();
}
if args.civitai_token.is_none() {
args.civitai_token = std::env::var("CIVITAI_TOKEN").ok();
}
let interactive = args.tui || args.target.is_none();
if args.tui && !io::stdout().is_terminal() {
return Err(ThesaError::Message(
"--tui requires an interactive terminal".to_string(),
));
}
let mut terminal_session = if interactive && io::stdout().is_terminal() {
Some(TerminalSession::new()?)
} else {
None
};
if interactive {
let mut screen = TuiScreen::Mode;
let mut site_options = SiteTuiOptions::default();
loop {
match screen {
TuiScreen::Mode => {
if let Some(session) = terminal_session.as_mut() {
match prompt_mode_tui(&mut session.terminal)? {
Some(mode) => {
args.mode = mode;
if mode == Mode::Models {
screen = TuiScreen::Provider;
} else {
screen = TuiScreen::Target;
}
}
None => return Ok(()),
}
} else {
args.mode = prompt_mode()?;
if args.mode == Mode::Models {
screen = TuiScreen::Provider;
} else {
screen = TuiScreen::Target;
}
}
}
TuiScreen::Provider => {
if let Some(session) = terminal_session.as_mut() {
match prompt_model_provider_tui(&mut session.terminal, args.provider)? {
Some(provider) => {
args.provider = provider;
screen = TuiScreen::Target;
}
None => screen = TuiScreen::Mode,
}
} else {
args.provider = prompt_model_provider(args.provider)?;
screen = TuiScreen::Target;
}
}
TuiScreen::Target => {
if args.target.is_some() {
screen = if args.mode == Mode::Sites {
TuiScreen::SiteOptions
} else {
TuiScreen::HfToken
};
continue;
}
if let Some(session) = terminal_session.as_mut() {
match prompt_target_tui_inner(
&mut session.terminal,
args.mode,
Some(args.provider),
)? {
Some(target) => {
args.target = Some(target);
screen = if args.mode == Mode::Sites {
TuiScreen::SiteOptions
} else {
TuiScreen::HfToken
};
}
None => {
if args.mode == Mode::Models {
screen = TuiScreen::Provider;
} else {
screen = TuiScreen::Mode;
}
}
}
} else {
args.target = Some(prompt_target(args.mode, Some(args.provider))?);
screen = if args.mode == Mode::Sites {
TuiScreen::SiteOptions
} else {
TuiScreen::HfToken
};
}
}
TuiScreen::SiteOptions => {
if let Some(session) = terminal_session.as_mut() {
match prompt_site_options_tui(
&mut session.terminal,
args.target.as_deref().unwrap_or_default(),
site_options,
)? {
Some(options) => {
site_options = options;
break;
}
None => screen = TuiScreen::Target,
}
} else {
site_options = prompt_site_options(site_options)?;
break;
}
}
TuiScreen::HfToken => {
if (args.mode == Mode::Models
|| (args.mode == Mode::Datasets
&& args.dataset_provider == types::DatasetProvider::Hf))
&& args.hf_token.is_none()
&& (args.mode == Mode::Datasets || args.provider == ModelProvider::Hf)
{
if let Some(session) = terminal_session.as_mut() {
match prompt_hf_token_tui(&mut session.terminal)? {
Some(token) => args.hf_token = Some(token),
None => {
screen = TuiScreen::Target;
continue;
}
}
} else if io::stdout().is_terminal() {
args.hf_token = prompt_hf_token()?;
}
}
break;
}
}
}
if args.mode == Mode::Sites {
return run_sites_mode(&mut args, &mut terminal_session, Some(site_options));
}
}
let client = Client::builder()
.user_agent("thesa")
.build()
.map_err(ThesaError::Request)?;
match args.mode {
Mode::Repos => run_repos_mode(&mut args, &client, &mut terminal_session),
Mode::Models => run_models_mode(&mut args, &client, &mut terminal_session),
Mode::Datasets => run_datasets_mode(&mut args, &mut terminal_session),
Mode::Sites => run_sites_mode(&mut args, &mut terminal_session, None),
}
}
fn run_repos_mode(
args: &mut Cli,
client: &Client,
terminal_session: &mut Option<TerminalSession>,
) -> Result<()> {
let target = parse_target(args.target.as_deref().unwrap_or_default())?;
if args.output == Path::new("./archives") {
args.output = default_output_path(&target);
}
let output = prepare_output_dir(&args.output)?;
let mut repos = if let Some(session) = terminal_session.as_mut() {
fetch_repositories_tui(
&mut session.terminal,
client,
&target,
args.token.as_deref(),
)?
} else {
fetch_repositories_for_target(client, &target, args.token.as_deref())?
};
let interactive = args.tui || args.target.is_some();
if interactive {
repos = match &target {
Target::Owner(_) if terminal_session.is_some() => {
let session = terminal_session.as_mut().expect("checked above");
pick_repositories_tui_inner(&mut session.terminal, &repos, &mut args.concurrency)?
}
Target::Owner(_) => pick_repositories_interactively(&repos)?,
Target::Repo { .. } => repos,
};
} else if let Some(pattern) = args.filter.as_deref() {
repos.retain(|repo| repo.name.contains(pattern));
}
if repos.is_empty() {
*terminal_session = None;
println!("No repositories matched your request.");
return Ok(());
}
if args.dry_run {
*terminal_session = None;
println!("Dry run: {} repositories would be processed", repos.len());
for repo in &repos {
println!(" - {}", repo.full_name);
}
return Ok(());
}
let mut ui_state = UiState {
show_details: true,
..UiState::default()
};
let summary = if let Some(session) = terminal_session.as_mut() {
clone_repositories_scrin_inner(&mut session.terminal, &repos, &output, args, &mut ui_state)?
} else if io::stdout().is_terminal() {
clone_repositories_scrin(&repos, &output, args, &mut ui_state)?
} else {
let (ui_tx, ui_rx) = mpsc::channel::<char>();
let _ui_thread = thread::spawn(move || {
let stdin = io::stdin();
loop {
let mut line = String::new();
if stdin.read_line(&mut line).is_err() || line.is_empty() {
break;
}
let mut chars = line.chars().filter(|c| !c.is_whitespace());
if let Some(command) = chars.next() {
if ui_tx.send(command).is_err() {
break;
}
}
}
});
if args.concurrency > 1 {
clone_repositories_batched(&repos, &output, args, &ui_rx, &mut ui_state)?
} else {
clone_repositories_serial(&repos, &output, args, &ui_rx, &mut ui_state)?
}
};
*terminal_session = None;
if ui_state.abort_requested {
println!("Aborted by user input");
}
let manifest = write_archive_manifest(
&output,
"github",
&target.archive_target_id(),
!ui_state.abort_requested && summary.failed.is_empty(),
)?;
println!(
"\nCompleted: {} cloned, {} skipped, {} failed",
summary.cloned,
summary.skipped,
summary.failed.len()
);
println!(
"Manifest: {}/manifest.json ({} files, {} bytes)",
THESA_METADATA_DIR, manifest.archive.file_count, manifest.archive.total_bytes
);
if !summary.failed.is_empty() {
eprintln!("Failures:");
for item in summary.failed {
eprintln!(" - {item}");
}
return Err(ThesaError::Message("some repositories failed".to_string()));
}
Ok(())
}
fn run_models_mode(
args: &mut Cli,
client: &Client,
terminal_session: &mut Option<TerminalSession>,
) -> Result<()> {
let target =
parse_model_target_for_provider(args.target.as_deref().unwrap_or_default(), args.provider)?;
if args.output == Path::new("./archives") {
args.output = PathBuf::from("./archives").join("models");
}
let output = prepare_output_dir(&args.output)?;
let models = if let Some(session) = terminal_session.as_mut() {
fetch_models_tui(&mut session.terminal, client, &target, args)?
} else {
fetch_models_for_target(client, &target, args)?
};
let picked = if let Some(session) = terminal_session.as_mut() {
pick_models_tui_inner(&mut session.terminal, &models, &mut args.concurrency)?
} else if io::stdout().is_terminal() {
pick_models_interactively(&models)?
} else {
models
};
if picked.is_empty() {
*terminal_session = None;
println!("No models matched your request.");
return Ok(());
}
if args.dry_run {
*terminal_session = None;
println!("Dry run: {} models would be downloaded", picked.len());
for model in &picked {
println!(
" - [{}] {} (downloads: {}, size: {})",
model.provider.short_label(),
model.id,
format_downloads(model.downloads),
format_model_size(model.size),
);
}
return Ok(());
}
let mut ui_state = UiState {
show_details: true,
..UiState::default()
};
let summary = if let Some(session) = terminal_session.as_mut() {
download_models_scrin_inner(&mut session.terminal, &picked, &output, args, &mut ui_state)?
} else if io::stdout().is_terminal() {
download_models_scrin(&picked, &output, args, &mut ui_state)?
} else {
download_models_serial(&picked, &output, args, &mut ui_state)?
};
*terminal_session = None;
if ui_state.abort_requested {
println!("Aborted by user input");
}
let manifest = write_archive_manifest(
&output,
&format!("model:{}", args.provider),
args.target.as_deref().unwrap_or("models"),
!ui_state.abort_requested && summary.failed.is_empty(),
)?;
println!(
"\nCompleted: {} downloaded, {} skipped, {} failed",
summary.cloned,
summary.skipped,
summary.failed.len()
);
println!(
"Manifest: {}/manifest.json ({} files, {} bytes)",
THESA_METADATA_DIR, manifest.archive.file_count, manifest.archive.total_bytes
);
if !summary.failed.is_empty() {
eprintln!("Failures:");
for item in summary.failed {
eprintln!(" - {item}");
}
return Err(ThesaError::Message(
"some model downloads failed".to_string(),
));
}
Ok(())
}
fn run_archive_command(source: ArchiveCommands, dry_run: bool) -> Result<()> {
match source {
ArchiveCommands::Refresh(args) => run_git_archive_refresh(&args, dry_run || args.dry_run),
ArchiveCommands::Dataset(args) => run_baseforge_archive_command(&args, dry_run),
ArchiveCommands::Site(args) | ArchiveCommands::Url(args) => {
if dry_run || args.dry_run {
print_site_archive_dry_run(&args)
} else {
run_site_archive(args)
}
}
ArchiveCommands::List(args) => run_siteforge_list(&args),
ArchiveCommands::Inspect(args) => run_siteforge_inspect(&args),
ArchiveCommands::Search(args) => run_siteforge_search(&args),
ArchiveCommands::Export(args) => run_siteforge_export(&args, dry_run),
ArchiveCommands::Pack(args) => run_siteforge_pack(&args, dry_run),
ArchiveCommands::Verify(args) => run_siteforge_verify(&args),
ArchiveCommands::Resume(args) => run_siteforge_resume(&args, dry_run),
ArchiveCommands::Diagnostics(args) => run_siteforge_diagnostics(&args),
}
}
fn prompt_mode_tui(terminal: &mut Terminal) -> Result<Option<Mode>> {
let mut cursor = 0usize;
let mut frame_index = 0usize;
loop {
draw_full_frame(terminal, |frame| {
draw_mode_selector_tui(frame, cursor, frame_index);
})?;
frame_index = frame_index.wrapping_add(1);
if !event::poll(Duration::from_millis(50))? {
continue;
}
match event::read()? {
Event::Mouse(mouse) => {
if let Some(inner) = mode_selector_list_inner(terminal.size()) {
match list_mouse_action(inner, MODE_OPTION_COUNT, cursor, &mouse) {
Some(ListMouseAction::Hover(index)) => cursor = index,
Some(ListMouseAction::Click(index)) => {
cursor = index;
return Ok(Some(mode_from_index(cursor)));
}
Some(ListMouseAction::ScrollUp) => cursor = cursor.saturating_sub(1),
Some(ListMouseAction::ScrollDown) if cursor + 1 < MODE_OPTION_COUNT => {
cursor += 1;
}
Some(ListMouseAction::ScrollDown) | None => {}
}
}
}
Event::Key(key) => {
if key.kind != KeyEventKind::Press {
continue;
}
match key.code {
KeyCode::Esc => return Ok(None),
KeyCode::Up | KeyCode::Left => cursor = cursor.saturating_sub(1),
KeyCode::Down | KeyCode::Right => {
if cursor + 1 < MODE_OPTION_COUNT {
cursor += 1;
}
}
KeyCode::Enter => {
return Ok(Some(mode_from_index(cursor)));
}
KeyCode::Char('r') => return Ok(Some(Mode::Repos)),
KeyCode::Char('m') => return Ok(Some(Mode::Models)),
KeyCode::Char('d') => return Ok(Some(Mode::Datasets)),
KeyCode::Char('s') | KeyCode::Char('u') => return Ok(Some(Mode::Sites)),
_ => {}
}
}
_ => {}
}
}
}
fn mode_from_index(index: usize) -> Mode {
match index {
1 => Mode::Models,
2 => Mode::Datasets,
3 => Mode::Sites,
_ => Mode::Repos,
}
}
fn prompt_model_provider_tui(
terminal: &mut Terminal,
current: ModelProvider,
) -> Result<Option<ModelProvider>> {
let mut cursor = model_provider_index(current);
let mut frame_index = 0usize;
loop {
draw_full_frame(terminal, |frame| {
draw_model_provider_selector_tui(frame, cursor, frame_index);
})?;
frame_index = frame_index.wrapping_add(1);
if !event::poll(Duration::from_millis(50))? {
continue;
}
match event::read()? {
Event::Mouse(mouse) => {
if let Some(inner) = provider_selector_list_inner(terminal.size()) {
match list_mouse_action(inner, 3, cursor, &mouse) {
Some(ListMouseAction::Hover(index)) => cursor = index,
Some(ListMouseAction::Click(index)) => {
cursor = index;
return Ok(Some(model_provider_from_index(cursor)));
}
Some(ListMouseAction::ScrollUp) => cursor = cursor.saturating_sub(1),
Some(ListMouseAction::ScrollDown) if cursor + 1 < 3 => {
cursor += 1;
}
Some(ListMouseAction::ScrollDown) | None => {}
}
}
}
Event::Key(key) => {
if key.kind != KeyEventKind::Press {
continue;
}
match key.code {
KeyCode::Esc => return Ok(None),
KeyCode::Up | KeyCode::Left => cursor = cursor.saturating_sub(1),
KeyCode::Down | KeyCode::Right => {
if cursor + 1 < 3 {
cursor += 1;
}
}
KeyCode::Enter => return Ok(Some(model_provider_from_index(cursor))),
KeyCode::Char('h') | KeyCode::Char('1') => return Ok(Some(ModelProvider::Hf)),
KeyCode::Char('o') | KeyCode::Char('2') => {
return Ok(Some(ModelProvider::Ollama));
}
KeyCode::Char('c') | KeyCode::Char('3') => {
return Ok(Some(ModelProvider::Civitai));
}
_ => {}
}
}
_ => {}
}
}
}
fn model_provider_index(provider: ModelProvider) -> usize {
match provider {
ModelProvider::Hf => 0,
ModelProvider::Ollama => 1,
ModelProvider::Civitai => 2,
}
}
fn model_provider_from_index(index: usize) -> ModelProvider {
match index {
1 => ModelProvider::Ollama,
2 => ModelProvider::Civitai,
_ => ModelProvider::Hf,
}
}
fn prompt_target_tui_inner(
terminal: &mut Terminal,
mode: Mode,
provider: Option<ModelProvider>,
) -> Result<Option<String>> {
let mut input = String::new();
let mut frame_index = 0usize;
let mut status = match mode {
Mode::Repos => "enter GitHub owner or repo name".to_string(),
Mode::Models => match provider.unwrap_or(ModelProvider::Hf) {
ModelProvider::Hf => "enter Hugging Face org, org/model, top, or latest".to_string(),
ModelProvider::Ollama => "enter Ollama model name, top, or latest".to_string(),
ModelProvider::Civitai => "enter CivitAI model id, keyword, top, or latest".to_string(),
},
Mode::Datasets => {
"enter dataset namespace, dataset:<id>, search:<query>, top, or latest".to_string()
}
Mode::Sites => "enter a website URL for Siteforge".to_string(),
};
loop {
draw_full_frame(terminal, |frame| {
draw_target_tui(
frame,
&input,
&status,
frame_index,
mode,
if matches!(mode, Mode::Models) {
provider
} else {
None
},
);
})?;
frame_index = frame_index.wrapping_add(1);
if !event::poll(Duration::from_millis(50))? {
continue;
}
let Event::Key(key) = event::read()? else {
continue;
};
if key.kind != KeyEventKind::Press {
continue;
}
match key.code {
KeyCode::Esc if input.is_empty() => return Ok(None),
KeyCode::Esc => {
input.clear();
status = match mode {
Mode::Repos => "enter GitHub owner or repo name".to_string(),
Mode::Models => match provider.unwrap_or(ModelProvider::Hf) {
ModelProvider::Hf => {
"enter Hugging Face org, org/model, top, or latest".to_string()
}
ModelProvider::Ollama => {
"enter Ollama model name, top, or latest".to_string()
}
ModelProvider::Civitai => {
"enter CivitAI model id, keyword, top, or latest".to_string()
}
},
Mode::Datasets => {
"enter dataset namespace, dataset:<id>, search:<query>, top, or latest"
.to_string()
}
Mode::Sites => "enter a website URL for Siteforge".to_string(),
};
}
KeyCode::Enter => {
let target = input.trim().to_string();
if target.is_empty() {
status = "name is required".to_string();
continue;
}
let parse_result = match mode {
Mode::Repos => parse_target(&target).map(|_| target.clone()),
Mode::Models => parse_model_target_for_provider(
&target,
provider.unwrap_or(ModelProvider::Hf),
)
.map(|_| target.clone()),
Mode::Datasets => {
parse_dataset_target_for_provider(&target, types::DatasetProvider::Hf)
.map(|_| target.clone())
}
Mode::Sites => parse_site_url(&target).map(|_| target.clone()),
};
if parse_result.is_ok() {
return Ok(Some(target));
}
status = match mode {
Mode::Repos => "try octocat or octocat/Spoon-Knife".to_string(),
Mode::Models => match provider.unwrap_or(ModelProvider::Hf) {
ModelProvider::Hf => "try meta-llama or meta-llama/Llama-2-7b".to_string(),
ModelProvider::Ollama => "try llama3, latest, or qwen2.5:7b".to_string(),
ModelProvider::Civitai => {
"try 827184 or top, realistic-vision-v6".to_string()
}
},
Mode::Datasets => {
"try openai, dataset:squad, search:code data, or top".to_string()
}
Mode::Sites => "try https://example.com or http://127.0.0.1:8765/".to_string(),
};
}
KeyCode::Backspace => {
input.pop();
}
KeyCode::Char(ch) => {
input.push(ch);
}
_ => {}
}
}
}
fn prompt_site_options_tui(
terminal: &mut Terminal,
url: &str,
initial: SiteTuiOptions,
) -> Result<Option<SiteTuiOptions>> {
let mut options = initial;
let mut cursor = 0usize;
let mut frame_index = 0usize;
let mut status = "tune Siteforge crawl options, then press Enter".to_string();
loop {
draw_full_frame(terminal, |frame| {
draw_site_options_tui(frame, url, options, cursor, &status, frame_index);
})?;
frame_index = frame_index.wrapping_add(1);
if !event::poll(Duration::from_millis(50))? {
continue;
}
let input_event = event::read()?;
if let Event::Mouse(mouse) = input_event {
if let Some(inner) = site_options_list_inner(terminal.size()) {
match list_mouse_action(inner, SITE_OPTION_COUNT, cursor, &mouse) {
Some(ListMouseAction::Hover(index)) => cursor = index,
Some(ListMouseAction::Click(index)) => {
cursor = index;
status = apply_site_option_row(&mut options, cursor);
}
Some(ListMouseAction::ScrollUp) => cursor = cursor.saturating_sub(1),
Some(ListMouseAction::ScrollDown) if cursor + 1 < SITE_OPTION_COUNT => {
cursor += 1;
}
Some(ListMouseAction::ScrollDown) | None => {}
}
}
continue;
}
let Event::Key(key) = input_event else {
continue;
};
if key.kind != KeyEventKind::Press {
continue;
}
match key.code {
KeyCode::Esc => return Ok(None),
KeyCode::Enter => return Ok(Some(options)),
KeyCode::Up => cursor = cursor.saturating_sub(1),
KeyCode::Down => {
if cursor + 1 < SITE_OPTION_COUNT {
cursor += 1;
}
}
KeyCode::Left | KeyCode::Right | KeyCode::Char(' ') => {
status = apply_site_option_row(&mut options, cursor);
}
KeyCode::Char('f') => {
options.full_site = !options.full_site;
status = site_crawl_status(options);
}
KeyCode::Char('d') => {
options.max_depth = next_site_depth(options.max_depth);
status = format!("max depth set to {}", options.max_depth);
}
KeyCode::Char('p') => {
options.max_pages = next_site_pages(options.max_pages);
status = format!("max pages set to {}", options.max_pages);
}
KeyCode::Char('j') => status = cycle_site_render_mode(&mut options),
KeyCode::Char('l') => {
options.archive_profile = next_site_archive_profile(options.archive_profile);
status = format!(
"archive profile set to {}",
site_archive_profile_label(options.archive_profile)
);
}
_ => {}
}
}
}
fn prompt_site_options(initial: SiteTuiOptions) -> Result<SiteTuiOptions> {
println!("Siteforge options:");
println!(
" crawl: {}",
if initial.full_site {
"same-domain full-site"
} else {
"single page"
}
);
println!(" max depth: {}", initial.max_depth);
println!(" max pages: {}", initial.max_pages);
println!(
" archive profile: {}",
site_archive_profile_label(initial.archive_profile)
);
println!(
" render JS: {}",
site_render_mode_label(initial.render_js, initial.render_js_auto)
);
let value = prompt_line("Use defaults? [Y/n]: ")?;
if matches!(value.as_str(), "n" | "N" | "no" | "No") {
println!("Using safe defaults for now; use command-form flags for detailed tuning.");
}
Ok(initial)
}
fn apply_site_option_row(options: &mut SiteTuiOptions, row: usize) -> String {
match row {
0 => {
options.full_site = !options.full_site;
site_crawl_status(*options)
}
1 => {
options.max_depth = next_site_depth(options.max_depth);
format!("max depth set to {}", options.max_depth)
}
2 => {
options.max_pages = next_site_pages(options.max_pages);
format!("max pages set to {}", options.max_pages)
}
3 => cycle_site_render_mode(options),
4 => {
options.archive_profile = next_site_archive_profile(options.archive_profile);
format!(
"archive profile set to {}",
site_archive_profile_label(options.archive_profile)
)
}
_ => "press Enter to run Siteforge".to_string(),
}
}
fn cycle_site_render_mode(options: &mut SiteTuiOptions) -> String {
match (options.render_js, options.render_js_auto) {
(false, false) => {
options.render_js = false;
options.render_js_auto = true;
}
(false, true) => {
options.render_js = true;
options.render_js_auto = false;
}
_ => {
options.render_js = false;
options.render_js_auto = false;
}
}
format!(
"JavaScript rendering set to {}",
site_render_mode_label(options.render_js, options.render_js_auto)
)
}
fn next_site_archive_profile(current: ArchiveOutputProfile) -> ArchiveOutputProfile {
match current {
ArchiveOutputProfile::Full => ArchiveOutputProfile::Agent,
ArchiveOutputProfile::Agent => ArchiveOutputProfile::Lean,
ArchiveOutputProfile::Lean => ArchiveOutputProfile::Full,
}
}
fn site_crawl_status(options: SiteTuiOptions) -> String {
if options.full_site {
"same-domain full-site crawl enabled".to_string()
} else {
"single-page archive selected".to_string()
}
}
fn next_site_depth(current: usize) -> usize {
match current {
1 => 2,
2 => 4,
4 => 8,
_ => 1,
}
}
fn next_site_pages(current: usize) -> usize {
match current {
25 => 100,
100 => 500,
500 => 1000,
_ => 25,
}
}
fn pick_models_tui_inner(
terminal: &mut Terminal,
models: &[ModelItem],
batch_size: &mut usize,
) -> Result<Vec<ModelItem>> {
if models.is_empty() {
return Ok(vec![]);
}
let mut query = String::new();
let mut selected = BTreeSet::new();
let mut cursor = 0usize;
let mut frame_index = 0usize;
let mut batch_overlay = false;
let mut status =
"Space checks models; Enter downloads checked models, or visible if none checked"
.to_string();
loop {
let filtered = filtered_model_indices(models, &query);
if cursor >= filtered.len() {
cursor = filtered.len().saturating_sub(1);
}
let page_step = picker_page_step(terminal.size().height);
draw_full_frame(terminal, |frame| {
draw_model_picker_tui(
frame,
ModelPickerView {
models,
filtered: &filtered,
selected: &selected,
cursor,
query: &query,
status: &status,
tick: frame_index,
batch_size: *batch_size,
batch_overlay,
},
);
})?;
frame_index = frame_index.wrapping_add(1);
if !event::poll(Duration::from_millis(50))? {
continue;
}
let input_event = event::read()?;
if let Event::Mouse(mouse) = input_event {
if !batch_overlay {
if let Some(inner) = picker_list_inner(terminal.size(), 104) {
match list_mouse_action(inner, filtered.len(), cursor, &mouse) {
Some(ListMouseAction::Hover(index)) => cursor = index,
Some(ListMouseAction::Click(index)) => {
cursor = index;
if let Some(model_index) = filtered.get(cursor).copied() {
if !selected.insert(model_index) {
selected.remove(&model_index);
}
let checked = selected.len();
status = if checked == 0 {
"nothing checked; Enter downloads visible models".to_string()
} else {
format!(
"{checked} checked; Enter downloads checked models only"
)
};
}
}
Some(ListMouseAction::ScrollUp) => cursor = cursor.saturating_sub(1),
Some(ListMouseAction::ScrollDown) if cursor + 1 < filtered.len() => {
cursor += 1;
}
Some(ListMouseAction::ScrollDown) | None => {}
}
}
}
continue;
}
let Event::Key(key) = input_event else {
continue;
};
if key.kind != KeyEventKind::Press {
continue;
}
if batch_overlay {
match key.code {
KeyCode::Esc | KeyCode::Char('b') | KeyCode::Enter => {
batch_overlay = false;
status = format!("batch size set to {}; Enter downloads models", *batch_size);
}
KeyCode::Char(ch) => {
if let Some(value) = batch_size_from_key(ch) {
*batch_size = value;
batch_overlay = false;
status = format!("batch size set to {value}; Enter downloads models");
}
}
KeyCode::Left | KeyCode::Up => {
*batch_size = previous_batch_size(*batch_size);
status = format!("batch size set to {}; Enter accepts", *batch_size);
}
KeyCode::Right | KeyCode::Down => {
*batch_size = next_batch_size(*batch_size);
status = format!("batch size set to {}; Enter accepts", *batch_size);
}
_ => {}
}
continue;
}
match key.code {
KeyCode::Esc => return Err(ThesaError::Message("aborted by user".to_string())),
KeyCode::Enter => {
let picked = if selected.is_empty() {
filtered
} else {
selected.iter().copied().collect::<Vec<_>>()
};
if picked.is_empty() {
status = "no models selected or visible".to_string();
continue;
}
return Ok(picked
.into_iter()
.map(|index| models[index].clone())
.collect());
}
KeyCode::Up => cursor = cursor.saturating_sub(1),
KeyCode::Down => {
if cursor + 1 < filtered.len() {
cursor += 1;
}
}
KeyCode::PageUp => {
let step = page_step.min(filtered.len().max(1));
cursor = cursor.saturating_sub(step);
status = if filtered.is_empty() {
"no models to move to".to_string()
} else {
format!("moved up by {step}")
};
}
KeyCode::PageDown => {
if filtered.is_empty() {
status = "no models to move to".to_string();
} else {
let step = page_step.min(filtered.len());
cursor = (cursor + step).min(filtered.len() - 1);
status = format!("moved down by {step}");
}
}
KeyCode::Home => {
if !filtered.is_empty() {
cursor = 0;
status = "jumped to first model".to_string();
} else {
status = "no models to move to".to_string();
}
}
KeyCode::End => {
if !filtered.is_empty() {
cursor = filtered.len() - 1;
status = "jumped to last model".to_string();
} else {
status = "no models to move to".to_string();
}
}
KeyCode::Char(' ') => {
if let Some(model_index) = filtered.get(cursor).copied() {
if !selected.insert(model_index) {
selected.remove(&model_index);
}
let checked = selected.len();
status = if checked == 0 {
"nothing checked; Enter downloads visible models".to_string()
} else {
format!("{checked} checked; Enter downloads checked models only")
};
}
}
KeyCode::Char('a') => {
for model_index in &filtered {
selected.insert(*model_index);
}
status = format!("checked {} visible models", filtered.len());
}
KeyCode::Char('c') => {
selected.clear();
query.clear();
cursor = 0;
status = "cleared filter and selection".to_string();
}
KeyCode::Char('b') => {
batch_overlay = true;
status = "choose batch size: 1 2 3 6 8".to_string();
}
KeyCode::Backspace => {
query.pop();
cursor = 0;
}
KeyCode::Char(ch) => {
query.push(ch);
cursor = 0;
}
_ => {}
}
}
}
fn filtered_model_indices(models: &[ModelItem], query: &str) -> Vec<usize> {
if query.trim().is_empty() {
return (0..models.len()).collect();
}
let query = query.to_lowercase();
models
.iter()
.enumerate()
.filter(|(_, m)| m.id.to_lowercase().contains(&query))
.map(|(index, _)| index)
.collect()
}
fn download_models_scrin(
models: &[ModelItem],
output: &Path,
args: &Cli,
ui_state: &mut UiState,
) -> Result<CloneSummary> {
let mut session = TerminalSession::new()?;
download_models_scrin_inner(&mut session.terminal, models, output, args, ui_state)
}
fn download_models_scrin_inner(
terminal: &mut Terminal,
models: &[ModelItem],
output: &Path,
args: &Cli,
ui_state: &mut UiState,
) -> Result<CloneSummary> {
let batch_size = args.concurrency.max(1);
let total = models.len();
let total_batches = total.div_ceil(batch_size);
let mut summary = CloneSummary::default();
let mut effect_kind = EffectKind::Waves;
let started_at = Instant::now();
let mut tick = 0usize;
let mut batch_index = 1;
while batch_index <= total {
let batch_end = (batch_index + batch_size - 1).min(total);
let batch = &models[batch_index - 1..batch_end];
let names: Vec<String> = batch.iter().map(|m| m.id.clone()).collect();
let mut state = vec![BatchState::Pending; batch.len()];
let (tx, rx) = mpsc::channel::<(usize, BatchEvent)>();
let mut handles = Vec::new();
for (offset, model) in batch.iter().enumerate() {
let model = model.clone();
let output = output.to_path_buf();
let args = args.clone();
let tx = tx.clone();
handles.push((
model.id.clone(),
thread::spawn(move || {
let started = tx.send((offset, BatchEvent::Started));
if started.is_err() {
return;
}
let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
download_model(&model, &output, &args)
}));
let event = match outcome {
Ok(Ok(CloneStatus::Cloned)) => BatchEvent::Result(BatchResult::Done),
Ok(Ok(CloneStatus::Skipped)) => BatchEvent::Result(BatchResult::Skipped),
Ok(Err(err)) => BatchEvent::Result(BatchResult::Error(err.to_string())),
Err(_) => BatchEvent::Result(BatchResult::Error(
"worker thread panicked while downloading".to_string(),
)),
};
let _ = tx.send((offset, event));
}),
));
}
drop(tx);
let mut completed = 0;
while completed < batch.len() {
while event::poll(Duration::from_millis(0))? {
if let Event::Key(key) = event::read()? {
match key.code {
KeyCode::Char('q') | KeyCode::Esc => ui_state.abort_requested = true,
KeyCode::Char('t') => ui_state.show_details = !ui_state.show_details,
KeyCode::Char('m') => effect_kind = EffectKind::Waves,
KeyCode::Char('b') => effect_kind = EffectKind::Burn,
KeyCode::Char('w') => effect_kind = EffectKind::Wipe,
_ => {}
}
}
}
while let Ok((offset, event)) = rx.try_recv() {
match event {
BatchEvent::Started => state[offset] = BatchState::Running,
BatchEvent::Result(BatchResult::Done) => {
state[offset] = BatchState::Done;
summary.cloned += 1;
completed += 1;
}
BatchEvent::Result(BatchResult::Skipped) => {
state[offset] = BatchState::Skipped;
summary.skipped += 1;
completed += 1;
}
BatchEvent::Result(BatchResult::Error(message)) => {
state[offset] = BatchState::Failed;
summary
.failed
.push(format!("{}: {}", names[offset], message));
completed += 1;
}
}
}
draw_full_frame(terminal, |frame| {
render_download_dashboard(
frame,
DownloadDashboard {
names: &names,
state: &state,
summary: &summary,
batch_index,
batch_size,
total_models: total,
total_batches,
tick,
elapsed: started_at.elapsed(),
effect_kind,
ui_state,
},
)
})?;
tick = tick.wrapping_add(1);
if completed < batch.len() {
thread::sleep(Duration::from_millis(80));
}
}
for (model_name, handle) in handles {
if handle.join().is_err() {
summary
.failed
.push(format!("{model_name}: worker thread panicked"));
}
}
if ui_state.abort_requested {
break;
}
batch_index = batch_end + 1;
}
Ok(summary)
}
fn download_models_serial(
models: &[ModelItem],
output: &Path,
args: &Cli,
ui_state: &mut UiState,
) -> Result<CloneSummary> {
let mut summary = CloneSummary::default();
let total = models.len();
for (index, model) in models.iter().enumerate() {
if ui_state.abort_requested {
break;
}
match download_model(model, output, args) {
Ok(CloneStatus::Cloned) => {
println!("[done] {} ({}/{})", model.id, index + 1, total);
summary.cloned += 1;
}
Ok(CloneStatus::Skipped) => {
println!("[skip] {} ({}/{})", model.id, index + 1, total);
summary.skipped += 1;
}
Err(err) => {
eprintln!("[fail] {} ({}/{}): {err}", model.id, index + 1, total);
summary.failed.push(format!("{}: {err}", model.id));
}
}
}
Ok(summary)
}
fn pick_repositories_interactively(repos: &[RepoItem]) -> Result<Vec<RepoItem>> {
if repos.is_empty() {
return Ok(vec![]);
}
let snapshot: Vec<RepoItem> = repos.to_vec();
loop {
println!("Found {} public repositories", snapshot.len());
let query = prompt_line("Search public repos (blank = all, q = quit): ")?;
if query.eq_ignore_ascii_case("q") {
return Err(ThesaError::Message("aborted by user".to_string()));
}
let filtered = if query.is_empty() {
snapshot.clone()
} else {
let normalized_query = query.to_lowercase();
snapshot
.iter()
.filter(|repo| repo.name.to_lowercase().contains(&normalized_query))
.cloned()
.collect::<Vec<_>>()
};
if filtered.is_empty() {
println!("No matches for '{query}'.");
continue;
}
for (index, repo) in filtered.iter().enumerate() {
println!("{:>3}. {}", index + 1, repo.full_name);
}
let selection = prompt_line("Clone [A]ll, specific indexes (1,3-5) or [q] to quit [a]: ")?;
if selection.is_empty() || selection.eq_ignore_ascii_case("a") {
return Ok(filtered);
}
if selection.eq_ignore_ascii_case("q") {
return Err(ThesaError::Message("aborted by user".to_string()));
}
match parse_indices(&selection, filtered.len()) {
Ok(indices) => {
let selected = indices
.into_iter()
.map(|idx| filtered[idx].clone())
.collect::<Vec<_>>();
return Ok(selected);
}
Err(message) => {
println!("{message}");
}
}
}
}
fn pick_models_interactively(models: &[ModelItem]) -> Result<Vec<ModelItem>> {
if models.is_empty() {
return Ok(vec![]);
}
let snapshot = models.to_vec();
loop {
println!("Found {} models", snapshot.len());
let query = prompt_line("Search models (blank = all, q = quit): ")?;
if query.eq_ignore_ascii_case("q") {
return Err(ThesaError::Message("aborted by user".to_string()));
}
let filtered = if query.is_empty() {
snapshot.clone()
} else {
let q = query.to_lowercase();
snapshot
.iter()
.filter(|m| m.id.to_lowercase().contains(&q))
.cloned()
.collect::<Vec<_>>()
};
if filtered.is_empty() {
println!("No matches for '{query}'.");
continue;
}
for (index, model) in filtered.iter().enumerate() {
println!("{:>3}. {} [dl: {}]", index + 1, model.id, model.downloads);
}
let selection =
prompt_line("Download [A]ll, specific indexes (1,3-5) or [q] to quit [a]: ")?;
if selection.is_empty() || selection.eq_ignore_ascii_case("a") {
return Ok(filtered);
}
if selection.eq_ignore_ascii_case("q") {
return Err(ThesaError::Message("aborted by user".to_string()));
}
match parse_indices(&selection, filtered.len()) {
Ok(indices) => {
return Ok(indices
.into_iter()
.map(|idx| filtered[idx].clone())
.collect());
}
Err(message) => {
println!("{message}");
}
}
}
}
fn draw_hf_token_tui(
frame: &mut Frame<'_>,
input: &str,
status: &str,
tick: usize,
show_plain: bool,
) {
let area = frame.area();
paint_surface(frame, area);
if area.width < 24 || area.height < 8 {
let line = if show_plain {
format!("HF token> {input}\n{status}")
} else {
format!("HF token> {}\n{status}", "*".repeat(input.len()))
};
frame.render_widget(
Paragraph::from_text(Text::new(vec![Line::styled(
&line,
Style::default().fg(theme_white()),
)])),
area,
);
return;
}
let layout = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(header_height(area)),
Constraint::Min(5),
Constraint::Length(2),
])
.split(area);
draw_thesa_header(frame, layout[0], tick, status, "hf auth");
let display_input = if show_plain {
input.to_string()
} else {
"*".repeat(input.len())
};
let body = Paragraph::from_text(Text::new(vec![
Line::new(vec![
Span::styled("HF token> ", prompt_style()),
Span::styled(&display_input, Style::default().fg(theme_white())),
]),
Line::raw(""),
Line::new(vec![
Span::styled("hint ", Style::default().fg(theme_dim())),
Span::styled(
"https://huggingface.co/settings/tokens",
Style::default().fg(theme_blue()),
),
]),
Line::raw(""),
Line::styled(
"Press Enter with blank to skip (will rate-limit).",
Style::default().fg(theme_dim()),
),
]))
.wrap(WrapMode::Character { trim: false });
frame.render_block_widget(
layout[1],
body,
knott_block(" Hugging Face Token ", theme_blue()),
);
draw_thesa_bottom_bar(frame, layout[2], status, &["Enter accept", "Esc back"]);
let input_area = panel_inner(layout[1]);
let cursor_x = input_area
.x
.saturating_add(10)
.saturating_add(input.chars().count() as u16)
.min(input_area.x + input_area.width.saturating_sub(1));
frame.set_cursor_position(cursor_x, input_area.y);
}
fn draw_mode_selector_tui(frame: &mut Frame<'_>, cursor: usize, tick: usize) {
let area = frame.area();
paint_surface(frame, area);
if area.width < 28 || area.height < 10 {
let line = format!(
"select mode: repos={} models={} sites={}",
cursor == 0,
cursor == 1,
cursor == 2,
);
frame.render_widget(
Paragraph::from_text(Text::new(vec![Line::styled(
&line,
Style::default().fg(theme_white()),
)])),
area,
);
return;
}
let layout = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(header_height(area)),
Constraint::Min(8),
Constraint::Length(2),
])
.split(area);
draw_thesa_header(
frame,
layout[0],
tick,
"choose an archive source",
"landing",
);
let body = Layout::default()
.direction(Direction::Horizontal)
.constraints(if layout[1].width >= 94 {
[Constraint::Percentage(62), Constraint::Percentage(38)]
} else {
[Constraint::Percentage(100), Constraint::Percentage(0)]
})
.split(layout[1]);
let modes = [
(
"repos",
"GitHub repositories (owner/org/repo)",
"r",
theme_blue(),
"clone + manifest",
),
(
"models",
"Hugging Face, Ollama, CivitAI models",
"m",
theme_green(),
"model artifacts",
),
(
"baseforge",
"Hugging Face and Zenodo datasets",
"d",
TuiColor::rgb(247, 183, 89),
"dataset files",
),
(
"siteforge",
"Website archive from a URL",
"s",
TuiColor::rgb(163, 113, 247),
"site crawl",
),
];
let items = modes
.iter()
.enumerate()
.map(|(i, (name, desc, key, color, tag))| {
let selected = cursor == i;
let marker = if selected { ">>" } else { " " };
let style = if selected {
selected_style(tick)
} else {
Style::default().fg(theme_white())
};
ListItem::with_line(Line::new(vec![
Span::styled(
&format!(" {marker} "),
style.add_modifier(if selected {
Modifier::BOLD
} else {
Modifier::empty()
}),
),
Span::styled(
&format!("[{key}]"),
Style::default().fg(*color).add_modifier(Modifier::BOLD),
),
Span::raw(" "),
Span::styled(&name.to_uppercase(), style.add_modifier(Modifier::BOLD)),
Span::raw(" "),
Span::styled(desc, Style::default().fg(theme_dim())),
Span::raw(" "),
Span::styled(
&format!("{tag:<16}"),
Style::default().fg(*color).add_modifier(Modifier::BOLD),
),
]))
})
.collect::<Vec<_>>();
let list = List::new(&items)
.with_highlight_style(selected_style(tick))
.with_highlight_symbol(">>");
let mut state = ListState::default();
state.select(Some(cursor));
frame.render_block_stateful_list(
body[0],
&list,
&mut state,
knott_block(" Archive Source ", theme_blue()),
);
if body[1].width > 0 {
render_aisling_text_panel(
frame,
body[1],
AislingTextPanel {
title: " thesa command signal ",
kind: EffectKind::Waves,
tick,
effect_text: "thesa\narchive\nverify\nrestore",
lines: vec![
section_line("Preservation Routes"),
help_pair_line("repos", "GitHub clone batches"),
help_pair_line("models", "HF/Ollama/CivitAI"),
help_pair_line("baseforge", "HF/Zenodo datasets"),
help_pair_line("siteforge", "same-domain web crawl"),
help_pair_line("metadata", ".thesa manifest + BLAKE3"),
Line::raw(""),
section_line("Controls"),
help_pair_line("r/m/d/s", "jump source"),
help_pair_line("Enter", "open selected route"),
help_pair_line("Mouse", "click row"),
],
color: theme_blue(),
},
);
}
draw_thesa_bottom_bar(
frame,
layout[2],
"Up/Down or r/m/d/s to select, Enter to confirm, Esc back/quit",
&[
"Up/Down move",
"r repos",
"m models",
"d baseforge",
"s siteforge",
"Mouse click",
"Enter select",
"Esc back",
],
);
}
fn draw_model_provider_selector_tui(frame: &mut Frame<'_>, cursor: usize, tick: usize) {
let area = frame.area();
paint_surface(frame, area);
if area.width < 24 || area.height < 8 {
frame.render_widget(
Paragraph::from_text(Text::new(vec![Line::styled(
"Model provider\nhf\nollama\ncivitai",
Style::default().fg(theme_white()),
)])),
area,
);
return;
}
let layout = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(header_height(area)),
Constraint::Min(8),
Constraint::Length(3),
])
.split(area);
draw_thesa_header(
frame,
layout[0],
tick,
"select model provider before choosing target",
"model setup",
);
let providers = [
(
"hugging face",
"remote model hub: orgs, org/model, top, latest",
"h/1",
ModelProvider::Hf,
),
(
"ollama",
"local Ollama library: installed names, top, latest",
"o/2",
ModelProvider::Ollama,
),
(
"civitai",
"CivitAI models: numeric IDs, keyword search, top, latest",
"c/3",
ModelProvider::Civitai,
),
];
let items = providers
.iter()
.enumerate()
.map(|(i, (name, desc, key, provider))| {
let selected = cursor == i;
let marker = if selected { ">>" } else { " " };
let style = if selected {
selected_style(tick)
} else {
Style::default().fg(theme_white())
};
ListItem::with_line(Line::new(vec![
Span::styled(&format!(" {marker} "), style),
Span::styled(
&format!("[{key}]"),
Style::default()
.fg(model_provider_color(*provider))
.add_modifier(Modifier::BOLD),
),
Span::raw(" "),
Span::styled(&name.to_uppercase(), style.add_modifier(Modifier::BOLD)),
Span::raw(" "),
Span::styled(desc, Style::default().fg(theme_dim())),
]))
})
.collect::<Vec<_>>();
let list = List::new(&items)
.with_highlight_style(selected_style(tick))
.with_highlight_symbol(">>");
let mut state = ListState::default();
state.select(Some(cursor));
frame.render_block_stateful_list(
layout[1],
&list,
&mut state,
knott_block(" Provider ", theme_green()),
);
draw_thesa_bottom_bar(
frame,
layout[2],
"Choose where model archives come from",
&[
"Up/Down move",
"h/o/c choose",
"1/2/3 choose",
"Mouse click",
"Enter continue",
"Esc back",
],
);
}
fn draw_target_tui(
frame: &mut Frame<'_>,
input: &str,
status: &str,
tick: usize,
mode: Mode,
provider: Option<ModelProvider>,
) {
let area = frame.area();
paint_surface(frame, area);
let prompt_label = match mode {
Mode::Repos => "repo/owner> ",
Mode::Models => "model target> ",
Mode::Datasets => "dataset target> ",
Mode::Sites => "site url> ",
};
if area.width < 24 || area.height < 8 {
let line = format!("{prompt_label}{input}\n{status}");
frame.render_widget(
Paragraph::from_text(Text::new(vec![Line::styled(
&line,
Style::default().fg(theme_white()),
)])),
area,
);
return;
}
let layout = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(header_height(area)),
Constraint::Min(5),
Constraint::Length(2),
])
.split(area);
let model_provider = provider.unwrap_or(ModelProvider::Hf);
let mode_label = match mode {
Mode::Repos => "github repos",
Mode::Models => match model_provider {
ModelProvider::Hf => "huggingface models",
ModelProvider::Ollama => "ollama models",
ModelProvider::Civitai => "civitai models",
},
Mode::Datasets => "baseforge datasets",
Mode::Sites => "siteforge websites",
};
draw_thesa_header(frame, layout[0], tick, status, mode_label);
let examples = match mode {
Mode::Repos => vec![
Span::styled("examples ", Style::default().fg(theme_dim())),
Span::styled("octocat", Style::default().fg(theme_blue())),
Span::raw(" "),
Span::styled("octocat/Spoon-Knife", Style::default().fg(theme_green())),
Span::raw(" "),
Span::styled(
"https://github.com/owner/repo",
Style::default().fg(theme_red()),
),
],
Mode::Models => match model_provider {
ModelProvider::Hf => vec![
Span::styled("examples ", Style::default().fg(theme_dim())),
Span::styled("meta-llama", Style::default().fg(theme_blue())),
Span::raw(" "),
Span::styled("meta-llama/Llama-2-7b", Style::default().fg(theme_green())),
Span::raw(" "),
Span::styled("top", Style::default().fg(theme_red())),
Span::raw(" "),
Span::styled(
"https://huggingface.co/openai-community/gpt2",
Style::default().fg(theme_red()),
),
],
ModelProvider::Ollama => vec![
Span::styled("examples ", Style::default().fg(theme_dim())),
Span::styled("llama3:8b", Style::default().fg(theme_blue())),
Span::raw(" "),
Span::styled("qwen2.5:7b", Style::default().fg(theme_green())),
Span::raw(" "),
Span::styled("latest", Style::default().fg(theme_red())),
Span::raw(" "),
Span::styled("phi3:3.8b", Style::default().fg(theme_red())),
],
ModelProvider::Civitai => vec![
Span::styled("examples ", Style::default().fg(theme_dim())),
Span::styled("827184", Style::default().fg(theme_blue())),
Span::raw(" "),
Span::styled("realistic-vision-v6", Style::default().fg(theme_green())),
Span::raw(" "),
Span::styled("top", Style::default().fg(theme_red())),
Span::raw(" "),
Span::styled(
"https://civitai.com/models/827184",
Style::default().fg(theme_red()),
),
],
},
Mode::Datasets => vec![
Span::styled("examples ", Style::default().fg(theme_dim())),
Span::styled("openai", Style::default().fg(theme_blue())),
Span::raw(" "),
Span::styled("dataset:squad", Style::default().fg(theme_green())),
Span::raw(" "),
Span::styled("search:code data", Style::default().fg(theme_red())),
Span::raw(" "),
Span::styled("top", Style::default().fg(theme_red())),
],
Mode::Sites => vec![
Span::styled("examples ", Style::default().fg(theme_dim())),
Span::styled("https://example.com", Style::default().fg(theme_blue())),
Span::raw(" "),
Span::styled("https://docs.rs", Style::default().fg(theme_green())),
Span::raw(" "),
Span::styled("http://127.0.0.1:8765/", Style::default().fg(theme_red())),
],
};
let body = Paragraph::from_text(Text::new(vec![
Line::new(vec![
Span::styled(prompt_label, prompt_style()),
Span::styled(input, Style::default().fg(theme_white())),
]),
Line::raw(""),
Line::new(examples),
]))
.wrap(WrapMode::Character { trim: false });
let title = match mode {
Mode::Sites => " Siteforge URL ",
Mode::Datasets => " BaseForge Target ",
_ => " Target ",
};
let color = match mode {
Mode::Sites => theme_green(),
Mode::Datasets => TuiColor::rgb(247, 183, 89),
_ => theme_blue(),
};
frame.render_block_widget(layout[1], body, knott_block(title, color));
draw_thesa_bottom_bar(
frame,
layout[2],
status,
match mode {
Mode::Sites => &["Enter crawl", "Esc back"],
Mode::Datasets => &["Enter scrape", "Esc back"],
_ => &["Enter accept", "Esc back"],
},
);
let input_area = panel_inner(layout[1]);
let cursor_x = input_area
.x
.saturating_add(prompt_label.len() as u16)
.saturating_add(input.chars().count() as u16)
.min(input_area.x + input_area.width.saturating_sub(1));
frame.set_cursor_position(cursor_x, input_area.y);
}
fn draw_site_options_tui(
frame: &mut Frame<'_>,
url: &str,
options: SiteTuiOptions,
cursor: usize,
status: &str,
tick: usize,
) {
let area = frame.area();
paint_surface(frame, area);
if area.width < 32 || area.height < 10 {
let line = format!(
"siteforge\n{}\ndepth {} pages {} profile {} js {}\n{}",
if options.full_site {
"full-site"
} else {
"single-page"
},
options.max_depth,
options.max_pages,
site_archive_profile_label(options.archive_profile),
site_render_mode_label(options.render_js, options.render_js_auto),
status,
);
frame.render_widget(
Paragraph::from_text(Text::new(vec![Line::styled(
&line,
Style::default().fg(theme_white()),
)])),
area,
);
return;
}
let layout = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(header_height(area)),
Constraint::Min(8),
Constraint::Length(2),
])
.split(area);
draw_thesa_header(frame, layout[0], tick, status, "siteforge options");
let body = Layout::default()
.direction(Direction::Horizontal)
.constraints(if layout[1].width >= 94 {
[Constraint::Percentage(64), Constraint::Percentage(36)]
} else {
[Constraint::Percentage(100), Constraint::Percentage(0)]
})
.split(layout[1]);
let rows: Vec<(&str, &str, String, &str)> = vec![
(
"f",
"crawl",
if options.full_site {
"same-domain full-site".to_string()
} else {
"single page only".to_string()
},
"toggle scope",
),
(
"d",
"depth",
format_site_option_value(options.max_depth),
"cycle 1/2/4/8",
),
(
"p",
"pages",
format_site_option_value(options.max_pages),
"cycle 25/100/500/1000",
),
(
"j",
"js mode",
site_render_mode_label(options.render_js, options.render_js_auto).to_string(),
"off/auto/all",
),
(
"l",
"profile",
site_archive_profile_label(options.archive_profile).to_string(),
"full/agent/lean",
),
];
let items = rows
.iter()
.enumerate()
.map(|(index, (key, label, value, detail))| {
let selected = cursor == index;
let style = if selected {
selected_style(tick)
} else {
Style::default().fg(theme_white())
};
ListItem::with_line(Line::new(vec![
Span::styled(
&format!("[{key}] "),
Style::default()
.fg(theme_green())
.add_modifier(Modifier::BOLD),
),
Span::styled(&format!("{label:<10}"), style.add_modifier(Modifier::BOLD)),
Span::styled(&format!(" {value:<24}"), Style::default().fg(theme_white())),
Span::styled(detail, Style::default().fg(theme_dim())),
]))
})
.collect::<Vec<_>>();
let mut state = ListState::default();
state.select(Some(cursor));
let list = List::new(&items)
.with_highlight_style(selected_style(tick))
.with_highlight_symbol("> ");
frame.render_block_stateful_list(
body[0],
&list,
&mut state,
knott_block(" Siteforge Run Plan ", theme_green()),
);
if body[1].width > 0 {
render_aisling_text_panel(
frame,
body[1],
AislingTextPanel {
title: " siteforge crawl signal ",
kind: EffectKind::Wipe,
tick,
effect_text: "siteforge\ncrawl\nreadable\nindex",
lines: vec![
section_line("URL"),
Line::styled(
&truncate_to_width(url, body[1].width.saturating_sub(4) as usize),
Style::default().fg(theme_white()),
),
Line::raw(""),
help_pair_line("output", "./archives/sites unless overridden"),
help_pair_line("domain", "same-domain only"),
help_pair_line("readable", "AGENTS.md + markdown/text"),
help_pair_line("verify", ".thesa manifest after crawl"),
],
color: theme_green(),
},
);
}
draw_thesa_bottom_bar(
frame,
layout[2],
status,
&[
"Enter run",
"Esc back",
"Space adjust",
"f scope",
"d depth",
"p pages",
"j js",
"l profile",
],
);
}
fn format_site_option_value(value: usize) -> String {
value.to_string()
}
struct ModelPickerView<'a> {
models: &'a [ModelItem],
filtered: &'a [usize],
selected: &'a BTreeSet<usize>,
cursor: usize,
query: &'a str,
status: &'a str,
tick: usize,
batch_size: usize,
batch_overlay: bool,
}
fn draw_model_picker_tui(frame: &mut Frame<'_>, view: ModelPickerView<'_>) {
let ModelPickerView {
models,
filtered,
selected,
cursor,
query,
status,
tick,
batch_size,
batch_overlay,
} = view;
let area = frame.area();
paint_surface(frame, area);
if area.width < 28 || area.height < 8 {
let line = format!(
"models {} visible / {} total\nfilter> {}\n{}",
filtered.len(),
models.len(),
query,
status,
);
frame.render_widget(
Paragraph::from_text(Text::new(vec![Line::styled(
&line,
Style::default().fg(theme_white()),
)])),
area,
);
return;
}
let layout = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(header_height(area)),
Constraint::Min(8),
Constraint::Length(2),
])
.split(area);
draw_thesa_header(frame, layout[0], tick, status, "model picker");
let body = Layout::default()
.direction(Direction::Horizontal)
.constraints(if layout[1].width >= 104 {
[Constraint::Percentage(72), Constraint::Percentage(28)]
} else {
[Constraint::Percentage(100), Constraint::Percentage(0)]
})
.split(layout[1]);
let clone_scope = if selected.is_empty() {
format!("visible: {}", filtered.len())
} else {
format!("checked only: {}", selected.len())
};
let max_name_width = if body[0].width > 56 {
(body[0].width as usize - 52).max(20)
} else {
(body[0].width as usize - 30).max(10)
};
let provider_counts = format!(
"hf:{} ollama:{} civitai:{}",
models
.iter()
.filter(|m| m.provider == ModelProvider::Hf)
.count(),
models
.iter()
.filter(|m| m.provider == ModelProvider::Ollama)
.count(),
models
.iter()
.filter(|m| m.provider == ModelProvider::Civitai)
.count(),
);
let items = filtered
.iter()
.map(|model_index| {
let model = &models[*model_index];
let marker = if selected.contains(model_index) {
"[x]"
} else {
"[ ]"
};
let tag = model.pipeline_tag.as_deref().unwrap_or("-");
let dl = format_downloads(model.downloads);
let size = format_model_size(model.size);
let name = truncate_to_width(&model.id, max_name_width);
let tag = truncate_to_width(tag, 10);
ListItem::with_line(Line::new(vec![
Span::styled(
marker,
Style::default()
.fg(theme_green())
.add_modifier(Modifier::BOLD),
),
Span::raw(" "),
Span::styled(
&format!("[{}]", model.provider.short_label()),
Style::default()
.fg(model_provider_color(model.provider))
.add_modifier(Modifier::BOLD),
),
Span::raw(" "),
Span::styled(
&format!("{name:<max_name_width$}"),
Style::default().fg(theme_white()),
),
Span::styled(&format!(" {dl:>7} "), Style::default().fg(theme_blue())),
Span::styled(&format!(" {size:>9} "), Style::default().fg(theme_dim())),
Span::styled(&format!(" [{tag}]"), Style::default().fg(theme_dim())),
]))
})
.collect::<Vec<_>>();
let mut state = ListState::default();
if !filtered.is_empty() {
state.select(Some(cursor));
}
let list_title = format!(
" Models | filter: {} | {} | batch: {} | {} ",
if query.is_empty() { "all" } else { query },
clone_scope,
batch_size,
provider_counts,
);
let list = List::new(&items)
.with_highlight_style(selected_style(tick))
.with_highlight_symbol("> ");
frame.render_block_stateful_list(
body[0],
&list,
&mut state,
knott_block(&list_title, theme_blue()),
);
if body[1].width > 0 {
let side = Layout::default()
.direction(Direction::Vertical)
.constraints([Constraint::Percentage(65), Constraint::Percentage(35)])
.split(body[1]);
let mut sidebar_lines = vec![
section_line("Controls"),
help_pair_line("type", "filter models"),
help_pair_line("mouse", "hover focus, click check"),
help_pair_line("Space", "check/uncheck model"),
help_pair_line("a", "check visible"),
help_pair_line("b", "batch size overlay"),
help_pair_line("c", "clear"),
help_pair_line("PgUp/PgDown", "jump pages"),
help_pair_line("Home/End", "first/last"),
help_pair_line("Enter", "checked only, or visible"),
Line::raw(""),
section_line("Selection"),
help_pair_line("checked", &selected.len().to_string()),
help_pair_line("filtered", &filtered.len().to_string()),
];
if let Some(model) = filtered.get(cursor).and_then(|i| models.get(*i)) {
sidebar_lines.push(Line::raw(""));
sidebar_lines.push(section_line("Active"));
sidebar_lines.push(help_pair_line("provider", model.provider.label()));
sidebar_lines.push(Line::new(vec![
Span::styled(
"name ",
Style::default()
.fg(theme_blue())
.add_modifier(Modifier::BOLD),
),
Span::styled(
&truncate_to_width(&model.id, (body[1].width as usize).saturating_sub(16)),
Style::default().fg(theme_white()),
),
]));
sidebar_lines.push(help_pair_line(
"tag",
model.pipeline_tag.as_deref().unwrap_or("-"),
));
sidebar_lines.push(help_pair_line(
"downloads",
&format_downloads(model.downloads),
));
sidebar_lines.push(help_pair_line("size", &format_model_size(model.size)));
sidebar_lines.push(help_pair_line(
"updated",
model.last_modified.as_deref().unwrap_or("n/a"),
));
} else {
sidebar_lines.push(Line::raw(""));
sidebar_lines.push(section_line("Active"));
sidebar_lines.push(Line::raw("no model selected"));
}
let sidebar = Paragraph::from_text(Text::new(sidebar_lines))
.wrap(WrapMode::Character { trim: false });
frame.render_block_widget(
side[0],
sidebar,
knott_block(" controls + selection ", theme_dim()),
);
render_aisling_panel(
frame,
side[1],
" scrin aisling burn ",
EffectKind::Burn,
tick,
"models",
);
}
draw_thesa_bottom_bar(
frame,
layout[2],
status,
&[
"Up/Down move",
"PgUp/PgDown jump",
"Home/End jump",
"Mouse hover/click",
"Space check",
"b batch",
"Enter download",
"Esc quit",
],
);
if batch_overlay {
draw_batch_overlay(frame, area, batch_size);
}
}
fn picker_page_step(terminal_height: u16) -> usize {
let header_rows = if terminal_height >= 30 { 5 } else { 3 };
let list_rows = terminal_height.saturating_sub(header_rows + 2);
(list_rows.max(1) as usize).max(5) / 2
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ListMouseAction {
Hover(usize),
Click(usize),
ScrollUp,
ScrollDown,
}
fn mode_selector_list_inner(area: Rect) -> Option<Rect> {
if area.width < 28 || area.height < 10 {
return None;
}
let layout = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(header_height(area)),
Constraint::Min(8),
Constraint::Length(2),
])
.split(area);
let body = Layout::default()
.direction(Direction::Horizontal)
.constraints(if layout[1].width >= 94 {
[Constraint::Percentage(62), Constraint::Percentage(38)]
} else {
[Constraint::Percentage(100), Constraint::Percentage(0)]
})
.split(layout[1]);
Some(panel_inner(body[0]))
}
fn provider_selector_list_inner(area: Rect) -> Option<Rect> {
selector_list_inner(area, 24, 8, 3)
}
fn site_options_list_inner(area: Rect) -> Option<Rect> {
if area.width < 32 || area.height < 10 {
return None;
}
let layout = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(header_height(area)),
Constraint::Min(8),
Constraint::Length(2),
])
.split(area);
let body = Layout::default()
.direction(Direction::Horizontal)
.constraints(if layout[1].width >= 94 {
[Constraint::Percentage(64), Constraint::Percentage(36)]
} else {
[Constraint::Percentage(100), Constraint::Percentage(0)]
})
.split(layout[1]);
Some(panel_inner(body[0]))
}
fn selector_list_inner(
area: Rect,
min_width: u16,
min_height: u16,
footer_height: u16,
) -> Option<Rect> {
if area.width < min_width || area.height < min_height {
return None;
}
let layout = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(header_height(area)),
Constraint::Min(8),
Constraint::Length(footer_height),
])
.split(area);
Some(panel_inner(layout[1]))
}
fn picker_list_inner(area: Rect, sidebar_threshold: u16) -> Option<Rect> {
if area.width < 28 || area.height < 8 {
return None;
}
let layout = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(header_height(area)),
Constraint::Min(8),
Constraint::Length(2),
])
.split(area);
let body = Layout::default()
.direction(Direction::Horizontal)
.constraints(if layout[1].width >= sidebar_threshold {
[Constraint::Percentage(72), Constraint::Percentage(28)]
} else {
[Constraint::Percentage(100), Constraint::Percentage(0)]
})
.split(layout[1]);
Some(panel_inner(body[0]))
}
fn list_mouse_action(
inner: Rect,
item_count: usize,
cursor: usize,
mouse: &MouseEvent,
) -> Option<ListMouseAction> {
match mouse.kind {
MouseEventKind::Moved | MouseEventKind::Drag(MouseButton::Left) => {
visible_list_item_at(inner, mouse.column, mouse.row, item_count, cursor)
.map(ListMouseAction::Hover)
}
MouseEventKind::Down(MouseButton::Left) => {
visible_list_item_at(inner, mouse.column, mouse.row, item_count, cursor)
.map(ListMouseAction::Click)
}
MouseEventKind::ScrollUp if rect_contains_point(inner, mouse.column, mouse.row) => {
Some(ListMouseAction::ScrollUp)
}
MouseEventKind::ScrollDown if rect_contains_point(inner, mouse.column, mouse.row) => {
Some(ListMouseAction::ScrollDown)
}
_ => None,
}
}
fn visible_list_item_at(
inner: Rect,
x: u16,
y: u16,
item_count: usize,
cursor: usize,
) -> Option<usize> {
if item_count == 0 || inner.height == 0 || !rect_contains_point(inner, x, y) {
return None;
}
let visible_rows = inner.height as usize;
let offset = list_render_offset(item_count, cursor, visible_rows);
let row = y.saturating_sub(inner.y) as usize;
let index = offset + row;
(index < item_count).then_some(index)
}
fn list_render_offset(item_count: usize, cursor: usize, visible_rows: usize) -> usize {
if item_count == 0 || visible_rows == 0 {
return 0;
}
let cursor = cursor.min(item_count - 1);
if cursor >= visible_rows {
cursor + 1 - visible_rows
} else {
0
}
}
fn rect_contains_point(area: Rect, x: u16, y: u16) -> bool {
x >= area.x
&& y >= area.y
&& x < area.x.saturating_add(area.width)
&& y < area.y.saturating_add(area.height)
}
struct DownloadDashboard<'a> {
names: &'a [String],
state: &'a [BatchState],
summary: &'a CloneSummary,
batch_index: usize,
batch_size: usize,
total_models: usize,
total_batches: usize,
tick: usize,
elapsed: Duration,
effect_kind: EffectKind,
ui_state: &'a UiState,
}
fn render_download_dashboard(frame: &mut Frame, dashboard: DownloadDashboard<'_>) {
let DownloadDashboard {
names,
state,
summary,
batch_index,
batch_size,
total_models,
total_batches,
tick,
elapsed,
effect_kind,
ui_state,
} = dashboard;
let area = frame.area();
paint_surface(frame, area);
let overall_done = summary.cloned + summary.skipped + summary.failed.len();
let status = if ui_state.abort_requested {
"stopping after current batch"
} else {
"downloading models"
};
if area.width < 36 || area.height < 10 {
let loader = Loader::with_config(
LoaderKind::Tqdm,
LoaderConfig::default()
.with_size(area.width.saturating_sub(4) as usize, 1)
.with_label("thesa")
.with_unit("model")
.with_fraction(true),
);
let line = loader.line(
tick,
LoaderProgress::from_counts(overall_done as u64, total_models as u64),
);
let panel = Paragraph::from_text(Text::new(vec![
Line::styled(
"thesa",
Style::default()
.fg(theme_white())
.add_modifier(Modifier::BOLD),
),
Line::styled(&line, Style::default().fg(theme_white())),
Line::styled(status, Style::default().fg(theme_dim())),
]));
frame.render_block_widget(area, panel, knott_block(" thesa models ", theme_blue()));
return;
}
let root = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(header_height(area)),
Constraint::Min(8),
Constraint::Length(2),
])
.split(area);
draw_thesa_header(frame, root[0], tick, status, "model download");
let body = Layout::default()
.direction(Direction::Horizontal)
.constraints(if root[1].width >= 104 {
[Constraint::Percentage(68), Constraint::Percentage(32)]
} else {
[Constraint::Percentage(100), Constraint::Percentage(0)]
})
.split(root[1]);
let left = Layout::default()
.direction(Direction::Vertical)
.constraints([Constraint::Length(5), Constraint::Min(5)])
.split(body[0]);
let current_batch = ((batch_index - 1) / batch_size) + 1;
render_command_progress_panel(
frame,
left[0],
" command progress ",
CommandProgress {
verb: "download models",
unit: "model",
done_label: "downloaded",
done: summary.cloned,
skipped: summary.skipped,
failed: summary.failed.len(),
total: total_models,
batch_index: current_batch,
batch_size,
total_batches,
elapsed,
},
tick,
);
let model_loader = Loader::with_config(
LoaderKind::Tqdm,
LoaderConfig::default()
.with_size(left[1].width.saturating_sub(34).max(10) as usize, 1)
.with_label("model")
.with_unit("step")
.with_fraction(true),
);
let mut rows = Vec::new();
if ui_state.show_details {
for (index, (name, model_state)) in names.iter().zip(state.iter()).enumerate() {
rows.push(repo_loader_row(
&model_loader,
tick + index,
name,
*model_state,
));
}
} else {
rows.push(Line::styled(
"details hidden; press t to show concurrent model loaders",
Style::default().fg(theme_dim()),
));
}
let panel = Paragraph::from_text(Text::new(rows)).wrap(WrapMode::Character { trim: false });
frame.render_block_widget(
left[1],
panel,
knott_block(" Concurrent Downloads | Aisling Tqdm ", theme_blue()),
);
if body[1].width > 0 {
let (pending, running, done, skipped, failed) = batch_state_counts(state);
let side_layout = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(8),
Constraint::Length(8),
Constraint::Min(4),
])
.split(body[1]);
render_command_info_panel(
frame,
side_layout[0],
" telemetry ",
vec![
section_line("Batch State"),
help_pair_line("running", &running.to_string()),
help_pair_line("pending", &pending.to_string()),
help_pair_line("done", &done.to_string()),
help_pair_line("skipped", &skipped.to_string()),
help_pair_line("failed", &failed.to_string()),
],
theme_blue(),
);
let side_lines = vec![
section_line("Controls"),
help_pair_line("q / Esc", "stop after current batch"),
help_pair_line("t", "toggle model rows"),
help_pair_line("m", "motion effect"),
help_pair_line("b", "burn effect"),
help_pair_line("w", "wipe effect"),
];
render_command_info_panel(
frame,
side_layout[1],
" command keys ",
side_lines,
theme_dim(),
);
render_aisling_panel(
frame,
side_layout[2],
&format!(" scrin aisling {} ", effect_kind.name()),
effect_kind,
tick,
"models\ndownload",
);
}
draw_thesa_bottom_bar(
frame,
root[2],
status,
&["q stop", "t details", "m motion", "b burn", "w wipe"],
);
}
fn format_downloads(n: u64) -> String {
if n >= 1_000_000 {
format!("{:.1}M", n as f64 / 1_000_000.0)
} else if n >= 1_000 {
format!("{:.1}K", n as f64 / 1_000.0)
} else {
format!("{n}")
}
}
fn format_model_size(size: Option<u64>) -> String {
match size {
None => "n/a".to_string(),
Some(bytes) => {
let gb = 1024f64 * 1024f64 * 1024f64;
let mb = 1024f64 * 1024f64;
let kb = 1024f64;
if bytes as f64 >= gb {
format!("{:.1}GB", bytes as f64 / gb)
} else if bytes as f64 >= mb {
format!("{:.1}MB", bytes as f64 / mb)
} else if bytes as f64 >= kb {
format!("{:.1}KB", bytes as f64 / kb)
} else {
format!("{bytes} B")
}
}
}
}
fn truncate_to_width(value: &str, width: usize) -> String {
if width == 0 {
return String::new();
}
let chars: Vec<char> = value.chars().collect();
if chars.len() <= width {
value.to_string()
} else if width <= 3 {
chars.iter().take(width).copied().collect()
} else {
let mut out: String = chars.iter().take(width - 3).copied().collect();
out.push_str("...");
out
}
}
fn model_provider_color(provider: ModelProvider) -> TuiColor {
match provider {
ModelProvider::Hf => TuiColor::rgb(130, 190, 255),
ModelProvider::Ollama => TuiColor::rgb(130, 255, 130),
ModelProvider::Civitai => TuiColor::rgb(255, 200, 130),
}
}
fn fetch_repositories_for_target(
client: &Client,
target: &Target,
token: Option<&str>,
) -> Result<Vec<RepoItem>> {
match target {
Target::Owner(owner) => match list_repos_for_owner(client, owner, token) {
Ok(items) => Ok(items),
Err(ThesaError::NotFound(_)) => list_repos_for_org(client, owner, token),
Err(other) => Err(other),
},
Target::Repo { owner, repo } => Ok(vec![fetch_single_repo(client, owner, repo, token)?]),
}
}
fn fetch_repositories_tui(
terminal: &mut Terminal,
client: &Client,
target: &Target,
token: Option<&str>,
) -> Result<Vec<RepoItem>> {
let client = client.clone();
let target = target.clone();
let token = token.map(str::to_string);
let loading_label = match &target {
Target::Owner(owner) => format!("loading public repos for {owner}"),
Target::Repo { owner, repo } => format!("loading repo {owner}/{repo}"),
};
let (tx, rx) = mpsc::channel();
thread::spawn(move || {
let result = fetch_repositories_for_target(&client, &target, token.as_deref());
let _ = tx.send(result);
});
let mut tick = 0usize;
loop {
if let Ok(result) = rx.try_recv() {
return result;
}
draw_full_frame(terminal, |frame| {
draw_loading_tui(frame, &loading_label, tick, Mode::Repos, None)
})?;
tick = tick.wrapping_add(1);
if event::poll(Duration::from_millis(80))? {
let Event::Key(key) = event::read()? else {
continue;
};
if key.kind == KeyEventKind::Press && matches!(key.code, KeyCode::Esc) {
return Err(ThesaError::Message("aborted by user".to_string()));
}
}
}
}
fn fetch_models_tui(
terminal: &mut Terminal,
client: &Client,
target: &ModelTarget,
args: &Cli,
) -> Result<Vec<ModelItem>> {
let client = client.clone();
let target = target.clone();
let args_clone = args.clone();
let loading_label = match &target {
ModelTarget::Org(org) => format!("loading models for {org} from {}", args.provider.label()),
ModelTarget::Model(id) => format!("loading model {id} from {}", args.provider.label()),
ModelTarget::Top => format!("loading top models from {}", args.provider.label()),
ModelTarget::Latest => {
format!("loading latest models from {}", args.provider.label())
}
};
let (tx, rx) = mpsc::channel();
thread::spawn(move || {
let result = fetch_models_for_target(&client, &target, &args_clone);
let _ = tx.send(result);
});
let mut tick = 0usize;
loop {
if let Ok(result) = rx.try_recv() {
return result;
}
draw_full_frame(terminal, |frame| {
draw_loading_tui(
frame,
&loading_label,
tick,
Mode::Models,
Some(args.provider),
)
})?;
tick = tick.wrapping_add(1);
if event::poll(Duration::from_millis(80))? {
let Event::Key(key) = event::read()? else {
continue;
};
if key.kind == KeyEventKind::Press && matches!(key.code, KeyCode::Esc) {
return Err(ThesaError::Message("aborted by user".to_string()));
}
}
}
}
fn draw_loading_tui(
frame: &mut Frame<'_>,
label: &str,
tick: usize,
mode: Mode,
provider: Option<ModelProvider>,
) {
let area = frame.area();
paint_surface(frame, area);
if area.width < 30 || area.height < 8 {
let message = format!("thesa\n{label}");
frame.render_widget(
Paragraph::from_text(Text::new(vec![Line::styled(
&message,
Style::default().fg(theme_white()),
)])),
area,
);
return;
}
let layout = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(header_height(area)),
Constraint::Min(5),
Constraint::Length(2),
])
.split(area);
let mode_label = match mode {
Mode::Repos => "github",
Mode::Models => match provider.unwrap_or(ModelProvider::Hf) {
ModelProvider::Hf => "huggingface",
ModelProvider::Ollama => "ollama",
ModelProvider::Civitai => "civitai",
},
Mode::Datasets => "baseforge",
Mode::Sites => "siteforge",
};
draw_thesa_header(frame, layout[0], tick, label, mode_label);
let loader = Loader::with_config(
LoaderKind::Tqdm,
LoaderConfig::default()
.with_size(layout[1].width.saturating_sub(4).max(12) as usize, 1)
.with_label(mode_label)
.with_unit(match mode {
Mode::Repos => "repo",
Mode::Models => "model",
Mode::Datasets => "dataset",
Mode::Sites => "site",
})
.with_fraction(false),
);
let loader_line = loader.line(
tick,
LoaderProgress::from_counts((tick % 97) as u64 + 1, 100),
);
let body = Layout::default()
.direction(Direction::Horizontal)
.constraints(if layout[1].width >= 74 {
[Constraint::Percentage(55), Constraint::Percentage(45)]
} else {
[Constraint::Percentage(100), Constraint::Percentage(0)]
})
.split(layout[1]);
let panel = Paragraph::from_text(Text::new(vec![
section_line("Loading"),
Line::styled(label, Style::default().fg(theme_white())),
Line::raw(&loader_line),
]))
.wrap(WrapMode::Character { trim: false });
frame.render_block_widget(
body[0],
panel,
knott_block(" Aisling Loader ", theme_blue()),
);
if body[1].width > 0 {
render_aisling_panel(
frame,
body[1],
" scrin aisling wipe ",
EffectKind::Wipe,
tick,
label,
);
}
draw_thesa_bottom_bar(frame, layout[2], label, &["Esc cancel"]);
}
fn pick_repositories_tui_inner(
terminal: &mut Terminal,
repos: &[RepoItem],
batch_size: &mut usize,
) -> Result<Vec<RepoItem>> {
if repos.is_empty() {
return Ok(vec![]);
}
let mut query = String::new();
let mut selected = BTreeSet::new();
let mut cursor = 0usize;
let mut frame_index = 0usize;
let mut batch_overlay = false;
let mut status =
"Space checks repos; Enter clones checked repos, or visible if none checked".to_string();
loop {
let filtered = filtered_repo_indices(repos, &query);
if cursor >= filtered.len() {
cursor = filtered.len().saturating_sub(1);
}
let page_step = picker_page_step(terminal.size().height);
draw_full_frame(terminal, |frame| {
draw_repo_picker_tui(
frame,
RepoPickerView {
repos,
filtered: &filtered,
selected: &selected,
cursor,
query: &query,
status: &status,
tick: frame_index,
batch_size: *batch_size,
batch_overlay,
},
);
})?;
frame_index = frame_index.wrapping_add(1);
if !event::poll(Duration::from_millis(50))? {
continue;
}
let input_event = event::read()?;
if let Event::Mouse(mouse) = input_event {
if !batch_overlay {
if let Some(inner) = picker_list_inner(terminal.size(), 96) {
match list_mouse_action(inner, filtered.len(), cursor, &mouse) {
Some(ListMouseAction::Hover(index)) => cursor = index,
Some(ListMouseAction::Click(index)) => {
cursor = index;
if let Some(repo_index) = filtered.get(cursor).copied() {
if !selected.insert(repo_index) {
selected.remove(&repo_index);
}
let checked = selected.len();
status = if checked == 0 {
"nothing checked; Enter clones visible repos".to_string()
} else {
format!("{checked} checked; Enter clones checked repos only")
};
}
}
Some(ListMouseAction::ScrollUp) => cursor = cursor.saturating_sub(1),
Some(ListMouseAction::ScrollDown) if cursor + 1 < filtered.len() => {
cursor += 1;
}
Some(ListMouseAction::ScrollDown) | None => {}
}
}
}
continue;
}
let Event::Key(key) = input_event else {
continue;
};
if key.kind != KeyEventKind::Press {
continue;
}
if batch_overlay {
match key.code {
KeyCode::Esc | KeyCode::Char('b') | KeyCode::Enter => {
batch_overlay = false;
status = format!("batch size set to {}; Enter clones repos", *batch_size);
}
KeyCode::Char(ch) => {
if let Some(value) = batch_size_from_key(ch) {
*batch_size = value;
batch_overlay = false;
status = format!("batch size set to {value}; Enter clones repos");
}
}
KeyCode::Left | KeyCode::Up => {
*batch_size = previous_batch_size(*batch_size);
status = format!("batch size set to {}; Enter accepts", *batch_size);
}
KeyCode::Right | KeyCode::Down => {
*batch_size = next_batch_size(*batch_size);
status = format!("batch size set to {}; Enter accepts", *batch_size);
}
_ => {}
}
continue;
}
match key.code {
KeyCode::Esc => return Err(ThesaError::Message("aborted by user".to_string())),
KeyCode::Enter => {
let picked = if selected.is_empty() {
filtered
} else {
selected.iter().copied().collect::<Vec<_>>()
};
if picked.is_empty() {
status = "no repositories selected or visible".to_string();
continue;
}
return Ok(picked
.into_iter()
.map(|index| repos[index].clone())
.collect());
}
KeyCode::Up => cursor = cursor.saturating_sub(1),
KeyCode::Down => {
if cursor + 1 < filtered.len() {
cursor += 1;
}
}
KeyCode::PageUp => {
let step = page_step.min(filtered.len().max(1));
cursor = cursor.saturating_sub(step);
status = if filtered.is_empty() {
"no repositories to move to".to_string()
} else {
format!("moved up by {step}")
};
}
KeyCode::PageDown => {
if filtered.is_empty() {
status = "no repositories to move to".to_string();
} else {
let step = page_step.min(filtered.len());
cursor = (cursor + step).min(filtered.len() - 1);
status = format!("moved down by {step}");
}
}
KeyCode::Home => {
if !filtered.is_empty() {
cursor = 0;
status = "jumped to first repo".to_string();
} else {
status = "no repositories to move to".to_string();
}
}
KeyCode::End => {
if !filtered.is_empty() {
cursor = filtered.len() - 1;
status = "jumped to last repo".to_string();
} else {
status = "no repositories to move to".to_string();
}
}
KeyCode::Char(' ') => {
if let Some(repo_index) = filtered.get(cursor).copied() {
if !selected.insert(repo_index) {
selected.remove(&repo_index);
}
let checked = selected.len();
status = if checked == 0 {
"nothing checked; Enter clones visible repos".to_string()
} else {
format!("{checked} checked; Enter clones checked repos only")
};
}
}
KeyCode::Char('a') => {
for repo_index in &filtered {
selected.insert(*repo_index);
}
status = format!("checked {} visible repos", filtered.len());
}
KeyCode::Char('c') => {
selected.clear();
query.clear();
cursor = 0;
status = "cleared filter and selection".to_string();
}
KeyCode::Char('b') => {
batch_overlay = true;
status = "choose batch size: 1 2 3 6 8".to_string();
}
KeyCode::Backspace => {
query.pop();
cursor = 0;
}
KeyCode::Char(ch) => {
query.push(ch);
cursor = 0;
}
_ => {}
}
}
}
fn filtered_repo_indices(repos: &[RepoItem], query: &str) -> Vec<usize> {
if query.trim().is_empty() {
return (0..repos.len()).collect();
}
let query = query.to_lowercase();
repos
.iter()
.enumerate()
.filter(|(_, repo)| {
repo.name.to_lowercase().contains(&query)
|| repo.full_name.to_lowercase().contains(&query)
})
.map(|(index, _)| index)
.collect()
}
struct RepoPickerView<'a> {
repos: &'a [RepoItem],
filtered: &'a [usize],
selected: &'a BTreeSet<usize>,
cursor: usize,
query: &'a str,
status: &'a str,
tick: usize,
batch_size: usize,
batch_overlay: bool,
}
fn draw_repo_picker_tui(frame: &mut Frame<'_>, view: RepoPickerView<'_>) {
let RepoPickerView {
repos,
filtered,
selected,
cursor,
query,
status,
tick,
batch_size,
batch_overlay,
} = view;
let area = frame.area();
paint_surface(frame, area);
if area.width < 28 || area.height < 8 {
let line = format!(
"repos {} visible / {} total\nfilter> {}\n{}",
filtered.len(),
repos.len(),
query,
status,
);
frame.render_widget(
Paragraph::from_text(Text::new(vec![Line::styled(
&line,
Style::default().fg(theme_white()),
)])),
area,
);
return;
}
let layout = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(header_height(area)),
Constraint::Min(8),
Constraint::Length(2),
])
.split(area);
draw_thesa_header(frame, layout[0], tick, status, "repo picker");
let body = Layout::default()
.direction(Direction::Horizontal)
.constraints(if layout[1].width >= 96 {
[Constraint::Percentage(72), Constraint::Percentage(28)]
} else {
[Constraint::Percentage(100), Constraint::Percentage(0)]
})
.split(layout[1]);
let clone_scope = if selected.is_empty() {
format!("visible: {}", filtered.len())
} else {
format!("checked only: {}", selected.len())
};
let items = filtered
.iter()
.map(|repo_index| {
let repo = &repos[*repo_index];
let marker = if selected.contains(repo_index) {
"[x]"
} else {
"[ ]"
};
ListItem::with_line(Line::new(vec![
Span::styled(
marker,
Style::default()
.fg(theme_green())
.add_modifier(Modifier::BOLD),
),
Span::raw(" "),
Span::styled(&repo.full_name, Style::default().fg(theme_white())),
]))
})
.collect::<Vec<_>>();
let mut state = ListState::default();
if !filtered.is_empty() {
state.select(Some(cursor));
}
let list_title = format!(
" Repositories | filter: {} | {} | batch: {} ",
if query.is_empty() { "all" } else { query },
clone_scope,
batch_size,
);
let list = List::new(&items)
.with_highlight_style(selected_style(tick))
.with_highlight_symbol("> ");
frame.render_block_stateful_list(
body[0],
&list,
&mut state,
knott_block(&list_title, theme_blue()),
);
if body[1].width > 0 {
let side = Layout::default()
.direction(Direction::Vertical)
.constraints([Constraint::Percentage(62), Constraint::Percentage(38)])
.split(body[1]);
let sidebar = Paragraph::from_text(Text::new(vec![
section_line("Controls"),
help_pair_line("type", "filter public repos"),
help_pair_line("mouse", "hover focus, click check"),
help_pair_line("Space", "check/uncheck repo"),
help_pair_line("a", "check visible"),
help_pair_line("b", "batch size overlay"),
help_pair_line("c", "clear"),
help_pair_line("PgUp/PgDown", "jump pages"),
help_pair_line("Home/End", "first/last"),
help_pair_line("Enter", "checked only, or visible"),
]));
frame.render_block_widget(side[0], sidebar, knott_block(" controls ", theme_dim()));
render_aisling_panel(
frame,
side[1],
" scrin aisling waves ",
EffectKind::Waves,
tick,
"repos",
);
}
draw_thesa_bottom_bar(
frame,
layout[2],
status,
&[
"Up/Down move",
"PgUp/PgDown jump",
"Home/End jump",
"Mouse hover/click",
"Space check",
"b batch",
"Enter clone",
"Esc quit",
],
);
if batch_overlay {
draw_batch_overlay(frame, area, batch_size);
}
}
fn clone_repositories_scrin(
repos: &[RepoItem],
output: &Path,
args: &Cli,
ui_state: &mut UiState,
) -> Result<CloneSummary> {
let mut session = TerminalSession::new()?;
clone_repositories_scrin_inner(&mut session.terminal, repos, output, args, ui_state)
}
fn clone_repositories_scrin_inner(
terminal: &mut Terminal,
repos: &[RepoItem],
output: &Path,
args: &Cli,
ui_state: &mut UiState,
) -> Result<CloneSummary> {
let batch_size = args.concurrency.max(1);
let total = repos.len();
let total_batches = total.div_ceil(batch_size);
let mut summary = CloneSummary::default();
let mut effect_kind = EffectKind::Waves;
let started_at = Instant::now();
let mut tick = 0usize;
let mut batch_index = 1;
while batch_index <= total {
let batch_end = (batch_index + batch_size - 1).min(total);
let batch = &repos[batch_index - 1..batch_end];
let names = batch
.iter()
.map(|repo| repo.full_name.clone())
.collect::<Vec<_>>();
let mut state = vec![BatchState::Pending; batch.len()];
let (tx, rx) = mpsc::channel::<(usize, BatchEvent)>();
let mut handles = Vec::new();
for (offset, repo) in batch.iter().enumerate() {
let repo = repo.clone();
let output = output.to_path_buf();
let args = args.clone();
let tx = tx.clone();
handles.push((
repo.full_name.clone(),
thread::spawn(move || {
let started = tx.send((offset, BatchEvent::Started));
if started.is_err() {
return;
}
let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
clone_repository(&repo, &output, &args)
}));
let event = match outcome {
Ok(Ok(CloneStatus::Cloned)) => BatchEvent::Result(BatchResult::Done),
Ok(Ok(CloneStatus::Skipped)) => BatchEvent::Result(BatchResult::Skipped),
Ok(Err(err)) => BatchEvent::Result(BatchResult::Error(err.to_string())),
Err(_) => BatchEvent::Result(BatchResult::Error(
"worker thread panicked while cloning".to_string(),
)),
};
let _ = tx.send((offset, event));
}),
));
}
drop(tx);
let mut completed = 0;
while completed < batch.len() {
while event::poll(Duration::from_millis(0))? {
if let Event::Key(key) = event::read()? {
match key.code {
KeyCode::Char('q') | KeyCode::Esc => ui_state.abort_requested = true,
KeyCode::Char('t') => ui_state.show_details = !ui_state.show_details,
KeyCode::Char('m') => effect_kind = EffectKind::Waves,
KeyCode::Char('b') => effect_kind = EffectKind::Burn,
KeyCode::Char('w') => effect_kind = EffectKind::Wipe,
_ => {}
}
}
}
while let Ok((offset, event)) = rx.try_recv() {
match event {
BatchEvent::Started => state[offset] = BatchState::Running,
BatchEvent::Result(BatchResult::Done) => {
state[offset] = BatchState::Done;
summary.cloned += 1;
completed += 1;
}
BatchEvent::Result(BatchResult::Skipped) => {
state[offset] = BatchState::Skipped;
summary.skipped += 1;
completed += 1;
}
BatchEvent::Result(BatchResult::Error(message)) => {
state[offset] = BatchState::Failed;
summary
.failed
.push(format!("{}: {}", names[offset], message));
completed += 1;
}
}
}
draw_full_frame(terminal, |frame| {
render_clone_dashboard(
frame,
CloneDashboard {
names: &names,
state: &state,
summary: &summary,
batch_index,
batch_size,
total_repos: total,
total_batches,
tick,
elapsed: started_at.elapsed(),
effect_kind,
ui_state,
},
)
})?;
tick = tick.wrapping_add(1);
if completed < batch.len() {
thread::sleep(Duration::from_millis(80));
}
}
for (repo_name, handle) in handles {
if handle.join().is_err() {
summary
.failed
.push(format!("{repo_name}: worker thread panicked"));
}
}
draw_full_frame(terminal, |frame| {
render_clone_dashboard(
frame,
CloneDashboard {
names: &names,
state: &state,
summary: &summary,
batch_index,
batch_size,
total_repos: total,
total_batches,
tick,
elapsed: started_at.elapsed(),
effect_kind,
ui_state,
},
)
})?;
if ui_state.abort_requested {
break;
}
batch_index = batch_end + 1;
}
Ok(summary)
}
struct CloneDashboard<'a> {
names: &'a [String],
state: &'a [BatchState],
summary: &'a CloneSummary,
batch_index: usize,
batch_size: usize,
total_repos: usize,
total_batches: usize,
tick: usize,
elapsed: Duration,
effect_kind: EffectKind,
ui_state: &'a UiState,
}
fn render_clone_dashboard(frame: &mut Frame, dashboard: CloneDashboard<'_>) {
let CloneDashboard {
names,
state,
summary,
batch_index,
batch_size,
total_repos,
total_batches,
tick,
elapsed,
effect_kind,
ui_state,
} = dashboard;
let area = frame.area();
paint_surface(frame, area);
let overall_done = summary.cloned + summary.skipped + summary.failed.len();
let status = if ui_state.abort_requested {
"stopping after current batch"
} else {
"cloning"
};
if area.width < 36 || area.height < 10 {
let loader = Loader::with_config(
LoaderKind::Tqdm,
LoaderConfig::default()
.with_size(area.width.saturating_sub(4) as usize, 1)
.with_label("thesa")
.with_unit("repo")
.with_fraction(true),
);
let line = loader.line(
tick,
LoaderProgress::from_counts(overall_done as u64, total_repos as u64),
);
let panel = Paragraph::from_text(Text::new(vec![
Line::styled(
"thesa",
Style::default()
.fg(theme_white())
.add_modifier(Modifier::BOLD),
),
Line::styled(&line, Style::default().fg(theme_white())),
Line::styled(status, Style::default().fg(theme_dim())),
]));
frame.render_block_widget(area, panel, knott_block(" thesa ", theme_blue()));
return;
}
let root = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(header_height(area)),
Constraint::Min(8),
Constraint::Length(2),
])
.split(area);
draw_thesa_header(frame, root[0], tick, status, "clone dashboard");
let body = Layout::default()
.direction(Direction::Horizontal)
.constraints(if root[1].width >= 104 {
[Constraint::Percentage(68), Constraint::Percentage(32)]
} else {
[Constraint::Percentage(100), Constraint::Percentage(0)]
})
.split(root[1]);
let left = Layout::default()
.direction(Direction::Vertical)
.constraints([Constraint::Length(5), Constraint::Min(5)])
.split(body[0]);
let current_batch = ((batch_index - 1) / batch_size) + 1;
render_command_progress_panel(
frame,
left[0],
" command progress ",
CommandProgress {
verb: "clone repositories",
unit: "repo",
done_label: "cloned",
done: summary.cloned,
skipped: summary.skipped,
failed: summary.failed.len(),
total: total_repos,
batch_index: current_batch,
batch_size,
total_batches,
elapsed,
},
tick,
);
let repo_loader = Loader::with_config(
LoaderKind::Tqdm,
LoaderConfig::default()
.with_size(left[1].width.saturating_sub(34).max(10) as usize, 1)
.with_label("repo")
.with_unit("step")
.with_fraction(true),
);
let mut rows = Vec::new();
if ui_state.show_details {
for (index, (name, repo_state)) in names.iter().zip(state.iter()).enumerate() {
rows.push(repo_loader_row(
&repo_loader,
tick + index,
name,
*repo_state,
));
}
} else {
rows.push(Line::styled(
"details hidden; press t to show concurrent repo loaders",
Style::default().fg(theme_dim()),
));
}
let repos = Paragraph::from_text(Text::new(rows)).wrap(WrapMode::Character { trim: false });
frame.render_block_widget(
left[1],
repos,
knott_block(" Concurrent Pulls | Aisling Tqdm ", theme_blue()),
);
if body[1].width > 0 {
let (pending, running, done, skipped, failed) = batch_state_counts(state);
let side_layout = Layout::default()
.direction(Direction::Vertical)
.constraints([
Constraint::Length(8),
Constraint::Length(8),
Constraint::Min(4),
])
.split(body[1]);
render_command_info_panel(
frame,
side_layout[0],
" telemetry ",
vec![
section_line("Batch State"),
help_pair_line("running", &running.to_string()),
help_pair_line("pending", &pending.to_string()),
help_pair_line("done", &done.to_string()),
help_pair_line("skipped", &skipped.to_string()),
help_pair_line("failed", &failed.to_string()),
],
theme_blue(),
);
let side_lines = vec![
section_line("Controls"),
help_pair_line("q / Esc", "stop after current batch"),
help_pair_line("t", "toggle repo rows"),
help_pair_line("m", "motion effect"),
help_pair_line("b", "burn effect"),
help_pair_line("w", "wipe effect"),
];
render_command_info_panel(
frame,
side_layout[1],
" command keys ",
side_lines,
theme_dim(),
);
render_aisling_panel(
frame,
side_layout[2],
&format!(" scrin aisling {} ", effect_kind.name()),
effect_kind,
tick,
"repos\nclone",
);
}
draw_thesa_bottom_bar(
frame,
root[2],
status,
&["q stop", "t details", "m motion", "b burn", "w wipe"],
);
}
fn batch_size_from_key(ch: char) -> Option<usize> {
match ch {
'1' => Some(1),
'2' => Some(2),
'3' => Some(3),
'6' => Some(6),
'8' => Some(8),
_ => None,
}
}
fn batch_state_counts(state: &[BatchState]) -> (usize, usize, usize, usize, usize) {
let mut pending = 0;
let mut running = 0;
let mut done = 0;
let mut skipped = 0;
let mut failed = 0;
for item in state {
match item {
BatchState::Pending => pending += 1,
BatchState::Running => running += 1,
BatchState::Done => done += 1,
BatchState::Skipped => skipped += 1,
BatchState::Failed => failed += 1,
}
}
(pending, running, done, skipped, failed)
}
fn previous_batch_size(current: usize) -> usize {
let index = BATCH_SIZE_CHOICES
.iter()
.position(|value| *value == current)
.unwrap_or(0);
BATCH_SIZE_CHOICES[index.saturating_sub(1)]
}
fn next_batch_size(current: usize) -> usize {
let index = BATCH_SIZE_CHOICES
.iter()
.position(|value| *value == current)
.unwrap_or(0);
BATCH_SIZE_CHOICES[(index + 1).min(BATCH_SIZE_CHOICES.len() - 1)]
}
fn draw_batch_overlay(frame: &mut Frame<'_>, area: Rect, batch_size: usize) {
let width = area.width.min(58).max(30.min(area.width));
let height = area.height.min(11).max(7.min(area.height));
let x = area.x + area.width.saturating_sub(width) / 2;
let y = area.y + area.height.saturating_sub(height) / 2;
let overlay = Rect::new(x, y, width, height);
frame.render_widget(Clear::new(), overlay);
let mut choice_spans = Vec::new();
for value in BATCH_SIZE_CHOICES {
if !choice_spans.is_empty() {
choice_spans.push(Span::raw(" "));
}
let style = if value == batch_size {
Style::default()
.bg(TuiColor::WHITE)
.fg(TuiColor::BLACK)
.add_modifier(Modifier::BOLD)
} else {
Style::default()
.fg(theme_blue())
.add_modifier(Modifier::BOLD)
};
choice_spans.push(Span::styled(&format!(" {value} "), style));
}
let panel = Paragraph::from_text(Text::new(vec![
section_line("Batch Size"),
Line::styled(
"How many items to process at once",
Style::default().fg(theme_dim()),
),
Line::raw(""),
Line::new(choice_spans),
Line::raw(""),
help_pair_line("1/2/3/6/8", "select and close"),
help_pair_line("Left/Right", "cycle choices"),
help_pair_line("b / Enter", "close overlay"),
]))
.wrap(WrapMode::Character { trim: false });
frame.render_block_widget(overlay, panel, knott_block(" Batch Options ", theme_red()));
}
fn draw_thesa_header(frame: &mut Frame<'_>, area: Rect, tick: usize, status: &str, mode: &str) {
if area.height == 0 {
return;
}
if area.height >= 5 && area.width >= 88 {
let deck = Layout::default()
.direction(Direction::Horizontal)
.constraints([Constraint::Percentage(62), Constraint::Percentage(38)])
.split(area);
draw_thesa_header_text(frame, deck[0], tick, status, mode);
render_aisling_panel(
frame,
deck[1],
" scrin aisling matrix ",
EffectKind::Matrix,
tick,
"thesa",
);
return;
}
draw_thesa_header_text(frame, area, tick, status, mode);
}
fn draw_thesa_header_text(
frame: &mut Frame<'_>,
area: Rect,
tick: usize,
status: &str,
mode: &str,
) {
let accent = animated_accent(tick);
let mut primary = Vec::new();
primary.push(Span::styled(
" thesa ",
Style::default()
.bg(accent)
.fg(TuiColor::BLACK)
.add_modifier(Modifier::BOLD),
));
append_status_chip(&mut primary, "mode", mode, theme_blue());
append_status_chip(&mut primary, "state", status, theme_green());
append_status_chip(&mut primary, "effect", "aisling", theme_red());
append_status_chip(&mut primary, "frame", tick.to_string(), theme_dim());
let mut lines = vec![Line::new(primary)];
if area.height >= 5 {
lines.push(Line::raw(""));
lines.push(Line::new(vec![
Span::styled("archive ", Style::default().fg(theme_dim())),
Span::styled("public repos", Style::default().fg(theme_blue())),
Span::styled(" + ", Style::default().fg(theme_dim())),
Span::styled("models", Style::default().fg(theme_green())),
Span::styled(" with live scrin panes", Style::default().fg(theme_dim())),
]));
}
let header = Paragraph::from_text(Text::new(lines)).alignment(Alignment::Left);
frame.render_block_widget(area, header, knott_block(" thesa command deck ", accent));
}
fn draw_thesa_bottom_bar(frame: &mut Frame<'_>, area: Rect, status: &str, keys: &[&str]) {
if area.height == 0 {
return;
}
frame
.buffer()
.fill(area, ' ', theme_white(), Some(theme_surface()));
let mut status_spans = vec![
Span::styled("status: ", Style::default().fg(theme_dim())),
Span::styled(status, Style::default().fg(theme_white())),
];
append_status_chip(&mut status_spans, "ui", "scrin", theme_blue());
append_status_chip(&mut status_spans, "fx", "aisling", theme_green());
append_status_chip(&mut status_spans, "hover", "active", theme_dim());
let key_line = Line::new(
keys.iter()
.flat_map(|key| {
let (head, detail) = key.split_once(' ').unwrap_or((key, ""));
let detail_spans = if detail.is_empty() {
vec![Span::raw(" ")]
} else {
vec![Span::raw(&format!(" {detail} "))]
};
let mut spans = Vec::new();
spans.push(Span::styled(head, key_style()));
spans.extend(detail_spans);
spans
})
.collect::<Vec<_>>(),
);
let lines = if area.height <= 1 {
vec![Line::new(status_spans)]
} else {
vec![Line::new(status_spans), key_line]
};
frame.render_widget(
Paragraph::from_text(Text::new(lines)).with_style(Style::default().fg(theme_white())),
area,
);
normalize_bg(frame.buffer(), area, theme_surface());
}
fn knott_block<'a>(title: &'a str, color: TuiColor) -> Block<'a> {
Block::new(title)
.with_borders(BorderStyle::Rounded)
.with_border_color(color)
.with_bg(theme_surface())
}
fn panel_inner(area: Rect) -> Rect {
knott_block("", theme_dim()).inner(area)
}
fn theme_white() -> TuiColor {
TuiColor::rgb(201, 209, 217)
}
fn theme_red() -> TuiColor {
TuiColor::rgb(248, 81, 73)
}
fn theme_blue() -> TuiColor {
TuiColor::rgb(88, 166, 255)
}
fn theme_green() -> TuiColor {
TuiColor::rgb(63, 185, 80)
}
fn theme_dim() -> TuiColor {
TuiColor::rgb(110, 118, 129)
}
fn theme_bg() -> TuiColor {
TuiColor::rgb(13, 17, 23)
}
fn theme_surface() -> TuiColor {
TuiColor::rgb(22, 27, 34)
}
fn prompt_style() -> Style {
Style::default()
.fg(theme_blue())
.add_modifier(Modifier::BOLD)
}
fn key_style() -> Style {
Style::default()
.bg(theme_blue())
.fg(TuiColor::BLACK)
.add_modifier(Modifier::BOLD)
}
fn selected_style(tick: usize) -> Style {
Style::default()
.bg(animated_accent(tick))
.fg(TuiColor::BLACK)
.add_modifier(Modifier::BOLD)
}
fn header_height(area: Rect) -> u16 {
if area.height >= 30 { 5 } else { 3 }
}
fn animated_accent(tick: usize) -> TuiColor {
let _ = tick;
theme_blue()
}
fn section_line(label: &str) -> Line {
Line::new(vec![Span::styled(
label,
Style::default()
.fg(theme_white())
.add_modifier(Modifier::BOLD),
)])
}
fn help_pair_line(head: &str, detail: &str) -> Line {
Line::new(vec![
Span::styled(
&format!("{head:<12}"),
Style::default()
.fg(theme_blue())
.add_modifier(Modifier::BOLD),
),
Span::styled(detail, Style::default().fg(theme_dim())),
])
}
fn append_status_chip(
spans: &mut Vec<Span>,
label: &str,
value: impl Into<String>,
color: TuiColor,
) {
if !spans.is_empty() {
spans.push(Span::raw(" "));
}
spans.push(Span::styled("[", Style::default().fg(theme_dim())));
spans.push(Span::styled(label, Style::default().fg(theme_dim())));
spans.push(Span::raw(" "));
let value = value.into();
spans.push(Span::styled(
&value,
Style::default().fg(color).add_modifier(Modifier::BOLD),
));
spans.push(Span::styled("]", Style::default().fg(theme_dim())));
}
fn paint_surface(frame: &mut Frame<'_>, area: Rect) {
if area.width == 0 || area.height == 0 {
return;
}
frame
.buffer()
.fill(area, ' ', theme_white(), Some(theme_bg()));
}
fn normalize_bg(buffer: &mut Buffer, area: Rect, bg: TuiColor) {
for y in area.y as usize..area.bottom() as usize {
for x in area.x as usize..area.right() as usize {
let Some(cell) = buffer.get(x, y).copied() else {
continue;
};
if cell.bg.is_none() {
buffer.set(
x,
y,
scrin::core::buffer::Cell {
bg: Some(bg),
..cell
},
);
}
}
}
}
fn render_aisling_panel(
frame: &mut Frame<'_>,
area: Rect,
title: &str,
kind: EffectKind,
tick: usize,
text: &str,
) {
if area.width < 4 || area.height < 4 {
return;
}
let block = knott_block(title, animated_accent(tick)).with_borders(BorderStyle::Thick);
block.render(frame.buffer(), area);
let inner = block.inner(area);
if inner.width == 0 || inner.height == 0 {
return;
}
frame
.buffer()
.fill(inner, ' ', theme_white(), Some(theme_surface()));
render_aisling_effect(frame, inner, kind, tick, text);
normalize_bg(frame.buffer(), inner, theme_surface());
}
struct AislingTextPanel<'a> {
title: &'a str,
kind: EffectKind,
tick: usize,
effect_text: &'a str,
lines: Vec<Line>,
color: TuiColor,
}
fn render_aisling_text_panel(frame: &mut Frame<'_>, area: Rect, panel: AislingTextPanel<'_>) {
let AislingTextPanel {
title,
kind,
tick,
effect_text,
lines,
color,
} = panel;
if area.width < 4 || area.height < 4 {
return;
}
let block = knott_block(title, color).with_borders(BorderStyle::Thick);
block.render(frame.buffer(), area);
let inner = block.inner(area);
if inner.width == 0 || inner.height == 0 {
return;
}
frame
.buffer()
.fill(inner, ' ', theme_white(), Some(theme_surface()));
render_aisling_effect(frame, inner, kind, tick, effect_text);
let panel = Paragraph::from_text(Text::new(lines)).wrap(WrapMode::Character { trim: false });
panel.render(frame.buffer(), inner);
normalize_bg(frame.buffer(), inner, theme_surface());
}
fn render_aisling_effect(
frame: &mut Frame<'_>,
area: Rect,
kind: EffectKind,
tick: usize,
text: &str,
) {
if area.width == 0 || area.height == 0 {
return;
}
let mut player = EffectPlayer::new(kind, text)
.with_size(area.width as usize, area.height as usize)
.with_duration(96)
.with_seed(17)
.with_gradient_colors(
vec![
theme_blue(),
theme_green(),
theme_red(),
TuiColor::rgb(163, 113, 247),
],
45.0,
)
.with_accent(theme_blue());
player.set_frame(tick % player.total_frames().max(1));
player.render_to_buffer(frame.buffer(), area);
}
struct CommandProgress<'a> {
verb: &'a str,
unit: &'a str,
done_label: &'a str,
done: usize,
skipped: usize,
failed: usize,
total: usize,
batch_index: usize,
batch_size: usize,
total_batches: usize,
elapsed: Duration,
}
fn render_command_progress_panel(
frame: &mut Frame<'_>,
area: Rect,
title: &str,
data: CommandProgress<'_>,
tick: usize,
) {
if area.width < 4 || area.height < 4 {
return;
}
let block = knott_block(title, theme_green()).with_borders(BorderStyle::Thick);
block.render(frame.buffer(), area);
let inner = block.inner(area);
if inner.width == 0 || inner.height == 0 {
return;
}
frame
.buffer()
.fill(inner, ' ', theme_white(), Some(theme_surface()));
let completed = data.done + data.skipped + data.failed;
let progress_area = Rect::new(
inner.x,
inner.bottom().saturating_sub(1),
inner.width,
1.min(inner.height),
);
let text_height = inner.height.saturating_sub(progress_area.height);
let text_area = Rect::new(inner.x, inner.y, inner.width, text_height);
let mut chips = Vec::new();
append_status_chip(
&mut chips,
data.done_label,
data.done.to_string(),
theme_green(),
);
append_status_chip(
&mut chips,
"skipped",
data.skipped.to_string(),
TuiColor::YELLOW,
);
append_status_chip(&mut chips, "failed", data.failed.to_string(), theme_red());
append_status_chip(
&mut chips,
"elapsed",
format!("{}s", data.elapsed.as_secs()),
theme_dim(),
);
let text = Paragraph::from_text(Text::new(vec![
Line::new(chips),
Line::new(vec![
Span::styled("command ", Style::default().fg(theme_dim())),
Span::styled(data.verb, Style::default().fg(theme_white())),
]),
Line::styled(
&format!(
"batch {}/{} concurrency {} total {} {}",
data.batch_index, data.total_batches, data.batch_size, data.total, data.unit
),
Style::default().fg(theme_dim()),
),
]));
text.render(frame.buffer(), text_area);
normalize_bg(frame.buffer(), text_area, theme_surface());
let loader = LoaderPlayer::new(LoaderKind::Bar)
.with_size(progress_area.width as usize, 1)
.with_label("thesa".to_string())
.with_unit(data.unit)
.with_fraction(true)
.with_accent(theme_blue())
.with_gradient_colors(vec![theme_blue(), theme_green()], 0.0);
loader.render(
tick,
LoaderProgress::from_counts(completed as u64, data.total.max(1) as u64),
frame.buffer(),
progress_area,
);
normalize_bg(frame.buffer(), progress_area, theme_surface());
}
fn render_command_info_panel(
frame: &mut Frame<'_>,
area: Rect,
title: &str,
lines: Vec<Line>,
color: TuiColor,
) {
let panel = Paragraph::from_text(Text::new(lines)).wrap(WrapMode::Character { trim: false });
frame.render_block_widget(area, panel, knott_block(title, color));
}
fn repo_loader_row(loader: &Loader, tick: usize, name: &str, state: BatchState) -> Line {
let done = matches!(
state,
BatchState::Done | BatchState::Skipped | BatchState::Failed
);
let loader_line = loader.line(tick, LoaderProgress::from_counts(done as u64, 1));
let style = match state {
BatchState::Pending => Style::default().fg(TuiColor::DARK_GRAY),
BatchState::Running => Style::default().fg(TuiColor::MAGENTA),
BatchState::Done => Style::default().fg(TuiColor::CYAN),
BatchState::Skipped => Style::default().fg(TuiColor::YELLOW),
BatchState::Failed => Style::default().fg(TuiColor::RED),
};
let state_label = format!("{:<7}", state.label());
Line::new(vec![
Span::styled(&state_label, style.add_modifier(Modifier::BOLD)),
Span::raw(" "),
Span::raw(&loader_line),
Span::raw(" "),
Span::styled(name, style),
])
}
fn clone_repositories_serial(
repos: &[RepoItem],
output: &Path,
args: &Cli,
ui_rx: &mpsc::Receiver<char>,
ui_state: &mut UiState,
) -> Result<CloneSummary> {
let spinner = Loader::with_config(
LoaderKind::Tqdm,
LoaderConfig::default()
.with_size(54, 1)
.with_label("thesa")
.with_unit("repo")
.with_fraction(true),
);
let mut stdout = io::stdout();
let mut summary = CloneSummary::default();
let total = repos.len();
for (index, repo) in repos.iter().enumerate() {
if ui_state.abort_requested {
break;
}
apply_ui_events(ui_rx, ui_state);
match clone_repository_with_progress(
repo,
output,
args,
&mut stdout,
&spinner,
index + 1,
total,
) {
Ok(CloneStatus::Cloned) => summary.cloned += 1,
Ok(CloneStatus::Skipped) => summary.skipped += 1,
Err(err) => summary.failed.push(format!("{}: {}", repo.full_name, err)),
}
}
Ok(summary)
}
fn clone_repositories_batched(
repos: &[RepoItem],
output: &Path,
args: &Cli,
ui_rx: &mpsc::Receiver<char>,
ui_state: &mut UiState,
) -> Result<CloneSummary> {
let batch_size = args.concurrency.max(1);
let total = repos.len();
let mut summary = CloneSummary::default();
let mut batch_index = 1;
let total_batches = total.div_ceil(batch_size);
let mut stop_after_batch = false;
while batch_index <= total {
let batch_end = (batch_index + batch_size - 1).min(total);
let batch = &repos[batch_index - 1..batch_end];
let batch_positions: Vec<String> =
batch.iter().map(|repo| repo.full_name.clone()).collect();
let (tx, rx) = mpsc::channel::<(usize, BatchEvent)>();
println!(
"Starting batch {}/{} ({})...",
(batch_index - 1) / batch_size + 1,
total_batches,
batch.len()
);
let mut state = vec![BatchState::Pending; batch.len()];
apply_ui_events(ui_rx, ui_state);
if ui_state.abort_requested {
break;
}
let mut handles = Vec::new();
for (offset, repo) in batch.iter().enumerate() {
let repo = repo.clone();
let output = output.to_path_buf();
let args = args.clone();
let position = batch_index + offset;
let tx = tx.clone();
handles.push((
position,
repo.full_name.clone(),
thread::spawn(move || {
let started = tx.send((offset, BatchEvent::Started));
if started.is_err() {
return;
}
let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
clone_repository(&repo, &output, &args)
}));
let event = match outcome {
Ok(Ok(CloneStatus::Cloned)) => BatchEvent::Result(BatchResult::Done),
Ok(Ok(CloneStatus::Skipped)) => BatchEvent::Result(BatchResult::Skipped),
Ok(Err(err)) => BatchEvent::Result(BatchResult::Error(err.to_string())),
Err(_) => BatchEvent::Result(BatchResult::Error(
"worker thread panicked while cloning".to_string(),
)),
};
let _ = tx.send((offset, event));
}),
));
}
drop(tx);
let mut completed = 0;
while completed < batch.len() {
let mut progress_updated = false;
apply_ui_events(ui_rx, ui_state);
if ui_state.abort_requested {
stop_after_batch = true;
}
while let Ok((offset, event)) = rx.try_recv() {
match event {
BatchEvent::Started => {
state[offset] = BatchState::Running;
progress_updated = true;
}
BatchEvent::Result(BatchResult::Done) => {
state[offset] = BatchState::Done;
summary.cloned += 1;
completed += 1;
progress_updated = true;
}
BatchEvent::Result(BatchResult::Skipped) => {
state[offset] = BatchState::Skipped;
summary.skipped += 1;
completed += 1;
progress_updated = true;
}
BatchEvent::Result(BatchResult::Error(message)) => {
state[offset] = BatchState::Failed;
summary
.failed
.push(format!("{}: {}", batch_positions[offset], message));
completed += 1;
progress_updated = true;
}
}
}
if progress_updated {
render_batch_state(
&mut io::stdout(),
BatchStateView {
batch_number: (batch_index - 1) / batch_size + 1,
total_batches,
names: batch_positions.as_slice(),
state: state.as_slice(),
show_details: ui_state.show_details,
panel_lines: &mut ui_state.panel_lines,
batch_size: batch.len(),
total_repos: total,
},
)?;
}
if completed < batch.len() {
thread::sleep(Duration::from_millis(100));
}
}
for (position, repo_name, handle) in handles {
if handle.join().is_err() {
summary.failed.push(format!(
"{repo_name} ({position}/{total}): worker thread panicked"
));
}
}
if stop_after_batch {
break;
}
batch_index = batch_end + 1;
}
Ok(summary)
}
fn apply_ui_events(ui_rx: &mpsc::Receiver<char>, ui_state: &mut UiState) {
while let Ok(command) = ui_rx.try_recv() {
match command {
't' | 'T' => {
ui_state.show_details = !ui_state.show_details;
}
'q' | 'Q' => {
ui_state.abort_requested = true;
}
_ => {}
}
}
}
struct BatchStateView<'a> {
batch_number: usize,
total_batches: usize,
names: &'a [String],
state: &'a [BatchState],
show_details: bool,
panel_lines: &'a mut usize,
batch_size: usize,
total_repos: usize,
}
fn render_batch_state(stdout: &mut io::Stdout, view: BatchStateView<'_>) -> io::Result<()> {
let BatchStateView {
batch_number,
total_batches,
names,
state,
show_details,
panel_lines,
batch_size,
total_repos,
} = view;
if !stdout.is_terminal() {
return Ok(());
}
let mut done = 0;
let mut skipped = 0;
let mut failed = 0;
let mut running = 0;
let mut pending = 0;
for entry in state {
match entry {
BatchState::Done => done += 1,
BatchState::Skipped => skipped += 1,
BatchState::Failed => failed += 1,
BatchState::Running => running += 1,
BatchState::Pending => pending += 1,
}
}
let completed = done + skipped + failed;
let loader = Loader::with_config(
LoaderKind::Pulse,
LoaderConfig::default()
.with_size(58, 1)
.with_label("thesa")
.with_unit("repo")
.with_fraction(true)
.with_gradient(thesa_gradient(), GradientDirection::Horizontal),
);
let loader_line = loader
.frame(
completed + running,
LoaderProgress::from_counts(completed as u64, batch_size as u64),
)
.to_ansi_string();
let cyan = AislingColor::rgb(0, 255, 210);
let violet = AislingColor::rgb(142, 92, 255);
let amber = AislingColor::rgb(255, 196, 92);
let rose = AislingColor::rgb(255, 80, 210);
let slate = AislingColor::rgb(132, 150, 168);
let mut lines = vec![
paint(
"+--------------------------------------------------------------------+",
violet,
),
format!(
"| {} {} | {} |",
paint_bold("THESA", cyan),
paint("public-repo collector", slate),
paint(
format!("batch {batch_number}/{total_batches} total {total_repos}"),
amber,
)
),
paint(
"+--------------------------------------------------------------------+",
violet,
),
format!("| {} |", loader_line.trim_end()),
format!(
"| {} {} {} {} {} {} {} {} {} {} |",
paint("done", cyan),
done,
paint("skip", amber),
skipped,
paint("fail", rose),
failed,
paint("run", violet),
running,
paint("wait", slate),
pending,
),
format!(
"| {} |",
paint(
"keys: t + Enter toggles detail, q + Enter stops after batch",
slate
)
),
paint(
"+--------------------------------------------------------------------+",
violet,
),
];
if show_details {
for (name, repo_state) in names.iter().zip(state.iter()) {
lines.push(format!(" {} {}", repo_state.icon(), name));
}
} else {
lines.push(paint(
" detail hidden; press t + Enter to show repos",
slate,
));
}
lines.push(String::new());
if *panel_lines > 0 {
write!(stdout, "\x1b[{}A", *panel_lines)?;
}
let written_lines = (*panel_lines).max(lines.len());
for index in 0..written_lines {
let line = lines.get(index).map(String::as_str).unwrap_or("");
write!(stdout, "\x1b[2K\r{line}\n")?;
}
*panel_lines = written_lines;
stdout.flush()?;
Ok(())
}
#[derive(Debug, Clone)]
enum BatchResult {
Done,
Skipped,
Error(String),
}
#[derive(Debug, Clone)]
enum BatchEvent {
Started,
Result(BatchResult),
}
impl BatchState {
fn label(self) -> &'static str {
match self {
BatchState::Pending => "queued",
BatchState::Running => "pulling",
BatchState::Done => "done",
BatchState::Skipped => "skip",
BatchState::Failed => "failed",
}
}
fn icon(self) -> &'static str {
match self {
BatchState::Pending => " [ ]",
BatchState::Running => "[-x]",
BatchState::Done => " [*]",
BatchState::Skipped => " [/]",
BatchState::Failed => " [!]",
}
}
}
fn prompt_target(mode: Mode, provider: Option<ModelProvider>) -> Result<String> {
loop {
let value = match mode {
Mode::Repos => prompt_line("GitHub target [owner or owner/repo]: ")?,
Mode::Models => match provider.unwrap_or(ModelProvider::Hf) {
ModelProvider::Hf => prompt_line("Model target [org, org/model, top, latest]: ")?,
ModelProvider::Ollama => {
prompt_line("Model target [name, keyword, top, latest]: ")?
}
ModelProvider::Civitai => prompt_line("Model target [id, keyword, top, latest]: ")?,
},
Mode::Datasets => prompt_line(
"Dataset target [namespace, dataset:<id>, search:<query>, top, latest]: ",
)?,
Mode::Sites => prompt_line("Siteforge URL [https://example.com]: ")?,
};
if value.is_empty() {
println!("Target is required.");
continue;
}
let ok = match mode {
Mode::Repos => parse_target(&value).is_ok(),
Mode::Models => {
parse_model_target_for_provider(&value, provider.unwrap_or(ModelProvider::Hf))
.is_ok()
}
Mode::Datasets => {
parse_dataset_target_for_provider(&value, types::DatasetProvider::Hf).is_ok()
}
Mode::Sites => parse_site_url(&value).is_ok(),
};
if ok {
return Ok(value);
}
match mode {
Mode::Repos => println!(
"Invalid target. Examples: torvalds, torvalds/linux, https://github.com/torvalds/linux"
),
Mode::Models => match provider.unwrap_or(ModelProvider::Hf) {
ModelProvider::Hf => println!(
"Invalid target. Examples: meta-llama, meta-llama/Llama-2-7b, top, latest"
),
ModelProvider::Ollama => {
println!("Invalid target. Examples: llama3, llama3:8b, latest")
}
ModelProvider::Civitai => {
println!("Invalid target. Examples: 827184, realistic-vision-v6, top, latest")
}
},
Mode::Datasets => println!(
"Invalid target. Examples: openai, dataset:squad, search:code data, top, latest"
),
Mode::Sites => {
println!("Invalid URL. Examples: https://example.com, http://127.0.0.1:8765/")
}
}
}
}
fn prompt_line(prompt: &str) -> Result<String> {
print!("{prompt}");
io::stdout().flush()?;
let mut value = String::new();
io::stdin().read_line(&mut value).map_err(ThesaError::Io)?;
Ok(value.trim().to_string())
}
fn prompt_mode() -> Result<Mode> {
println!("Archive mode:");
println!(" 1) repos - GitHub repositories");
println!(" 2) models - Hugging Face, Ollama, or CivitAI models");
println!(" 3) datasets - BaseForge dataset archives");
println!(" 4) sites - Siteforge website archives");
let value = prompt_line("Select [1/2/3/4 or r/m/d/s, default: repos]: ")?;
match value.as_str() {
"2" | "m" | "models" => Ok(Mode::Models),
"3" | "d" | "data" | "dataset" | "datasets" | "baseforge" => Ok(Mode::Datasets),
"4" | "s" | "site" | "sites" | "siteforge" | "url" => Ok(Mode::Sites),
_ => Ok(Mode::Repos),
}
}
fn prompt_model_provider(current: ModelProvider) -> Result<ModelProvider> {
println!("Model provider:");
println!(" 1) hf - Hugging Face models");
println!(" 2) ollama - local Ollama models");
println!(" 3) civitai - CivitAI models");
let value = prompt_line(&format!("Select [1/2/3 or h/o/c, default: {current}]: "))?;
match value.as_str() {
"2" | "o" | "ollama" => Ok(ModelProvider::Ollama),
"3" | "c" | "civitai" => Ok(ModelProvider::Civitai),
"1" | "h" | "hf" | "huggingface" | "hugging-face" => Ok(ModelProvider::Hf),
"" => Ok(current),
_ => {
println!("Invalid provider; keeping {current}.");
Ok(current)
}
}
}
fn prompt_hf_token() -> Result<Option<String>> {
println!("Hugging Face API may require a token for model or dataset access.");
println!("Get one at https://huggingface.co/settings/tokens");
let value = prompt_line("HF token (blank to skip, will rate-limit): ")?;
if value.is_empty() {
Ok(None)
} else {
Ok(Some(value))
}
}
fn prompt_hf_token_tui(terminal: &mut Terminal) -> Result<Option<String>> {
let mut input = String::new();
let mut frame_index = 0usize;
let status = "Hugging Face API may require a token for models or datasets".to_string();
let mut show_plain = false;
loop {
draw_full_frame(terminal, |frame| {
draw_hf_token_tui(frame, &input, &status, frame_index, show_plain);
})?;
frame_index = frame_index.wrapping_add(1);
if !event::poll(Duration::from_millis(50))? {
continue;
}
let Event::Key(key) = event::read()? else {
continue;
};
if key.kind != KeyEventKind::Press {
continue;
}
match key.code {
KeyCode::Esc => return Ok(None),
KeyCode::Enter => {
if input.is_empty() {
return Ok(None);
}
return Ok(Some(input));
}
KeyCode::Backspace => {
input.pop();
}
KeyCode::Char(ch) => {
input.push(ch);
show_plain = true;
}
_ => {}
}
}
}
fn with_auth(
mut request: reqwest::blocking::RequestBuilder,
token: Option<&str>,
) -> reqwest::blocking::RequestBuilder {
request = request.header("Accept", "application/vnd.github+json");
if let Some(token) = token {
request = request.bearer_auth(token);
}
request
}
fn parse_api_error(response: Response) -> ThesaError {
let status = response.status();
let body = response
.text()
.unwrap_or_else(|_| "<unable to read body>".to_string());
ThesaError::ApiError {
status: status.as_u16(),
body,
}
}
fn list_repos_for_owner(
client: &Client,
owner: &str,
token: Option<&str>,
) -> Result<Vec<RepoItem>> {
list_repos_with_scope(client, "users", owner, token)
}
fn list_repos_for_org(client: &Client, org: &str, token: Option<&str>) -> Result<Vec<RepoItem>> {
list_repos_with_scope(client, "orgs", org, token)
}
fn list_repos_with_scope(
client: &Client,
scope: &str,
entity: &str,
token: Option<&str>,
) -> Result<Vec<RepoItem>> {
let mut page = 1usize;
let mut repos = Vec::new();
loop {
let url = format!(
"{}/{}/{}/repos?type=public&per_page={}&page={}",
GITHUB_API, scope, entity, DEFAULT_PAGE_SIZE, page
);
let response = with_auth(client.get(&url), token).send()?;
let status = response.status();
if status == StatusCode::NOT_FOUND {
return Err(ThesaError::NotFound(entity.to_string()));
}
if !status.is_success() {
return Err(parse_api_error(response));
}
let batch: Vec<RepoResponse> = response.json()?;
let count = batch.len();
repos.extend(batch.into_iter().filter_map(|repo| {
if repo.private {
return None;
}
Some(RepoItem {
name: repo.name,
full_name: repo.full_name,
clone_url: repo.clone_url,
})
}));
if count < DEFAULT_PAGE_SIZE {
break;
}
page = page.saturating_add(1);
}
Ok(repos)
}
fn fetch_single_repo(
client: &Client,
owner: &str,
repo: &str,
token: Option<&str>,
) -> Result<RepoItem> {
let url = format!("{}/repos/{}/{}", GITHUB_API, owner, repo);
let response = with_auth(client.get(&url), token).send()?;
if response.status() == StatusCode::NOT_FOUND {
return Err(ThesaError::NotFound(format!("{owner}/{repo}")));
}
if !response.status().is_success() {
return Err(parse_api_error(response));
}
let data: RepoResponse = response.json()?;
if data.private {
return Err(ThesaError::PrivateRepo(data.full_name));
}
Ok(RepoItem {
name: data.name,
full_name: data.full_name,
clone_url: data.clone_url,
})
}
fn clone_repository(repo: &RepoItem, output: &Path, args: &Cli) -> Result<CloneStatus> {
match prepare_clone(repo, output, args)? {
CloneAction::Skipped => Ok(CloneStatus::Skipped),
CloneAction::Running(mut child) => {
let status = child.wait()?;
if status.success() {
return Ok(CloneStatus::Cloned);
}
let code = status
.code()
.map_or_else(|| "signal/unknown".to_string(), |code| code.to_string());
Err(ThesaError::CloneFailed {
repo: repo.full_name.clone(),
message: format!("exit code {code}"),
})
}
}
}
fn clone_repository_with_progress(
repo: &RepoItem,
output: &Path,
args: &Cli,
stdout: &mut io::Stdout,
spinner: &Loader,
position: usize,
total: usize,
) -> Result<CloneStatus> {
let mut child = match prepare_clone(repo, output, args)? {
CloneAction::Skipped => {
write!(stdout, "\r\x1b[2K")?;
println!("[skip] {} ({}/{}) exists", repo.full_name, position, total);
return Ok(CloneStatus::Skipped);
}
CloneAction::Running(child) => child,
};
let mut tick = 0usize;
loop {
if let Some(status) = child.try_wait()? {
if status.success() {
write!(stdout, "\r\x1b[2K")?;
println!("[done] {} ({}/{})", repo.full_name, position, total);
return Ok(CloneStatus::Cloned);
}
let code = status
.code()
.map_or_else(|| "signal/unknown".to_string(), |code| code.to_string());
return Err(ThesaError::CloneFailed {
repo: repo.full_name.clone(),
message: format!("exit code {code}"),
});
}
let progress = LoaderProgress::from_counts(position as u64, total as u64);
let frame = spinner.line(tick, progress);
let status = format!("[{}/{}] cloning {}", position, total, repo.full_name);
write!(stdout, "\r\x1b[2K{} {}", frame, status)?;
stdout.flush()?;
tick = tick.saturating_add(1);
thread::sleep(Duration::from_millis(120));
}
}
fn prepare_clone(repo: &RepoItem, output: &Path, args: &Cli) -> Result<CloneAction> {
let destination = output.join(&repo.name);
if destination.exists() {
if args.skip_existing {
return Ok(CloneAction::Skipped);
}
if destination.is_dir() {
remove_dir_all(&destination)?;
} else {
remove_file(&destination)?;
}
}
let child = spawn_git_clone(repo, output, args)?;
Ok(CloneAction::Running(child))
}
fn spawn_git_clone(repo: &RepoItem, output: &Path, args: &Cli) -> Result<Child> {
let mut command_args = vec!["clone".to_string()];
if let Some(depth) = args.depth {
command_args.push("--depth".to_string());
command_args.push(depth.to_string());
}
command_args.push(repo.clone_url.clone());
command_args.push(repo.name.clone());
match Command::new("git")
.current_dir(output)
.args(&command_args)
.stdout(Stdio::null())
.stderr(Stdio::null())
.spawn()
{
Ok(child) => Ok(child),
Err(error) if error.kind() == io::ErrorKind::NotFound => Err(ThesaError::MissingGit),
Err(error) => Err(error.into()),
}
}
fn paint(text: impl AsRef<str>, color: AislingColor) -> String {
let color = fg(color);
format!("{color}{}\x1b[0m", text.as_ref())
}
fn paint_bold(text: impl AsRef<str>, color: AislingColor) -> String {
let color = fg(color);
format!("\x1b[1m{color}{}\x1b[0m", text.as_ref())
}
fn fg(color: AislingColor) -> String {
format!("\x1b[38;2;{};{};{}m", color.r, color.g, color.b)
}
fn thesa_gradient() -> Gradient {
Gradient::new(
vec![
AislingColor::rgb(0, 255, 210),
AislingColor::rgb(142, 92, 255),
AislingColor::rgb(255, 80, 210),
AislingColor::rgb(255, 196, 92),
],
36,
)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parse_model_target_for_provider_hf() {
let hf_target = parse_model_target_for_provider("meta-llama/Llama-2-7b", ModelProvider::Hf)
.expect("valid hf target");
match hf_target {
ModelTarget::Model(model) => assert_eq!(model, "meta-llama/Llama-2-7b"),
ModelTarget::Org(_) => panic!("expected model target"),
ModelTarget::Top | ModelTarget::Latest => panic!("unexpected preset target"),
}
let hf_preset = parse_model_target_for_provider("top", ModelProvider::Hf)
.expect("valid hf preset target");
assert!(matches!(hf_preset, ModelTarget::Top));
let hf_latest = parse_model_target_for_provider("latest", ModelProvider::Hf)
.expect("valid hf latest target");
assert!(matches!(hf_latest, ModelTarget::Latest));
let hf_url = parse_model_target_for_provider(
"https://huggingface.co/openai-community/gpt2",
ModelProvider::Hf,
)
.expect("valid hf url target");
match hf_url {
ModelTarget::Model(model) => assert_eq!(model, "openai-community/gpt2"),
ModelTarget::Org(_) => panic!("expected model target from URL"),
ModelTarget::Top | ModelTarget::Latest => panic!("unexpected preset target"),
}
}
#[test]
fn parse_model_target_for_provider_civitai() {
let civitai_id = parse_model_target_for_provider("827184", ModelProvider::Civitai)
.expect("valid numeric id");
match civitai_id {
ModelTarget::Model(model) => assert_eq!(model, "827184"),
ModelTarget::Org(_) => panic!("expected numeric model target"),
ModelTarget::Top | ModelTarget::Latest => panic!("unexpected preset target"),
}
let civitai_preset = parse_model_target_for_provider("newest", ModelProvider::Civitai)
.expect("valid civitai preset target");
assert!(matches!(civitai_preset, ModelTarget::Latest));
let civitai_url = parse_model_target_for_provider(
"https://civitai.com/models/827184",
ModelProvider::Civitai,
)
.expect("valid civitai url");
match civitai_url {
ModelTarget::Model(model) => assert_eq!(model, "827184"),
ModelTarget::Org(_) => panic!("expected numeric model target from URL"),
ModelTarget::Top | ModelTarget::Latest => panic!("unexpected preset target"),
}
let civitai_keyword =
parse_model_target_for_provider("realistic-vision", ModelProvider::Civitai)
.expect("valid keyword target");
match civitai_keyword {
ModelTarget::Org(org) => assert_eq!(org, "realistic-vision"),
ModelTarget::Model(_) => panic!("expected keyword treated as org"),
ModelTarget::Top | ModelTarget::Latest => panic!("unexpected preset target"),
}
}
#[test]
fn parse_model_target_for_provider_ollama() {
let ollama_keyword = parse_model_target_for_provider("llama3", ModelProvider::Ollama)
.expect("valid ollama target");
match ollama_keyword {
ModelTarget::Org(org) => assert_eq!(org, "llama3"),
ModelTarget::Model(_) => panic!("expected org-style target for keyword"),
ModelTarget::Top | ModelTarget::Latest => panic!("unexpected preset target"),
}
let ollama_preset = parse_model_target_for_provider("trending", ModelProvider::Ollama)
.expect("valid ollama preset target");
assert!(matches!(ollama_preset, ModelTarget::Top));
}
#[test]
fn parse_dataset_target_for_provider_hf_and_zenodo() {
let hf_dataset = parse_dataset_target_for_provider(
"https://huggingface.co/datasets/squad",
types::DatasetProvider::Hf,
)
.expect("valid Hugging Face dataset target");
assert!(matches!(
hf_dataset,
baseforge::DatasetTarget::Dataset(ref id) if id == "squad"
));
let hf_search =
parse_dataset_target_for_provider("search:code data", types::DatasetProvider::Hf)
.expect("valid Hugging Face search target");
assert!(matches!(
hf_search,
baseforge::DatasetTarget::Search(ref query) if query == "code data"
));
let zenodo_record = parse_dataset_target_for_provider(
"https://zenodo.org/records/12345",
types::DatasetProvider::Zenodo,
)
.expect("valid Zenodo record target");
assert!(matches!(
zenodo_record,
baseforge::DatasetTarget::Dataset(ref id) if id == "12345"
));
}
#[test]
fn format_model_size_and_truncate_to_width() {
assert_eq!(format_model_size(None), "n/a");
assert_eq!(format_model_size(Some(512)), "512 B");
assert_eq!(format_model_size(Some(2048)), "2.0KB");
assert_eq!(format_model_size(Some(4_000_000)), "3.8MB");
assert_eq!(truncate_to_width("hello", 0), "");
assert_eq!(truncate_to_width("hello", 10), "hello");
assert_eq!(truncate_to_width("hello", 3), "hel");
assert_eq!(truncate_to_width("hello", 4), "h...");
assert_eq!(truncate_to_width("hello", 5), "hello");
}
#[test]
fn parse_site_url_accepts_http_and_https_only() {
assert!(parse_site_url("https://example.com").is_ok());
assert!(parse_site_url("http://127.0.0.1:8765/").is_ok());
assert!(parse_site_url("file:///tmp/site.html").is_err());
assert!(parse_site_url("not a url").is_err());
}
#[test]
fn parse_site_headers_accepts_name_value_pairs() {
let headers = parse_site_headers(&[
"Authorization: Bearer token".to_string(),
"X-Archive: thesa".to_string(),
])
.expect("valid headers");
assert_eq!(
headers,
vec![
("Authorization".to_string(), "Bearer token".to_string()),
("X-Archive".to_string(), "thesa".to_string()),
]
);
assert!(parse_site_headers(&["missing-colon".to_string()]).is_err());
}
#[test]
fn load_site_seed_urls_reads_additional_seed_file() {
let root = test_temp_dir("site_seed_file");
fs::create_dir_all(&root).expect("create fixture dir");
let input = root.join("seeds.txt");
fs::write(
&input,
"# docs\nhttps://example.com/docs\n\nhttp://127.0.0.1:8765/page\n",
)
.expect("write seed file");
let seeds = load_site_seed_urls(Some("https://example.com"), Some(&input))
.expect("load site seed urls");
assert_eq!(seeds.len(), 3);
assert_eq!(seeds[0].as_str(), "https://example.com/");
assert_eq!(seeds[1].as_str(), "https://example.com/docs");
assert_eq!(seeds[2].as_str(), "http://127.0.0.1:8765/page");
fs::remove_dir_all(root).expect("cleanup temp dir");
}
#[test]
fn archive_url_cli_dry_run_before_subcommand_parses_as_archive_command() {
let cli = Cli::try_parse_from([
"thesa",
"--dry-run",
"archive",
"url",
"https://example.com",
])
.expect("parse archive url dry-run command");
assert!(cli.dry_run);
let Some(Commands::Archive { dry_run, source }) = cli.command else {
panic!("expected archive command");
};
assert!(!dry_run);
let ArchiveCommands::Url(args) = source else {
panic!("expected url archive source");
};
assert_eq!(args.url.as_deref(), Some("https://example.com"));
assert!(!args.dry_run);
}
#[test]
fn archive_url_cli_dry_run_after_subcommand_parses_as_archive_command() {
let cli = Cli::try_parse_from([
"thesa",
"archive",
"url",
"https://example.com",
"--dry-run",
])
.expect("parse archive url command-level dry-run");
assert!(!cli.dry_run);
let Some(Commands::Archive { dry_run, source }) = cli.command else {
panic!("expected archive command");
};
assert!(!dry_run);
let ArchiveCommands::Url(args) = source else {
panic!("expected url archive source");
};
assert_eq!(args.url.as_deref(), Some("https://example.com"));
assert!(args.dry_run);
}
#[test]
fn archive_url_cli_dry_run_after_archive_parses_as_archive_command() {
let cli = Cli::try_parse_from([
"thesa",
"archive",
"--dry-run",
"url",
"https://example.com",
])
.expect("parse archive-level dry-run");
assert!(!cli.dry_run);
let Some(Commands::Archive { dry_run, source }) = cli.command else {
panic!("expected archive command");
};
assert!(dry_run);
let ArchiveCommands::Url(args) = source else {
panic!("expected url archive source");
};
assert_eq!(args.url.as_deref(), Some("https://example.com"));
assert!(!args.dry_run);
}
#[test]
fn archive_url_cli_parses_siteforge_extended_options() {
let cli = Cli::try_parse_from([
"thesa",
"archive",
"url",
"https://example.com",
"--input",
"seeds.txt",
"-j",
"8",
"--max-page-size-bytes",
"1024",
"--max-asset-size-bytes",
"2048",
"--max-total-archive-size-bytes",
"4096",
"--browser-command",
"chromium --headless {url}",
"--browser-profile",
"profile",
"--render-wait-ms",
"250",
"--header",
"X-Test: true",
"--cookie",
"session=test",
"--user-agent",
"thesa-test",
"--timeout-secs",
"7",
"--retry-count",
"3",
"--ocr",
])
.expect("parse extended siteforge options");
let Some(Commands::Archive { source, .. }) = cli.command else {
panic!("expected archive command");
};
let ArchiveCommands::Url(args) = source else {
panic!("expected url archive source");
};
assert_eq!(args.url.as_deref(), Some("https://example.com"));
assert_eq!(args.input, Some(PathBuf::from("seeds.txt")));
assert_eq!(args.concurrency, Some(8));
assert_eq!(args.max_page_size_bytes, Some(1024));
assert_eq!(args.max_asset_size_bytes, Some(2048));
assert_eq!(args.max_total_archive_size_bytes, Some(4096));
assert_eq!(
args.browser_command.as_deref(),
Some("chromium --headless {url}")
);
assert_eq!(args.browser_profile, Some(PathBuf::from("profile")));
assert_eq!(args.render_wait_ms, Some(250));
assert_eq!(args.headers, vec!["X-Test: true".to_string()]);
assert_eq!(args.cookie.as_deref(), Some("session=test"));
assert_eq!(args.user_agent.as_deref(), Some("thesa-test"));
assert_eq!(args.timeout_secs, Some(7));
assert_eq!(args.retry_count, Some(3));
assert!(args.ocr_enabled);
}
#[test]
fn archive_url_cli_allows_input_without_url() {
let cli = Cli::try_parse_from([
"thesa",
"archive",
"url",
"--input",
"seeds.txt",
"--dry-run",
])
.expect("parse input-only siteforge command");
let Some(Commands::Archive { source, .. }) = cli.command else {
panic!("expected archive command");
};
let ArchiveCommands::Url(args) = source else {
panic!("expected url archive source");
};
assert_eq!(args.url, None);
assert_eq!(args.input, Some(PathBuf::from("seeds.txt")));
assert!(args.dry_run);
}
#[test]
fn archive_dataset_cli_parses_baseforge_options() {
let cli = Cli::try_parse_from([
"thesa",
"archive",
"dataset",
"dataset:squad",
"--provider",
"hf",
"--archive-root",
"./datasets",
"--concurrency",
"8",
"--skip-existing",
"--filter",
"squad",
"--hf-token",
"token",
"--dry-run",
])
.expect("parse baseforge dataset command");
let Some(Commands::Archive { source, .. }) = cli.command else {
panic!("expected archive command");
};
let ArchiveCommands::Dataset(args) = source else {
panic!("expected dataset archive source");
};
assert_eq!(args.target, "dataset:squad");
assert_eq!(args.provider, types::DatasetProvider::Hf);
assert_eq!(args.archive_root, PathBuf::from("./datasets"));
assert_eq!(args.concurrency, 8);
assert!(args.skip_existing);
assert_eq!(args.filter.as_deref(), Some("squad"));
assert_eq!(args.hf_token.as_deref(), Some("token"));
assert!(args.dry_run);
}
#[test]
fn top_level_dataset_mode_cli_parses_provider() {
let cli = Cli::try_parse_from([
"thesa",
"--mode",
"datasets",
"--dataset-provider",
"zenodo",
"top",
"--dry-run",
])
.expect("parse top-level dataset mode");
assert_eq!(cli.mode, Mode::Datasets);
assert_eq!(cli.dataset_provider, types::DatasetProvider::Zenodo);
assert_eq!(cli.target.as_deref(), Some("top"));
assert!(cli.dry_run);
}
#[test]
fn archive_cli_parses_siteforge_management_commands() {
let list = Cli::try_parse_from(["thesa", "archive", "list", "--archive-root", "./sites"])
.expect("parse siteforge list");
let Some(Commands::Archive {
source: ArchiveCommands::List(args),
..
}) = list.command
else {
panic!("expected archive list command");
};
assert_eq!(args.archive_root, PathBuf::from("./sites"));
let search = Cli::try_parse_from([
"thesa",
"archive",
"search",
"install",
"--archive-id",
"docs",
"--limit",
"5",
])
.expect("parse siteforge search");
let Some(Commands::Archive {
source: ArchiveCommands::Search(args),
..
}) = search.command
else {
panic!("expected archive search command");
};
assert_eq!(args.query, "install");
assert_eq!(args.archive_id.as_deref(), Some("docs"));
assert_eq!(args.limit, 5);
let export = Cli::try_parse_from([
"thesa",
"archive",
"export",
"docs",
"--format",
"agent-tar",
"--output",
"./exports/docs-agent",
])
.expect("parse siteforge export");
let Some(Commands::Archive {
source: ArchiveCommands::Export(args),
..
}) = export.command
else {
panic!("expected archive export command");
};
assert_eq!(args.archive_id, "docs");
assert_eq!(args.format, SiteforgeExportFormat::AgentTar);
assert_eq!(args.output, Some(PathBuf::from("./exports/docs-agent")));
let pack =
Cli::try_parse_from(["thesa", "archive", "pack", "docs", "--max-tokens", "80000"])
.expect("parse siteforge pack");
let Some(Commands::Archive {
source: ArchiveCommands::Pack(args),
..
}) = pack.command
else {
panic!("expected archive pack command");
};
assert_eq!(args.archive_id, "docs");
assert_eq!(args.max_tokens, Some(80_000));
for subcommand in ["inspect", "verify", "resume", "diagnostics"] {
let cli = Cli::try_parse_from(["thesa", "archive", subcommand, "docs"])
.expect("parse siteforge archive id command");
let Some(Commands::Archive { source, .. }) = cli.command else {
panic!("expected archive command");
};
match (subcommand, source) {
("inspect", ArchiveCommands::Inspect(args))
| ("verify", ArchiveCommands::Verify(args))
| ("resume", ArchiveCommands::Resume(args)) => assert_eq!(args.archive_id, "docs"),
("diagnostics", ArchiveCommands::Diagnostics(args)) => {
assert_eq!(args.archive_id.as_deref(), Some("docs"));
}
_ => panic!("unexpected command variant"),
}
}
}
#[test]
fn archive_refresh_cli_parses_root_filter_and_dry_run() {
let cli = Cli::try_parse_from([
"thesa",
"archive",
"refresh",
"--archive-root",
"./archives/repos",
"--filter",
"octo",
"--dry-run",
"--concurrency",
"2",
])
.expect("refresh command should parse");
match cli.command.expect("archive command") {
Commands::Archive {
source: ArchiveCommands::Refresh(args),
..
} => {
assert_eq!(args.archive_root, PathBuf::from("./archives/repos"));
assert_eq!(args.owner, None);
assert_eq!(args.owner_filter, None);
assert_eq!(args.filter.as_deref(), Some("octo"));
assert!(args.dry_run);
assert_eq!(args.concurrency, 2);
}
_ => panic!("expected archive refresh command"),
}
}
#[test]
fn archive_refresh_cli_parses_owner_scope() {
let cli = Cli::try_parse_from([
"thesa",
"archive",
"refresh",
"Tknott95",
"--filter",
"agent",
"--dry-run",
])
.expect("refresh owner command should parse");
match cli.command.expect("archive command") {
Commands::Archive {
source: ArchiveCommands::Refresh(args),
..
} => {
assert_eq!(args.owner.as_deref(), Some("Tknott95"));
assert_eq!(args.owner_filter, None);
assert_eq!(args.filter.as_deref(), Some("agent"));
assert!(args.dry_run);
let owner = git_refresh_owner(&args).expect("valid owner");
assert_eq!(owner.as_deref(), Some("Tknott95"));
}
_ => panic!("expected archive refresh command"),
}
let cli = Cli::try_parse_from(["thesa", "archive", "pull", "--owner", "octocat"])
.expect("refresh owner flag should parse");
match cli.command.expect("archive command") {
Commands::Archive {
source: ArchiveCommands::Refresh(args),
..
} => {
assert_eq!(args.owner, None);
assert_eq!(args.owner_filter.as_deref(), Some("octocat"));
let owner = git_refresh_owner(&args).expect("valid owner");
assert_eq!(owner.as_deref(), Some("octocat"));
}
_ => panic!("expected archive refresh command"),
}
}
#[test]
fn archive_refresh_rejects_conflicting_owner_scope() {
let cli = Cli::try_parse_from([
"thesa",
"archive",
"refresh",
"octocat",
"--owner",
"rust-lang",
])
.expect("refresh owner command should parse before validation");
let Commands::Archive {
source: ArchiveCommands::Refresh(args),
..
} = cli.command.expect("archive command")
else {
panic!("expected archive refresh command");
};
assert!(git_refresh_owner(&args).is_err());
}
#[test]
fn archive_refresh_aliases_parse() {
for alias in ["update", "pull"] {
let cli = Cli::try_parse_from(["thesa", "archive", alias, "--dry-run"])
.expect("refresh alias should parse");
assert!(matches!(
cli.command,
Some(Commands::Archive {
source: ArchiveCommands::Refresh(_),
..
})
));
}
}
#[test]
fn parses_github_remote_slugs() {
assert_eq!(
repoforge::parse_github_remote_slug("https://github.com/octocat/Hello-World.git"),
Some("octocat/Hello-World".to_string())
);
assert_eq!(
repoforge::parse_github_remote_slug("git@github.com:Tknott95/Thesa.git"),
Some("Tknott95/Thesa".to_string())
);
assert_eq!(
repoforge::parse_github_remote_slug("ssh://git@github.com/owner/repo"),
Some("owner/repo".to_string())
);
assert_eq!(
repoforge::parse_github_remote_slug("https://example.com/owner/repo"),
None
);
}
#[test]
fn discovers_git_archives_from_archive_tree() {
let root = test_temp_dir("discover_git_archives");
let repo = root.join("octocat").join("Hello-World");
let skipped = root.join("sites").join("docs").join(THESA_METADATA_DIR);
fs::create_dir_all(repo.join(".git")).expect("create fake git worktree marker");
fs::create_dir_all(skipped).expect("create skipped metadata dir");
let archives = repoforge::discover_git_archives(&root, None).expect("discover archives");
assert_eq!(archives.len(), 1);
assert_eq!(archives[0].path, repo);
assert_eq!(archives[0].slug, "octocat/Hello-World");
let filtered =
repoforge::discover_git_archives(&root, Some("hello")).expect("filter archives");
assert_eq!(filtered.len(), 1);
let none =
repoforge::discover_git_archives(&root, Some("nomatch")).expect("filter archives");
assert!(none.is_empty());
fs::remove_dir_all(root).expect("cleanup temp dir");
}
#[test]
fn site_tui_options_cycle_safe_presets() {
let mut options = SiteTuiOptions::default();
assert!(options.full_site);
assert_eq!(options.max_depth, 2);
assert_eq!(options.max_pages, 100);
assert_eq!(options.archive_profile, ArchiveOutputProfile::Full);
assert!(!options.render_js);
assert!(!options.render_js_auto);
assert_eq!(next_site_depth(1), 2);
assert_eq!(next_site_depth(2), 4);
assert_eq!(next_site_depth(4), 8);
assert_eq!(next_site_depth(8), 1);
assert_eq!(next_site_pages(25), 100);
assert_eq!(next_site_pages(100), 500);
assert_eq!(next_site_pages(500), 1000);
assert_eq!(next_site_pages(1000), 25);
apply_site_option_row(&mut options, 0);
assert!(!options.full_site);
apply_site_option_row(&mut options, 1);
assert_eq!(options.max_depth, 4);
apply_site_option_row(&mut options, 2);
assert_eq!(options.max_pages, 500);
apply_site_option_row(&mut options, 3);
assert!(!options.render_js);
assert!(options.render_js_auto);
apply_site_option_row(&mut options, 3);
assert!(options.render_js);
assert!(!options.render_js_auto);
apply_site_option_row(&mut options, 4);
assert_eq!(options.archive_profile, ArchiveOutputProfile::Agent);
apply_site_option_row(&mut options, 4);
assert_eq!(options.archive_profile, ArchiveOutputProfile::Lean);
}
#[test]
fn parses_site_archive_profile_and_auto_render_flags() {
let cli = Cli::try_parse_from([
"thesa",
"archive",
"site",
"https://example.com",
"--archive-profile",
"lean",
"--render-js-auto",
])
.expect("site archive args should parse");
match cli.command.expect("archive command") {
Commands::Archive {
source: ArchiveCommands::Site(args),
..
} => {
assert_eq!(args.archive_profile, Some(SiteArchiveProfile::Lean));
assert!(args.render_js_auto);
assert!(!args.render_js);
}
_ => panic!("expected site archive command"),
}
}
#[test]
fn visible_list_item_hit_tracks_render_offset() {
let inner = Rect::new(4, 3, 20, 4);
assert_eq!(visible_list_item_at(inner, 4, 3, 10, 0), Some(0));
assert_eq!(visible_list_item_at(inner, 4, 6, 10, 0), Some(3));
assert_eq!(visible_list_item_at(inner, 4, 3, 10, 6), Some(3));
assert_eq!(visible_list_item_at(inner, 4, 6, 10, 6), Some(6));
assert_eq!(visible_list_item_at(inner, 4, 7, 10, 6), None);
}
#[test]
fn list_render_offset_matches_scrin_list_state_behavior() {
assert_eq!(list_render_offset(0, 5, 4), 0);
assert_eq!(list_render_offset(10, 0, 4), 0);
assert_eq!(list_render_offset(10, 3, 4), 0);
assert_eq!(list_render_offset(10, 4, 4), 1);
assert_eq!(list_render_offset(10, 9, 4), 6);
}
#[test]
fn archive_manifest_hashes_files_and_skips_sidecar_metadata() {
let root = test_temp_dir("manifest_hashes_files");
fs::create_dir_all(root.join("src")).expect("create nested fixture dir");
fs::write(root.join("README.md"), b"hello thesa\n").expect("write readme");
fs::write(root.join("src/lib.rs"), b"pub fn archived() {}\n").expect("write source");
fs::create_dir_all(root.join(THESA_METADATA_DIR)).expect("create metadata dir");
fs::write(root.join(THESA_METADATA_DIR).join("old"), b"ignore me").expect("write sidecar");
let manifest = build_archive_manifest(&root, "github", "owner/repo", true)
.expect("manifest should build");
assert_eq!(manifest.format_version, THESA_FORMAT_VERSION);
assert_eq!(manifest.tool.name, "thesa");
assert_eq!(manifest.source.platform, "github");
assert_eq!(manifest.source.target, "owner/repo");
assert!(manifest.archive.complete);
assert_eq!(manifest.archive.file_count, 2);
assert_eq!(manifest.files[0].path, "README.md");
assert_eq!(manifest.files[1].path, "src/lib.rs");
assert!(manifest.files[0].modified_unix_nanos.is_some());
assert!(manifest.files[1].modified_unix_nanos.is_some());
assert_eq!(
manifest.files[0].blake3,
blake3::hash(b"hello thesa\n").to_hex().to_string()
);
assert!(
manifest
.files
.iter()
.all(|file| !file.path.starts_with(THESA_METADATA_DIR))
);
fs::remove_dir_all(root).expect("cleanup temp dir");
}
#[test]
fn write_archive_manifest_creates_manifest_and_checksum_sidecars() {
let root = test_temp_dir("manifest_writes_sidecars");
fs::create_dir_all(&root).expect("create fixture dir");
fs::write(root.join("artifact.txt"), b"artifact bytes").expect("write artifact");
let manifest = write_archive_manifest(&root, "model:hf", "openai-community/gpt2", false)
.expect("write manifest");
assert!(!manifest.archive.complete);
let manifest_path = root.join(THESA_METADATA_DIR).join("manifest.json");
let checksums_path = root.join(THESA_METADATA_DIR).join("checksums.blake3");
assert!(manifest_path.exists());
assert!(checksums_path.exists());
let manifest_text = fs::read_to_string(manifest_path).expect("read manifest sidecar");
assert!(!manifest_text.contains('\n'));
let parsed: ArchiveManifest =
serde_json::from_str(&manifest_text).expect("manifest sidecar should parse");
assert_eq!(parsed.source.platform, "model:hf");
assert_eq!(parsed.archive.file_count, 1);
assert!(parsed.files[0].modified_unix_nanos.is_some());
let checksums = fs::read_to_string(checksums_path).expect("read checksums sidecar");
assert!(checksums.contains("artifact.txt"));
assert!(checksums.contains(&blake3::hash(b"artifact bytes").to_hex().to_string()));
fs::remove_dir_all(root).expect("cleanup temp dir");
}
#[test]
fn verify_archive_accepts_valid_manifest_and_checksums() {
let root = test_temp_dir("verify_valid_archive");
fs::create_dir_all(&root).expect("create fixture dir");
fs::write(root.join("artifact.txt"), b"artifact bytes").expect("write artifact");
write_archive_manifest(&root, "github", "owner/repo", true).expect("write manifest");
let report = verify_archive(&root);
assert!(
report.ok,
"unexpected verification errors: {:?}",
report.errors
);
assert_eq!(report.files_verified, 1);
assert_eq!(report.bytes_verified, "artifact bytes".len() as u64);
assert!(report.manifest_valid);
assert!(report.checksums_valid);
assert!(report.path_safety_valid);
fs::remove_dir_all(root).expect("cleanup temp dir");
}
#[test]
fn verify_archive_rejects_corrupted_file() {
let root = test_temp_dir("verify_rejects_corruption");
fs::create_dir_all(&root).expect("create fixture dir");
fs::write(root.join("artifact.txt"), b"artifact bytes").expect("write artifact");
write_archive_manifest(&root, "github", "owner/repo", true).expect("write manifest");
fs::write(root.join("artifact.txt"), b"changed bytes").expect("corrupt artifact");
let report = verify_archive(&root);
assert!(!report.ok);
assert!(report
.errors
.iter()
.any(|error| error.contains("size mismatch") || error.contains("checksum mismatch")));
fs::remove_dir_all(root).expect("cleanup temp dir");
}
#[test]
fn verify_archive_rejects_unsafe_manifest_path_before_file_access() {
let root = test_temp_dir("verify_rejects_unsafe_path");
fs::create_dir_all(&root).expect("create fixture dir");
fs::write(root.join("artifact.txt"), b"artifact bytes").expect("write artifact");
let mut manifest =
write_archive_manifest(&root, "github", "owner/repo", true).expect("write manifest");
manifest.files[0].path = "../artifact.txt".to_string();
let manifest_json = serde_json::to_string_pretty(&manifest).expect("serialize manifest");
fs::write(
root.join(THESA_METADATA_DIR).join("manifest.json"),
manifest_json,
)
.expect("overwrite manifest with unsafe path");
let report = verify_archive(&root);
assert!(!report.ok);
assert!(!report.path_safety_valid);
assert!(
report
.errors
.iter()
.any(|error| error.contains("unsafe manifest path"))
);
fs::remove_dir_all(root).expect("cleanup temp dir");
}
fn test_temp_dir(name: &str) -> PathBuf {
let nonce = SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("system clock")
.as_nanos();
std::env::temp_dir().join(format!("thesa-{name}-{nonce}"))
}
}