use std::collections::BTreeSet;
use std::io::{self, IsTerminal, Write};
use std::path::{Path, PathBuf};
use std::sync::mpsc;
use std::thread;
use std::time::Duration;
use aisling::{Loader, LoaderConfig, LoaderKind, LoaderProgress};
use crossterm::{
cursor,
event::{self, Event, KeyCode, KeyEventKind},
execute,
terminal::{Clear, ClearType},
};
use photoforge::{
ArchiveOptions, ArchiveProgress, ArchiveProgressPhase, ArchiveSummary, ArchivedPhoto, COMMONS_REUSE_URL,
DiscoveryResult, PhotoForge, PhotoSummary, PhotoTarget,
};
use crate::cli::{Cli, PhotoArchiveArgs};
use crate::constants::THESA_METADATA_DIR;
use crate::error::{Result, ThesaError};
use crate::manifest::{prepare_output_dir, write_archive_manifest};
use crate::selection::parse_indices;
use crate::terminal::{
RankedPickerEntry, TerminalSession, draw_full_frame, pick_ranked_results_tui, read_input_line,
};
use crate::types::Mode;
pub(crate) fn parse_photo_target(input: &str) -> Result<PhotoTarget> {
photoforge::parse_photo_target(input)
.map_err(|error| ThesaError::InvalidTarget(error.to_string()))
}
pub(crate) fn run_photos_mode(
args: &mut Cli,
terminal_session: &mut Option<TerminalSession>,
) -> Result<()> {
let target_text = args.target.as_deref().unwrap_or_default().to_string();
let target = parse_photo_target(&target_text)?;
if args.output == Path::new("./archives") {
args.output = PathBuf::from("./archives/photos");
}
let forge = photoforge_client(args.photo_limit)?;
let discovery = if let Some(session) = terminal_session.as_mut() {
discover_photos_tui(
&mut session.terminal,
forge.clone(),
target.clone(),
&target_text,
)?
} else {
forge
.discover_detailed(&target)
.map_err(|error| photoforge_error("discover", error))?
};
let mut selected = filtered_photos(discovery.photos.clone(), args.filter.as_deref());
let used_tui = terminal_session.is_some();
let mut selection_confirmed = false;
if target.is_collection() && selected.len() > 1 {
if let Some(session) = terminal_session.as_mut() {
selected = pick_photos_tui(&mut session.terminal, &selected)?;
selection_confirmed = true;
} else if io::stdout().is_terminal() {
selected = pick_photos_interactively(&selected)?;
selection_confirmed = true;
}
}
*terminal_session = None;
run_photoforge_archive(
&forge,
&target_text,
&target,
&args.output,
args.skip_existing,
args.photo_max_download_bytes,
args.dry_run,
!used_tui && io::stdout().is_terminal(),
&discovery,
selected,
selection_confirmed,
)
}
pub(crate) fn run_photoforge_archive_command(args: &PhotoArchiveArgs, dry_run: bool) -> Result<()> {
let target = parse_photo_target(&args.target)?;
let forge = photoforge_client(args.limit)?;
let discovery = forge
.discover_detailed(&target)
.map_err(|error| photoforge_error("discover", error))?;
let mut selected = filtered_photos(discovery.photos.clone(), args.filter.as_deref());
let selection_confirmed = false;
if target.is_collection() && selected.len() > 1 && io::stdout().is_terminal() {
selected = pick_photos_interactively(&selected)?;
}
run_photoforge_archive(
&forge,
&args.target,
&target,
&args.archive_root,
args.skip_existing,
args.max_download_bytes,
dry_run || args.dry_run,
io::stdout().is_terminal(),
&discovery,
selected,
selection_confirmed,
)
}
#[allow(clippy::too_many_arguments)]
fn run_photoforge_archive(
forge: &PhotoForge,
target_text: &str,
target: &PhotoTarget,
output: &Path,
skip_existing: bool,
max_download_bytes: u64,
dry_run: bool,
interactive: bool,
discovery: &DiscoveryResult,
mut selected: Vec<PhotoSummary>,
selection_confirmed: bool,
) -> Result<()> {
if selected.is_empty() {
if selection_confirmed {
println!("No photos selected; nothing to archive.");
} else {
println!("No eligible photos matched your request.");
}
print_candidate_evidence(discovery);
return Ok(());
}
if dry_run {
for photo in &selected {
if exceeds_download_limit(photo, max_download_bytes) {
return Err(ThesaError::Message(
"dry-run aborted because one selected photo exceeds byte limit".to_string(),
));
}
}
print_photo_plan(
target_text,
output,
skip_existing,
max_download_bytes,
discovery,
&selected,
);
return Ok(());
}
if interactive && target.is_collection() && selected.len() > 1 {
selected = pick_photos_interactively(&selected)?;
}
let output = prepare_output_dir(output)?;
println!(
"Archiving {} photo(s) with Photoforge to {}",
selected.len(),
output.display()
);
println!("Source: Wikimedia Commons");
println!("Commons reuse guidance: {COMMONS_REUSE_URL}");
println!("Output: {}", output.display());
print_skip_notice(skip_existing);
println!("Maximum photo bytes: {max_download_bytes}");
println!("Selected photos:");
for photo in &selected {
print_photo_line(photo);
}
io::stdout().flush()?;
let options = ArchiveOptions {
output: output.clone(),
skip_existing,
max_download_bytes,
};
let mut renderer = io::stderr().is_terminal().then(PhotoProgressRenderer::new);
let mut last_plain_completed = 0usize;
let mut last_plain_bytes = 0u64;
let summary = forge
.archive_with_progress(&selected, &options, |snapshot| {
if let Some(renderer) = renderer.as_mut() {
renderer.render(&snapshot);
} else {
let completed_changed = snapshot.completed_photos != last_plain_completed;
let bytes_advanced =
snapshot.bytes_written < last_plain_bytes
|| snapshot.bytes_written.saturating_sub(last_plain_bytes) >= 8 * 1024 * 1024;
if completed_changed || bytes_advanced {
print_plain_progress(&snapshot);
last_plain_completed = snapshot.completed_photos;
last_plain_bytes = snapshot.bytes_written;
}
}
})
.map_err(|error| photoforge_error("archive", error))?;
if let Some(renderer) = renderer.as_mut() {
renderer.finish();
}
let selected_ids = selected.iter().map(|photo| photo.id).collect::<Vec<_>>();
let verified_records = validate_and_verify_summary(&summary, &selected_ids)?;
let mut manifest_count = 0usize;
let mut manifest_file_count = 0usize;
let mut manifest_total_bytes = 0u64;
for record in &verified_records {
let manifest = write_archive_manifest(
&record.output,
&format!("photo:{}", record.manifest.photo.provider.slug()),
&record.manifest.photo.id.to_string(),
true,
)?;
manifest_count += 1;
manifest_file_count += manifest.archive.file_count;
manifest_total_bytes = manifest_total_bytes.saturating_add(manifest.archive.total_bytes);
println!(
"Photo archive: {} -> {}",
record.manifest.photo.id,
record.output.display()
);
}
println!(
"\nCompleted: {} photo(s) archived, {} verified-skipped, {} failed",
summary.archived,
summary.skipped,
summary.failed
);
println!(
"Total bytes written: {}",
format_bytes(summary.bytes_written)
);
println!(
"Manifests: {manifest_count} photo-local {THESA_METADATA_DIR}/manifest.json sidecars ({manifest_file_count} files, {manifest_total_bytes} bytes)"
);
if !summary.failures.is_empty() {
eprintln!("Failures:");
for failure in summary.failures {
eprintln!(" - {}: {}", failure.photo_id, failure.message);
}
return Err(ThesaError::Message(
"some photo downloads failed; successful and verified archives were retained".to_string(),
));
}
Ok(())
}
fn photoforge_client(discovery_limit: usize) -> Result<PhotoForge> {
PhotoForge::with_user_agent(concat!(
"thesa/",
env!("CARGO_PKG_VERSION"),
" (PhotoForge Wikimedia Commons archiver; https://github.com/Tknott95/PhotoForge)"
))
.map(|forge| forge.with_discovery_limit(discovery_limit))
.map_err(|error| photoforge_error("init", error))
}
fn discover_photos_tui(
terminal: &mut scrin::Terminal,
forge: PhotoForge,
target: PhotoTarget,
target_text: &str,
) -> Result<DiscoveryResult> {
let loading_label = format!("discovering Commons photos for {target_text}");
let (tx, rx) = mpsc::channel();
thread::spawn(move || {
let result = forge.discover_detailed(&target);
let _ = tx.send(result);
});
let mut tick = 0usize;
loop {
if let Ok(result) = rx.try_recv() {
return result.map_err(|error| photoforge_error("discover", error));
}
draw_full_frame(terminal, |frame| {
crate::draw_loading_tui(frame, &loading_label, tick, Mode::Photos, None)
})?;
tick = tick.wrapping_add(1);
if event::poll(Duration::from_millis(80))? {
let Event::Key(key) = event::read()? else {
continue;
};
if key.kind == KeyEventKind::Press && key.code == KeyCode::Esc {
return Err(ThesaError::Message("aborted by user".to_string()));
}
}
}
}
fn pick_photos_tui(
terminal: &mut scrin::Terminal,
photos: &[PhotoSummary],
) -> Result<Vec<PhotoSummary>> {
let entries = photos
.iter()
.map(|photo| RankedPickerEntry {
provider: photo.provider.slug().to_string(),
title: photo.title.clone(),
license: format!(
"{} | {} | {}",
photo.rights.license,
photo.rights.attribution_required,
photo.rights.usage_terms
),
metrics: format!(
"{} | {}x{} | {}",
format_bytes(photo.file.size),
photo.file.width,
photo.file.height,
photo.source_url
),
details: vec![
("id".to_string(), photo.id.to_string()),
(
"creator".to_string(),
if photo.rights.creator.is_empty() {
"unknown".to_string()
} else {
photo.rights.creator.clone()
},
),
("object".to_string(), photo.object_name.clone()),
("categories".to_string(), photo.categories.join(", ")),
("source".to_string(), photo.source_url.clone()),
],
})
.collect::<Vec<_>>();
let indexes = pick_ranked_results_tui(terminal, "photo picker", "photos", &entries)?;
Ok(indexes
.into_iter()
.map(|index| photos[index].clone())
.collect())
}
fn pick_photos_interactively(photos: &[PhotoSummary]) -> Result<Vec<PhotoSummary>> {
println!("Discovered {} ranked photos:", photos.len());
for (index, photo) in photos.iter().enumerate() {
print!(" {:>3})", index + 1);
print_photo_line(photo);
}
println!("Select photo indexes such as 1,3,5-8, or press Enter for all.");
print!("Selection: ");
io::stdout().flush()?;
let input = read_input_line(&mut io::stdin().lock())?;
let input = input.trim();
if input.is_empty() || input.eq_ignore_ascii_case("all") {
return Ok(photos.to_vec());
}
let indexes = parse_indices(input, photos.len()).map_err(ThesaError::Message)?;
Ok(indexes
.into_iter()
.map(|index| photos[index].clone())
.collect())
}
fn filtered_photos(photos: Vec<PhotoSummary>, filter: Option<&str>) -> Vec<PhotoSummary> {
photos
.into_iter()
.filter(|photo| photo_matches_filter(photo, filter))
.collect()
}
fn photo_matches_filter(photo: &PhotoSummary, filter: Option<&str>) -> bool {
let Some(filter) = filter.map(str::trim).filter(|value| !value.is_empty()) else {
return true;
};
let filter = filter.to_ascii_lowercase();
photo.id.to_string().contains(&filter)
|| photo.title.to_ascii_lowercase().contains(&filter)
|| photo.object_name.to_ascii_lowercase().contains(&filter)
|| photo.description.to_ascii_lowercase().contains(&filter)
|| photo.source_url.to_ascii_lowercase().contains(&filter)
|| photo
.categories
.iter()
.any(|category| category.to_ascii_lowercase().contains(&filter))
|| photo.rights.license.to_ascii_lowercase().contains(&filter)
|| photo
.rights
.raw_license_label
.to_ascii_lowercase()
.contains(&filter)
}
fn print_photo_plan(
target_text: &str,
output: &Path,
skip_existing: bool,
max_download_bytes: u64,
discovery: &DiscoveryResult,
photos: &[PhotoSummary],
) {
println!(
"Dry run: {} photo(s) selected from bounded PhotoForge discovery",
photos.len()
);
println!("Provider: Wikimedia Commons");
println!("Target: {target_text}");
println!("Output: {}", output.display());
print_skip_notice(skip_existing);
println!("Maximum photo bytes: {max_download_bytes}");
println!("Commons reuse guidance: {COMMONS_REUSE_URL}");
print_candidate_evidence(discovery);
for photo in photos {
print_photo_line(photo);
if exceeds_download_limit(photo, max_download_bytes) {
println!(
" archive outcome: will fail because the selected photo exceeds {max_download_bytes} bytes"
);
}
}
}
fn print_candidate_evidence(discovery: &DiscoveryResult) {
println!(
"Candidates: {} provider records, {} eligible, {} ranked photos; budget {}",
discovery.candidate_count,
discovery.eligible_candidate_count,
discovery.photos.len(),
discovery.candidate_budget
);
if discovery.candidate_cap_reached {
println!(
"Warning: the bounded {}-record candidate window was reached; matching photos may exist beyond this window.",
discovery.candidate_budget
);
}
}
fn print_photo_line(photo: &PhotoSummary) {
println!(
" - [{}] {} ({}, {}x{}) | {}",
photo.provider.slug(),
photo.title,
format_bytes(photo.file.size),
photo.file.width,
photo.file.height,
photo.source_url
);
println!(
" id: {} | usage: {} | creator: {}",
photo.id,
if photo.usage_count_complete {
photo.usage_count.to_string()
} else {
format!("{} (partial)", photo.usage_count)
},
if photo.rights.creator.is_empty() {
"unknown"
} else {
photo.rights.creator.as_str()
},
);
println!(
" rights: {} (attribution required: {})",
photo.rights.license, photo.rights.attribution_required
);
}
fn print_skip_notice(skip_existing: bool) {
if skip_existing {
println!(
"Skip existing: true (manifest and checksum verification occurs during archive; candidates may be skipped)"
);
} else {
println!("Skip existing: false");
}
}
fn exceeds_download_limit(photo: &PhotoSummary, max_download_bytes: u64) -> bool {
max_download_bytes > 0 && photo.file.size > max_download_bytes
}
#[derive(Clone, Copy)]
struct PhotoRecordEvidence {
photo_id: u64,
manifest_id: u64,
complete: bool,
}
fn validate_and_verify_summary<'a>(
summary: &'a ArchiveSummary,
selected_ids: &[u64],
) -> Result<Vec<&'a ArchivedPhoto>> {
let archived = summary.photos.iter().map(photo_record_evidence).collect::<Vec<_>>();
let skipped = summary
.skipped_photos
.iter()
.map(photo_record_evidence)
.collect::<Vec<_>>();
let failed_ids = summary.failures.iter().map(|failure| failure.photo_id).collect::<Vec<_>>();
let complete_ids = complete_photo_manifest_ids(
selected_ids,
&summary.selected_ids,
&archived,
&skipped,
&failed_ids,
summary.selected,
summary.archived,
summary.skipped,
summary.failed,
)?;
let records = summary
.photos
.iter()
.chain(summary.skipped_photos.iter())
.collect::<Vec<_>>();
for record in &records {
if !complete_ids.contains(&record.photo_id) {
return Err(ThesaError::Message(
"PhotoForge returned an unexpected archive record set".to_string(),
));
}
let verified = photoforge::verify_archive(&record.output)
.map_err(|error| photoforge_error("post-archive verification", error))?;
if verified.photo.id != record.photo_id || verified != record.manifest {
return Err(ThesaError::Message(
"PhotoForge verification did not reproduce the returned manifest; no Thesa sidecars were written".to_string(),
));
}
}
Ok(records)
}
fn photo_record_evidence(record: &ArchivedPhoto) -> PhotoRecordEvidence {
PhotoRecordEvidence {
photo_id: record.photo_id,
manifest_id: record.manifest.photo.id,
complete: record.manifest.complete,
}
}
fn complete_photo_manifest_ids(
selected_ids: &[u64],
summary_selected_ids: &[u64],
archived_records: &[PhotoRecordEvidence],
skipped_records: &[PhotoRecordEvidence],
failed_ids: &[u64],
selected_count: usize,
archived_count: usize,
skipped_count: usize,
failed_count: usize,
) -> Result<BTreeSet<u64>> {
let selected = selected_ids.iter().copied().collect::<BTreeSet<_>>();
let summary_selected = summary_selected_ids.iter().copied().collect::<BTreeSet<_>>();
let archived = archived_records.iter().map(|record| record.photo_id).collect::<Vec<_>>();
let skipped = skipped_records.iter().map(|record| record.photo_id).collect::<Vec<_>>();
let skipped_set = skipped.iter().copied().collect::<BTreeSet<_>>();
let archived_set = archived.iter().copied().collect::<BTreeSet<_>>();
let failed_set = failed_ids.iter().copied().collect::<BTreeSet<_>>();
let records_valid = archived_records
.iter()
.chain(skipped_records)
.all(|record| record.photo_id == record.manifest_id && record.complete);
let failures_valid = failed_ids
.iter()
.all(|id| selected.contains(id));
let counts_match = selected.len() == selected_count
&& summary_selected.len() == selected_count
&& summary_selected == selected
&& archived_count == archived_records.len()
&& archived_set.len() == archived_records.len()
&& skipped_count == skipped_records.len()
&& skipped_set.len() == skipped_records.len()
&& failed_count == failed_ids.len()
&& failed_set.len() == failed_ids.len()
&& selected_count == archived_count + skipped_count + failed_count
&& archived_set.is_disjoint(&skipped_set)
&& archived_set.is_disjoint(&failed_set)
&& skipped_set.is_disjoint(&failed_set);
let complete = archived_set
.union(&skipped_set)
.copied()
.collect::<BTreeSet<_>>();
if !records_valid
|| !failures_valid
|| !counts_match
|| !complete.is_subset(&selected)
{
return Err(ThesaError::Message(
"PhotoForge returned an inconsistent archive summary; no Thesa sidecars were written".to_string(),
));
}
Ok(complete)
}
fn format_active_bytes(progress: &ArchiveProgress) -> String {
progress
.active_total_bytes
.map_or_else(
|| format_bytes(progress.active_bytes_written),
|total| {
format!(
"{}/{}",
format_bytes(progress.active_bytes_written),
format_bytes(total)
)
},
)
}
struct PhotoProgressRenderer {
stderr: io::Stderr,
loader: Loader,
tick: usize,
wrote: bool,
}
impl PhotoProgressRenderer {
fn new() -> Self {
let mut stderr = io::stderr();
let _ = execute!(stderr, cursor::Hide);
Self {
stderr,
loader: Loader::with_config(
LoaderKind::Tqdm,
LoaderConfig::default()
.with_width(28)
.with_label("photoforge")
.with_unit("photos")
.with_fraction(true),
),
tick: 0,
wrote: false,
}
}
fn render(&mut self, progress: &ArchiveProgress) {
let total = progress.total_photos.max(1) as u64;
let completed = progress.completed_photos.min(progress.total_photos) as u64;
let loader = self
.loader
.frame(self.tick, LoaderProgress::from_counts(completed, total))
.to_ansi_string();
let active = progress
.active_title
.as_ref()
.map_or("photo", String::as_str);
let file = progress
.active_file
.as_deref()
.unwrap_or("candidate");
let line = format!(
"{loader} | {} | {} | {} / {}",
phase_label(progress.phase),
format_active_bytes(progress),
truncate(active, 34),
truncate(file, 26),
);
let _ = execute!(
self.stderr,
cursor::MoveToColumn(0),
Clear(ClearType::CurrentLine),
);
let _ = write!(self.stderr, "{line}");
let _ = self.stderr.flush();
self.tick = self.tick.wrapping_add(1);
self.wrote = true;
}
fn finish(&mut self) {
if self.wrote {
let _ = writeln!(self.stderr);
}
let _ = execute!(self.stderr, cursor::Show);
let _ = self.stderr.flush();
}
}
impl Drop for PhotoProgressRenderer {
fn drop(&mut self) {
let _ = execute!(self.stderr, cursor::Show);
}
}
fn print_plain_progress(snapshot: &ArchiveProgress) {
if snapshot.phase == ArchiveProgressPhase::Starting {
return;
}
if let (Some(_title), Some(file)) = (snapshot.active_title.as_ref(), snapshot.active_file.as_ref()) {
println!(
"Photoforge {}/{} photos: downloading {} ({} / {})",
snapshot.completed_photos,
snapshot.total_photos,
truncate(file, 36),
snapshot.active_photo_id
.map_or_else(|| "unknown".to_string(), |id| id.to_string()),
format_active_bytes(snapshot)
);
return;
}
println!(
"Photoforge {}/{} photos (success={} skipped={} failed={} bytes={})",
snapshot.completed_photos,
snapshot.total_photos,
snapshot.archived_photos,
snapshot.skipped_photos,
snapshot.failed_photos,
format_bytes(snapshot.bytes_written)
);
}
fn phase_label(phase: ArchiveProgressPhase) -> &'static str {
match phase {
ArchiveProgressPhase::Starting => "starting",
ArchiveProgressPhase::Downloading => "downloading",
ArchiveProgressPhase::Verifying => "verifying",
ArchiveProgressPhase::Installing => "installing",
ArchiveProgressPhase::Skipped => "skipped",
ArchiveProgressPhase::Failed => "failed",
ArchiveProgressPhase::Completed => "completed",
}
}
fn truncate(value: &str, max_chars: usize) -> String {
if value.chars().count() <= max_chars {
value.to_string()
} else {
format!(
"{}...",
value
.chars()
.take(max_chars.saturating_sub(3))
.collect::<String>()
)
}
}
fn format_bytes(bytes: u64) -> String {
const UNITS: [&str; 5] = ["B", "KiB", "MiB", "GiB", "TiB"];
let mut value = bytes as f64;
let mut unit = 0usize;
while value >= 1024.0 && unit + 1 < UNITS.len() {
value /= 1024.0;
unit += 1;
}
if unit == 0 {
format!("{bytes} B")
} else {
format!("{value:.2} {}", UNITS[unit])
}
}
fn photoforge_error(context: &str, error: impl std::fmt::Display) -> ThesaError {
ThesaError::Message(format!("photoforge {context} failed: {error}"))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parses_search_oriented_photo_targets() {
assert_eq!(parse_photo_target("top").expect("top parsed"), PhotoTarget::Top);
assert_eq!(parse_photo_target("latest").expect("latest parsed"), PhotoTarget::Latest);
assert!(matches!(
parse_photo_target("search:blue flower").expect("search parsed"),
PhotoTarget::Search(_)
));
assert!(matches!(
parse_photo_target("category:Featured pictures").expect("category parsed"),
PhotoTarget::Category(_)
));
assert!(matches!(
parse_photo_target("file:Blue_flower_(common).jpg").expect("file parsed"),
PhotoTarget::File(_)
));
assert!(parse_photo_target("https://commons.wikimedia.org/wiki/File:Blue_flower.jpg").is_ok());
assert!(parse_photo_target("bad-target").is_ok());
}
#[test]
fn validates_summary_id_sets_for_consistency() {
let target = PhotoTarget::Top;
assert!(target.is_collection());
let _ = ⌖
let _ = parse_photo_target("top").expect("top parse");
let _selected = vec![
PhotoRecordEvidence {
photo_id: 1,
manifest_id: 1,
complete: true,
},
PhotoRecordEvidence {
photo_id: 2,
manifest_id: 2,
complete: true,
},
];
let archived = vec![
PhotoRecordEvidence {
photo_id: 1,
manifest_id: 1,
complete: true,
},
];
let skipped = vec![PhotoRecordEvidence {
photo_id: 2,
manifest_id: 2,
complete: true,
}];
let failures = Vec::<u64>::new();
assert!(
complete_photo_manifest_ids(
&[1, 2],
&[1, 2],
&archived,
&skipped,
&failures,
2,
1,
1,
0,
)
.is_ok()
);
}
}