use anyhow::Result;
use futures::StreamExt;
use serde::{Deserialize, Serialize};
use std::io::Write;
use std::path::{Path, PathBuf};
use tokio::io::AsyncWriteExt;
use tracing::info;
use wme_models::SnapshotIdentifier;
use crate::client::map_client_error;
use crate::commands::GlobalOpts;
use crate::output::{format_table, OutputHandler};
use crate::util::build_params;
#[derive(Debug, Serialize, Deserialize, Default)]
struct DownloadCheckpoint {
snapshot_id: String,
output_path: String,
completed_chunks: Vec<String>,
}
impl DownloadCheckpoint {
fn path_for(output_path: &Path) -> PathBuf {
let mut p = output_path.to_path_buf();
let name = p
.file_name()
.unwrap_or_default()
.to_string_lossy()
.to_string();
p.set_file_name(format!("{}.progress", name));
p
}
fn load(output_path: &Path) -> Self {
let cp_path = Self::path_for(output_path);
if let Ok(data) = std::fs::read_to_string(&cp_path) {
if let Ok(cp) = serde_json::from_str(&data) {
return cp;
}
}
Self::default()
}
fn save(&self, output_path: &Path) -> anyhow::Result<()> {
let cp_path = Self::path_for(output_path);
let data = serde_json::to_string_pretty(self)?;
std::fs::write(&cp_path, data)?;
Ok(())
}
fn delete(&self, output_path: &Path) {
let cp_path = Self::path_for(output_path);
let _ = std::fs::remove_file(&cp_path);
}
fn is_completed(&self, chunk_id: &str) -> bool {
self.completed_chunks.iter().any(|c| c == chunk_id)
}
fn mark_completed(&mut self, chunk_id: &str, output_path: &Path) -> anyhow::Result<()> {
self.completed_chunks.push(chunk_id.to_string());
self.save(output_path)
}
}
pub async fn list(opts: &GlobalOpts) -> Result<()> {
info!("Listing snapshots");
let client = opts.build_client().await?;
let params = build_params(opts)?;
let snapshots = client
.snapshot()
.list_snapshots_with_params(params.as_ref())
.await
.map_err(map_client_error)?;
let filtered_snapshots: Vec<_> = if let Some(ref p) = params {
if let Some(ref filters) = p.filters {
snapshots
.into_iter()
.filter(|snapshot| {
filters.iter().all(|filter| {
let filter_value = match &filter.value {
wme_models::FilterValue::String(s) => s.clone(),
wme_models::FilterValue::Integer(i) => i.to_string(),
wme_models::FilterValue::Float(f) => f.to_string(),
wme_models::FilterValue::Boolean(b) => b.to_string(),
};
match filter.field.as_str() {
"identifier" => snapshot.identifier == filter_value,
"version" => snapshot.version == filter_value,
"date_modified" => snapshot.date_modified.to_string() == filter_value,
"in_language" => {
if let Some(ref lang) = snapshot.in_language.identifier {
lang == &filter_value
} else {
false
}
}
_ => true, }
})
})
.collect()
} else {
snapshots
}
} else {
snapshots
};
let output = OutputHandler::new(opts.output);
if matches!(opts.output, crate::output::OutputFormat::Table) {
let headers = vec!["Identifier", "Date Modified", "Size"];
let rows: Vec<Vec<String>> = filtered_snapshots
.iter()
.map(|snapshot| {
let size_str = crate::util::format_api_size(&snapshot.size);
vec![
snapshot.identifier.to_string(),
snapshot.date_modified.to_string(),
size_str,
]
})
.collect();
output.output_raw(&format_table(&headers, &rows));
} else {
let json_value = serde_json::to_value(&filtered_snapshots)?;
output.output(&json_value)?;
}
Ok(())
}
pub async fn info(identifier: &str, opts: &GlobalOpts) -> Result<()> {
info!("Getting snapshot info: {}", identifier);
let client = opts.build_client().await?;
let snapshot_id: SnapshotIdentifier =
identifier
.parse()
.map_err(|e: wme_models::error::ModelError| {
anyhow::anyhow!("Invalid snapshot identifier '{}': {}", identifier, e)
})?;
let params = build_params(opts)?;
let snapshot = client
.snapshot()
.get_snapshot_info_with_params(&snapshot_id, params.as_ref())
.await
.map_err(map_client_error)?;
let output = OutputHandler::new(opts.output);
let json_value = serde_json::to_value(&snapshot)?;
output.output(&json_value)?;
Ok(())
}
pub async fn download(
identifier: &str,
output_file: Option<PathBuf>,
range: Option<String>,
opts: &GlobalOpts,
) -> Result<()> {
let output_path =
output_file.unwrap_or_else(|| PathBuf::from(format!("{}.tar.gz", identifier)));
info!(
"Downloading snapshot: {} to {}",
identifier,
output_path.display()
);
if range.is_some() {
println!("Downloading snapshot: {}", identifier);
println!("Output: {}", output_path.display());
let client = opts.build_client().await?;
let snapshot_id: SnapshotIdentifier =
identifier
.parse()
.map_err(|e: wme_models::error::ModelError| {
anyhow::anyhow!("Invalid snapshot identifier '{}': {}", identifier, e)
})?;
let range_ref = range.as_deref();
let mut stream = client
.snapshot()
.download_snapshot(&snapshot_id, range_ref)
.await
.map_err(map_client_error)?;
let mut file = tokio::fs::File::create(&output_path).await?;
while let Some(chunk_result) = stream.next().await {
let chunk = chunk_result.map_err(map_client_error)?;
tokio::io::AsyncWriteExt::write_all(&mut file, &chunk).await?;
}
let metadata = tokio::fs::metadata(&output_path).await?;
println!("✓ Download complete!");
println!(" File: {}", output_path.display());
println!(" Size: {}", crate::util::format_size(metadata.len()));
return Ok(());
}
let mut checkpoint = DownloadCheckpoint::load(&output_path);
let resuming = !checkpoint.completed_chunks.is_empty()
&& checkpoint.snapshot_id == identifier
&& checkpoint.output_path == output_path.to_string_lossy();
if resuming {
println!(
"Resuming download: {} ({} chunks already done)",
identifier,
checkpoint.completed_chunks.len()
);
} else {
println!("Downloading snapshot: {}", identifier);
checkpoint = DownloadCheckpoint {
snapshot_id: identifier.to_string(),
output_path: output_path.to_string_lossy().to_string(),
completed_chunks: Vec::new(),
};
}
println!("Output: {}", output_path.display());
let client = opts.build_client().await?;
let snapshot_id: SnapshotIdentifier =
identifier
.parse()
.map_err(|e: wme_models::error::ModelError| {
anyhow::anyhow!("Invalid snapshot identifier '{}': {}", identifier, e)
})?;
println!("Fetching chunk list...");
let chunks = client
.snapshot()
.list_chunks(&snapshot_id)
.await
.map_err(map_client_error)?;
println!("Found {} chunks to download", chunks.len());
let remaining: Vec<_> = chunks
.iter()
.filter(|c| !checkpoint.is_completed(&c.identifier))
.collect();
if remaining.len() < chunks.len() {
println!(
"Skipping {}/{} already-completed chunks",
chunks.len() - remaining.len(),
chunks.len()
);
}
let file = if resuming && output_path.exists() {
tokio::fs::OpenOptions::new()
.append(true)
.open(&output_path)
.await?
} else {
tokio::fs::File::create(&output_path).await?
};
let mut file = file;
let total = chunks.len();
for chunk_info in &remaining {
let done = checkpoint.completed_chunks.len();
let file_size = tokio::fs::metadata(&output_path)
.await
.map(|m| m.len())
.unwrap_or(0);
print!(
"\r Chunk {}/{} | {} written | {} ...",
done + 1,
total,
crate::util::format_size(file_size),
chunk_info.identifier,
);
let _ = std::io::stdout().flush();
let mut stream = client
.snapshot()
.download_chunk(&snapshot_id, &chunk_info.identifier, None)
.await
.map_err(map_client_error)?;
let mut chunk_bytes: Vec<u8> = Vec::new();
while let Some(piece_result) = stream.next().await {
let piece = piece_result.map_err(|e| {
println!();
map_client_error(e)
})?;
chunk_bytes.extend_from_slice(&piece);
}
tokio::io::AsyncWriteExt::write_all(&mut file, &chunk_bytes)
.await
.map_err(|e| {
println!();
anyhow::anyhow!("Failed to write chunk {}: {}", chunk_info.identifier, e)
})?;
file.flush().await?;
checkpoint.mark_completed(&chunk_info.identifier, &output_path)?;
}
println!();
let metadata = tokio::fs::metadata(&output_path).await?;
checkpoint.delete(&output_path);
println!("✓ Download complete!");
println!(" File: {}", output_path.display());
println!(" Size: {}", crate::util::format_size(metadata.len()));
println!(" Chunks: {}", total);
Ok(())
}
pub async fn chunks_list(snapshot_id: &str, opts: &GlobalOpts) -> Result<()> {
info!("Listing chunks for snapshot: {}", snapshot_id);
let client = opts.build_client().await?;
let snapshot_identifier: SnapshotIdentifier =
snapshot_id
.parse()
.map_err(|e: wme_models::error::ModelError| {
anyhow::anyhow!("Invalid snapshot identifier '{}': {}", snapshot_id, e)
})?;
let params = build_params(opts)?;
let chunks = client
.snapshot()
.list_chunks_with_params(&snapshot_identifier, params.as_ref())
.await
.map_err(map_client_error)?;
let output = OutputHandler::new(opts.output);
if matches!(opts.output, crate::output::OutputFormat::Table) {
let headers = vec!["Identifier", "Size"];
let rows: Vec<Vec<String>> = chunks
.iter()
.map(|chunk| {
let size_str = crate::util::format_api_size(&chunk.size);
vec![chunk.identifier.clone(), size_str]
})
.collect();
output.output_raw(&format_table(&headers, &rows));
} else {
let json_value = serde_json::to_value(&chunks)?;
output.output(&json_value)?;
}
Ok(())
}
pub async fn chunks_info(snapshot_id: &str, chunk_id: &str, opts: &GlobalOpts) -> Result<()> {
info!("Getting chunk info: {} in {}", chunk_id, snapshot_id);
let client = opts.build_client().await?;
let snapshot_identifier: SnapshotIdentifier =
snapshot_id
.parse()
.map_err(|e: wme_models::error::ModelError| {
anyhow::anyhow!("Invalid snapshot identifier '{}': {}", snapshot_id, e)
})?;
let params = build_params(opts)?;
let chunk = client
.snapshot()
.get_chunk_info_with_params(&snapshot_identifier, chunk_id, params.as_ref())
.await
.map_err(map_client_error)?;
let output = OutputHandler::new(opts.output);
let json_value = serde_json::to_value(&chunk)?;
output.output(&json_value)?;
Ok(())
}
pub async fn chunks_download(
snapshot_id: &str,
chunk_id: &str,
output_file: Option<PathBuf>,
range: Option<String>,
opts: &GlobalOpts,
) -> Result<()> {
let output_path = output_file.unwrap_or_else(|| PathBuf::from(format!("{}.tar.gz", chunk_id)));
info!(
"Downloading chunk: {} from {} to {}",
chunk_id,
snapshot_id,
output_path.display()
);
println!("Downloading chunk: {}", chunk_id);
println!("Output: {}", output_path.display());
let client = opts.build_client().await?;
let snapshot_identifier: SnapshotIdentifier =
snapshot_id
.parse()
.map_err(|e: wme_models::error::ModelError| {
anyhow::anyhow!("Invalid snapshot identifier '{}': {}", snapshot_id, e)
})?;
let range_ref = range.as_deref();
let mut stream = client
.snapshot()
.download_chunk(&snapshot_identifier, chunk_id, range_ref)
.await
.map_err(map_client_error)?;
let mut file = tokio::fs::File::create(&output_path).await?;
while let Some(chunk_result) = stream.next().await {
let chunk = chunk_result.map_err(map_client_error)?;
tokio::io::AsyncWriteExt::write_all(&mut file, &chunk).await?;
}
let metadata = tokio::fs::metadata(&output_path).await?;
let size = metadata.len();
let size_str = crate::util::format_size(size);
println!("✓ Download complete!");
println!(" File: {}", output_path.display());
println!(" Size: {}", size_str);
Ok(())
}
pub async fn structured_list(opts: &GlobalOpts) -> Result<()> {
info!("Listing structured snapshots");
println!("Note: Structured snapshots API is in BETA");
let client = opts.build_client().await?;
let params = build_params(opts)?;
let snapshots = client
.snapshot()
.list_structured_snapshots_with_params(params.as_ref())
.await
.map_err(map_client_error)?;
let output = OutputHandler::new(opts.output);
if matches!(opts.output, crate::output::OutputFormat::Table) {
let headers = vec!["Identifier", "Date Modified", "Size"];
let rows: Vec<Vec<String>> = snapshots
.iter()
.map(|snapshot| {
let size_str = crate::util::format_api_size(&snapshot.size);
vec![
snapshot.identifier.to_string(),
snapshot.date_modified.to_string(),
size_str,
]
})
.collect();
output.output_raw(&format_table(&headers, &rows));
} else {
let json_value = serde_json::to_value(&snapshots)?;
output.output(&json_value)?;
}
Ok(())
}
pub async fn structured_info(identifier: &str, opts: &GlobalOpts) -> Result<()> {
info!("Getting structured snapshot info: {}", identifier);
println!("Note: Structured snapshots API is in BETA");
let client = opts.build_client().await?;
let snapshot_id: SnapshotIdentifier =
identifier
.parse()
.map_err(|e: wme_models::error::ModelError| {
anyhow::anyhow!("Invalid snapshot identifier '{}': {}", identifier, e)
})?;
let params = build_params(opts)?;
let snapshot = client
.snapshot()
.get_structured_snapshot_info_with_params(&snapshot_id, params.as_ref())
.await
.map_err(map_client_error)?;
let output = OutputHandler::new(opts.output);
let json_value = serde_json::to_value(&snapshot)?;
output.output(&json_value)?;
Ok(())
}
pub async fn structured_download(
identifier: &str,
output_file: Option<PathBuf>,
range: Option<String>,
opts: &GlobalOpts,
) -> Result<()> {
let output_path =
output_file.unwrap_or_else(|| PathBuf::from(format!("{}_structured.tar.gz", identifier)));
info!(
"Downloading structured snapshot: {} to {}",
identifier,
output_path.display()
);
println!("Note: Structured snapshots API is in BETA");
println!("Downloading structured snapshot: {}", identifier);
println!("Output: {}", output_path.display());
let client = opts.build_client().await?;
let snapshot_id: SnapshotIdentifier =
identifier
.parse()
.map_err(|e: wme_models::error::ModelError| {
anyhow::anyhow!("Invalid snapshot identifier '{}': {}", identifier, e)
})?;
let range_ref = range.as_deref();
let mut stream = client
.snapshot()
.download_structured_snapshot(&snapshot_id, range_ref)
.await
.map_err(map_client_error)?;
let mut file = tokio::fs::File::create(&output_path).await?;
while let Some(chunk_result) = stream.next().await {
let chunk = chunk_result.map_err(map_client_error)?;
tokio::io::AsyncWriteExt::write_all(&mut file, &chunk).await?;
}
let metadata = tokio::fs::metadata(&output_path).await?;
let size = metadata.len();
let size_str = crate::util::format_size(size);
println!("✓ Download complete!");
println!(" File: {}", output_path.display());
println!(" Size: {}", size_str);
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_checkpoint_path_for() {
let output_path = PathBuf::from("snapshot.tar.gz");
let checkpoint_path = DownloadCheckpoint::path_for(&output_path);
assert_eq!(
checkpoint_path.to_string_lossy(),
"snapshot.tar.gz.progress"
);
let output_path = PathBuf::from("/path/to/data.tar.gz");
let checkpoint_path = DownloadCheckpoint::path_for(&output_path);
assert_eq!(
checkpoint_path.to_string_lossy(),
"/path/to/data.tar.gz.progress"
);
}
#[test]
fn test_checkpoint_is_completed() {
let mut checkpoint = DownloadCheckpoint::default();
assert!(!checkpoint.is_completed("chunk_0"));
checkpoint.completed_chunks.push("chunk_0".to_string());
assert!(checkpoint.is_completed("chunk_0"));
assert!(!checkpoint.is_completed("chunk_1"));
}
#[test]
fn test_checkpoint_load_nonexistent() {
let temp_dir = std::env::temp_dir();
let output_path = temp_dir.join("nonexistent_test_file.tar.gz");
let checkpoint = DownloadCheckpoint::load(&output_path);
assert!(checkpoint.completed_chunks.is_empty());
assert!(checkpoint.snapshot_id.is_empty());
}
#[test]
fn test_checkpoint_save_and_load() {
let temp_dir = std::env::temp_dir();
let output_path = temp_dir.join("test_checkpoint_save.tar.gz");
let checkpoint = DownloadCheckpoint {
snapshot_id: "test_snapshot".to_string(),
output_path: output_path.to_string_lossy().to_string(),
completed_chunks: vec!["chunk_0".to_string(), "chunk_1".to_string()],
};
checkpoint.save(&output_path).unwrap();
let loaded = DownloadCheckpoint::load(&output_path);
assert_eq!(loaded.snapshot_id, "test_snapshot");
assert_eq!(loaded.completed_chunks.len(), 2);
assert!(loaded.is_completed("chunk_0"));
assert!(loaded.is_completed("chunk_1"));
assert!(!loaded.is_completed("chunk_2"));
checkpoint.delete(&output_path);
}
#[test]
fn test_checkpoint_mark_completed() {
let temp_dir = std::env::temp_dir();
let output_path = temp_dir.join("test_mark_completed.tar.gz");
let mut checkpoint = DownloadCheckpoint::default();
checkpoint.mark_completed("chunk_0", &output_path).unwrap();
assert!(checkpoint.is_completed("chunk_0"));
checkpoint.mark_completed("chunk_1", &output_path).unwrap();
assert_eq!(checkpoint.completed_chunks.len(), 2);
checkpoint.delete(&output_path);
}
#[test]
fn test_checkpoint_delete() {
let temp_dir = std::env::temp_dir();
let output_path = temp_dir.join("test_delete.tar.gz");
let checkpoint = DownloadCheckpoint::default();
checkpoint.save(&output_path).unwrap();
let cp_path = DownloadCheckpoint::path_for(&output_path);
assert!(cp_path.exists());
checkpoint.delete(&output_path);
assert!(!cp_path.exists());
}
#[test]
fn test_checkpoint_delete_nonexistent() {
let temp_dir = std::env::temp_dir();
let output_path = temp_dir.join("nonexistent_delete.tar.gz");
let checkpoint = DownloadCheckpoint::default();
checkpoint.delete(&output_path);
}
#[test]
fn test_checkpoint_serialization() {
let checkpoint = DownloadCheckpoint {
snapshot_id: "enwiki_namespace_0".to_string(),
output_path: "/data/enwiki.tar.gz".to_string(),
completed_chunks: vec!["enwiki_chunk_0".to_string(), "enwiki_chunk_1".to_string()],
};
let json = serde_json::to_string(&checkpoint).unwrap();
let loaded: DownloadCheckpoint = serde_json::from_str(&json).unwrap();
assert_eq!(loaded.snapshot_id, "enwiki_namespace_0");
assert_eq!(loaded.output_path, "/data/enwiki.tar.gz");
assert_eq!(loaded.completed_chunks.len(), 2);
}
}