use std::fs;
use std::io::Read;
use std::path::{Path, PathBuf};
use crate::model_uri::{address_to_ref, parse_model_address, ModelAddress};
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DatasetFile {
pub path: String,
pub sha256: String,
pub size_bytes: u64,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Resolved {
pub dataset_id: String,
pub digest: String,
pub name: String,
pub files: Vec<DatasetFile>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Lookup {
Resolve(String),
Direct(String),
}
pub fn plan_lookup(reference: &str) -> Result<Lookup, String> {
match parse_model_address(reference) {
Some(ModelAddress::Uuid(id)) => Ok(Lookup::Direct(id)),
Some(addr @ ModelAddress::Named { .. }) => Ok(Lookup::Resolve(address_to_ref(&addr))),
None => Err(format!(
"not a dataset reference: {reference:?}\n\
Expected zc://<owner>/<name> or zc://<uuid>."
)),
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct GetArgs {
pub reference: String,
pub out: Option<String>,
pub api_url: Option<String>,
}
pub fn parse_get_args(args: &[String]) -> Result<GetArgs, String> {
let mut reference: Option<String> = None;
let mut out: Option<String> = None;
let mut api_url: Option<String> = None;
let mut it = args.iter();
while let Some(a) = it.next() {
match a.as_str() {
"-o" | "--out" => {
out = Some(
it.next()
.cloned()
.ok_or_else(|| "-o needs a directory".to_string())?,
)
}
"--api-url" => {
api_url = Some(
it.next()
.cloned()
.ok_or_else(|| "--api-url needs a URL".to_string())?,
)
}
other if other.starts_with('-') => {
return Err(format!("unknown flag: {other}"));
}
other => {
if reference.is_some() {
return Err(format!("unexpected extra argument: {other}"));
}
reference = Some(other.to_string());
}
}
}
match reference {
Some(reference) => Ok(GetArgs {
reference,
out,
api_url,
}),
None => Err("a dataset reference is required".to_string()),
}
}
pub fn sha256_hex(bytes: &[u8]) -> String {
use ring::digest::{digest, SHA256};
digest(&SHA256, bytes)
.as_ref()
.iter()
.map(|b| format!("{b:02x}"))
.collect()
}
pub fn verify(file: &DatasetFile, actual: &str) -> Result<(), String> {
if actual == file.sha256.to_ascii_lowercase() {
return Ok(());
}
Err(format!(
"{}: sha256 mismatch (expected {}, got {})",
file.path, file.sha256, actual
))
}
pub fn safe_destination(dir: &Path, path: &str) -> Option<PathBuf> {
let mut out = dir.to_path_buf();
let mut wrote = false;
for part in path.split('/') {
if part.is_empty() || part == "." || part == ".." {
continue;
}
if part.contains('\\') || part.contains(':') {
return None;
}
out.push(part);
wrote = true;
}
if wrote {
Some(out)
} else {
None
}
}
fn api_base(explicit: Option<&str>) -> String {
explicit
.map(|s| s.to_string())
.unwrap_or_else(crate::credentials::default_api_url)
.trim_end_matches('/')
.to_string()
}
fn api_key() -> Option<String> {
crate::credentials::load_into_env();
std::env::var("ZAKURO_API_KEY")
.ok()
.map(|k| k.trim().to_string())
.filter(|k| !k.is_empty())
}
fn get_json(url: &str, key: Option<&str>) -> Result<(u16, String), String> {
let mut req = ureq::get(url);
if let Some(k) = key {
req = req.header("Authorization", &format!("Bearer {k}"));
}
match req.config().http_status_as_error(false).build().call() {
Ok(r) => {
let status = r.status().as_u16();
let body = r.into_body().read_to_string().unwrap_or_default();
Ok((status, body))
}
Err(e) => Err(e.to_string()),
}
}
#[derive(Debug)]
enum FetchError {
NotFound,
Status(u16),
Other(String),
}
fn stream_to_file(url: &str, key: Option<&str>, part: &Path) -> Result<(u64, String), FetchError> {
use ring::digest::{Context, SHA256};
use std::io::Write;
let mut req = ureq::get(url);
if let Some(k) = key {
req = req.header("Authorization", &format!("Bearer {k}"));
}
let resp = req
.config()
.http_status_as_error(false)
.build()
.call()
.map_err(|e| FetchError::Other(e.to_string()))?;
match resp.status().as_u16() {
200 => {}
401 | 403 | 404 => return Err(FetchError::NotFound),
other => return Err(FetchError::Status(other)),
}
let mut reader = resp.into_body().into_reader();
let mut out = fs::File::create(part).map_err(|e| FetchError::Other(e.to_string()))?;
let mut ctx = Context::new(&SHA256);
let mut buf = [0u8; 64 * 1024];
let mut total = 0u64;
loop {
let n = reader
.read(&mut buf)
.map_err(|e| FetchError::Other(e.to_string()))?;
if n == 0 {
break;
}
ctx.update(&buf[..n]);
out.write_all(&buf[..n])
.map_err(|e| FetchError::Other(e.to_string()))?;
total += n as u64;
}
out.flush().map_err(|e| FetchError::Other(e.to_string()))?;
let hex = ctx
.finish()
.as_ref()
.iter()
.map(|b| format!("{b:02x}"))
.collect();
Ok((total, hex))
}
fn not_found(reference: &str) -> String {
format!(
"Dataset not found: {reference}.\n\
It may not exist, or it may be private to another user. Private\n\
datasets download only with their owner's key (`zc login`)."
)
}
pub fn parse_files(body: &str) -> Result<Vec<DatasetFile>, String> {
let v: serde_json::Value = serde_json::from_str(body).map_err(|e| e.to_string())?;
let files = v
.get("files")
.and_then(|f| f.as_array())
.ok_or_else(|| "response carried no files".to_string())?;
Ok(files
.iter()
.filter_map(|f| {
Some(DatasetFile {
path: f.get("path")?.as_str()?.to_string(),
sha256: f.get("sha256")?.as_str()?.to_string(),
size_bytes: f.get("size_bytes").and_then(|s| s.as_u64()).unwrap_or(0),
})
})
.collect())
}
fn resolve(base: &str, reference: &str, key: Option<&str>) -> Result<Resolved, String> {
match plan_lookup(reference)? {
Lookup::Resolve(canonical) => {
let url = format!("{base}/api/datasets/resolve?ref={}", urlencode(&canonical));
let (status, body) = get_json(&url, key)?;
if matches!(status, 401 | 403 | 404 | 422) {
return Err(not_found(reference));
}
if status != 200 {
return Err(format!("resolve failed: HTTP {status}"));
}
let v: serde_json::Value = serde_json::from_str(&body).map_err(|e| e.to_string())?;
let dataset_id = v
.get("dataset_id")
.and_then(|s| s.as_str())
.ok_or("resolve returned no dataset_id")?
.to_string();
let digest = v
.get("digest")
.and_then(|s| s.as_str())
.ok_or("resolve returned no digest")?
.to_string();
let name = v
.get("name")
.and_then(|s| s.as_str())
.unwrap_or("dataset")
.to_string();
let files = version_files(base, &dataset_id, &digest, key)?;
Ok(Resolved {
dataset_id,
digest,
name,
files,
})
}
Lookup::Direct(id) => {
let (status, body) = get_json(&format!("{base}/api/datasets/{id}"), key)?;
if matches!(status, 401 | 403 | 404) {
return Err(not_found(reference));
}
if status != 200 {
return Err(format!("lookup failed: HTTP {status}"));
}
let v: serde_json::Value = serde_json::from_str(&body).map_err(|e| e.to_string())?;
let latest = v
.get("latest_version")
.filter(|l| !l.is_null())
.ok_or_else(|| format!("dataset {id} has no published version"))?;
let digest = latest
.get("digest")
.and_then(|s| s.as_str())
.ok_or("version carried no digest")?
.to_string();
let name = v
.get("name")
.and_then(|s| s.as_str())
.unwrap_or("dataset")
.to_string();
Ok(Resolved {
dataset_id: id,
digest,
name,
files: parse_files(&latest.to_string())?,
})
}
}
}
fn version_files(
base: &str,
id: &str,
digest: &str,
key: Option<&str>,
) -> Result<Vec<DatasetFile>, String> {
let (status, body) = get_json(&format!("{base}/api/datasets/{id}/versions/{digest}"), key)?;
if status != 200 {
return Err(format!("could not list files: HTTP {status}"));
}
parse_files(&body)
}
pub fn urlencode(s: &str) -> String {
s.bytes()
.map(|b| match b {
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => {
(b as char).to_string()
}
_ => format!("%{b:02X}"),
})
.collect()
}
fn part_path(dest: &Path) -> PathBuf {
let mut name = dest.file_name().unwrap_or_default().to_os_string();
name.push(".zc-part");
dest.with_file_name(name)
}
fn enc_path(p: &str) -> String {
p.split('/').map(urlencode).collect::<Vec<_>>().join("/")
}
fn download_file(
base: &str,
r: &Resolved,
file: &DatasetFile,
key: Option<&str>,
part: &Path,
) -> Result<(u64, String), FetchError> {
let stream = format!(
"{base}/api/datasets/{}/versions/{}/stream?path={}",
r.dataset_id,
r.digest,
urlencode(&file.path)
);
match stream_to_file(&stream, key, part) {
Err(FetchError::NotFound) => {
let legacy = format!(
"{base}/api/datasets/{}/versions/{}/files/{}",
r.dataset_id,
r.digest,
enc_path(&file.path)
);
stream_to_file(&legacy, None, part)
}
other => other,
}
}
pub fn run_get(args: &[String]) -> i32 {
let parsed = match parse_get_args(args) {
Ok(p) => p,
Err(e) => {
eprintln!("{e}");
usage();
return 2;
}
};
let key = api_key();
let key = key.as_deref();
let base = api_base(parsed.api_url.as_deref());
let resolved = match resolve(&base, &parsed.reference, key) {
Ok(r) => r,
Err(e) => {
eprintln!("{e}");
return 1;
}
};
if resolved.files.is_empty() {
eprintln!("{} has no files to download.", parsed.reference);
return 1;
}
let dir = PathBuf::from(parsed.out.unwrap_or_else(|| resolved.name.clone()));
if let Err(e) = fs::create_dir_all(&dir) {
eprintln!("cannot create {}: {e}", dir.display());
return 1;
}
let mut failed = 0;
for file in &resolved.files {
let Some(dest) = safe_destination(&dir, &file.path) else {
eprintln!("{}: unsafe path, skipped", file.path);
failed += 1;
continue;
};
if let Some(parent) = dest.parent() {
if let Err(e) = fs::create_dir_all(parent) {
eprintln!("{}: {e}", file.path);
failed += 1;
continue;
}
}
let part = part_path(&dest);
let size = match download_file(&base, &resolved, file, key, &part) {
Ok((n, sha)) => match verify(file, &sha) {
Ok(()) => n,
Err(e) => {
let _ = fs::remove_file(&part);
eprintln!("{e}");
failed += 1;
continue;
}
},
Err(e) => {
let _ = fs::remove_file(&part);
match e {
FetchError::NotFound => eprintln!("{}", not_found(&parsed.reference)),
FetchError::Status(s) => eprintln!("{}: HTTP {s}", file.path),
FetchError::Other(m) => eprintln!("{}: {m}", file.path),
}
failed += 1;
continue;
}
};
if let Err(e) = fs::rename(&part, &dest) {
let _ = fs::remove_file(&part);
eprintln!("{}: {e}", file.path);
failed += 1;
continue;
}
println!("{} ({} bytes)", dest.display(), size);
}
if failed > 0 {
eprintln!("{failed} file(s) failed.");
return 1;
}
println!("Downloaded {} to {}", resolved.name, dir.display());
0
}
fn usage() {
eprintln!(
"Usage: zc dataset get <zc://owner/name> [-o DIR] [--api-url URL]\n\
\n\
Examples:\n\
\x20 zc dataset get zc://alice/sentiment-mini\n\
\x20 zc dataset get zc://alice/sentiment-mini -o ./data\n\
\n\
Public datasets need no login; private ones you own download with your\n\
key (`zc login` or ZAKURO_API_KEY)."
);
}
pub fn run(args: &[String]) -> i32 {
match args.first().map(|s| s.as_str()) {
Some("get") => run_get(&args[1..]),
Some("-h") | Some("--help") | None => {
usage();
0
}
Some(other) => {
eprintln!("Unknown dataset command: {other}");
usage();
2
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::{Arc, Mutex};
type Seen = Arc<Mutex<Vec<(String, Option<String>)>>>;
fn stub(routes: Vec<(String, u16, Vec<u8>)>) -> (String, Seen) {
let server = tiny_http::Server::http("127.0.0.1:0").unwrap();
let base = format!("http://{}", server.server_addr().to_ip().unwrap());
let seen: Seen = Arc::default();
let log = seen.clone();
std::thread::spawn(move || {
for req in server.incoming_requests() {
let url = req.url().to_string();
let auth = req
.headers()
.iter()
.find(|h| h.field.equiv("Authorization"))
.map(|h| h.value.to_string());
log.lock().unwrap().push((url.clone(), auth));
let (status, body) = routes
.iter()
.find(|(p, _, _)| url == *p)
.map(|(_, s, b)| (*s, b.clone()))
.unwrap_or((404, b"{}".to_vec()));
let _ = req.respond(tiny_http::Response::from_data(body).with_status_code(status));
}
});
(base, seen)
}
const ID: &str = "e629e662-1d32-4e52-88e9-b0e83416c852";
const DIG: &str = "1111111111111111111111111111111111111111111111111111111111111111";
fn private_routes(body: &[u8]) -> Vec<(String, u16, Vec<u8>)> {
let ver = format!(
r#"{{"files":[{{"path":"data.csv","sha256":"{}","size_bytes":{}}}]}}"#,
sha256_hex(body),
body.len()
);
let resolve = format!(r#"{{"dataset_id":"{ID}","digest":"{DIG}","name":"mine"}}"#);
vec![
(
"/api/datasets/resolve?ref=zc%3A%2F%2Falice%2Fmine".to_string(),
200,
resolve.into_bytes(),
),
(
format!("/api/datasets/{ID}/versions/{DIG}"),
200,
ver.into_bytes(),
),
(
format!("/api/datasets/{ID}/versions/{DIG}/stream?path=data.csv"),
200,
body.to_vec(),
),
]
}
fn tempfile_dir(tag: &str) -> PathBuf {
let d = std::env::temp_dir().join(format!(
"zc-dataset-{tag}-{}-{:?}",
std::process::id(),
std::thread::current().id()
));
let _ = fs::remove_dir_all(&d);
fs::create_dir_all(&d).unwrap();
d
}
fn get(base: String, out: &Path) -> i32 {
run_get(&[
"zc://alice/mine".into(),
"-o".into(),
out.display().to_string(),
"--api-url".into(),
base,
])
}
#[test]
fn a_private_dataset_streams_with_the_key() {
let body = b"id,name\n1,ada\n".to_vec();
let (base, seen) = stub(private_routes(&body));
let out = tempfile_dir("priv");
let r = resolve(&base, "zc://alice/mine", Some("zk_1_abc")).unwrap();
let part = out.join("data.csv.zc-part");
let (n, sha) = download_file(&base, &r, &r.files[0], Some("zk_1_abc"), &part).unwrap();
assert_eq!(n, body.len() as u64);
assert!(verify(&r.files[0], &sha).is_ok());
assert_eq!(fs::read(&part).unwrap(), body);
let seen = seen.lock().unwrap();
assert_eq!(seen.len(), 3);
assert!(seen
.iter()
.all(|(_, a)| a.as_deref() == Some("Bearer zk_1_abc")));
assert!(seen[2].0.contains("/stream?path=data.csv"));
}
#[test]
fn run_get_writes_the_file_and_leaves_no_part_file() {
let body = b"hello".to_vec();
let (base, _) = stub(private_routes(&body));
let out = tempfile_dir("run");
std::env::set_var("ZAKURO_API_KEY", "zk_1_abc");
assert_eq!(get(base, &out), 0);
assert_eq!(fs::read(out.join("data.csv")).unwrap(), body);
assert!(!out.join("data.csv.zc-part").exists());
}
#[test]
fn a_foreign_private_dataset_reports_not_found_like_a_missing_one() {
let (base, _) = stub(vec![]);
let err = resolve(&base, "zc://alice/mine", Some("zk_1_other")).unwrap_err();
assert_eq!(err, not_found("zc://alice/mine"));
let (base, _) = stub(vec![(format!("/api/datasets/{ID}"), 403, vec![])]);
let err = resolve(&base, ID, Some("zk_1_other")).unwrap_err();
assert_eq!(err, not_found(ID));
}
#[test]
fn a_corrupt_stream_is_rejected_and_not_renamed() {
let mut routes = private_routes(b"hello");
routes[2].2 = b"hellX".to_vec();
let (base, _) = stub(routes);
let out = tempfile_dir("bad");
assert_eq!(get(base, &out), 1);
assert!(!out.join("data.csv").exists());
assert!(!out.join("data.csv.zc-part").exists());
}
#[test]
fn an_older_hub_without_stream_falls_back_to_the_legacy_route() {
let mut routes = private_routes(b"hello");
routes.remove(2);
routes.push((
format!("/api/datasets/{ID}/versions/{DIG}/files/data.csv"),
200,
b"hello".to_vec(),
));
let (base, seen) = stub(routes);
let out = tempfile_dir("legacy");
let r = resolve(&base, "zc://alice/mine", None).unwrap();
let part = out.join("data.csv.zc-part");
let (n, _) = download_file(&base, &r, &r.files[0], None, &part).unwrap();
assert_eq!(n, 5);
let last = seen.lock().unwrap().last().unwrap().clone();
assert!(last.0.ends_with("/files/data.csv"));
assert_eq!(last.1, None, "the legacy route must not carry the key");
}
#[test]
fn a_named_reference_resolves_through_the_resolve_route() {
assert_eq!(
plan_lookup("zc://alice/sentiment-mini").unwrap(),
Lookup::Resolve("zc://alice/sentiment-mini".to_string())
);
}
#[test]
fn a_pinned_digest_survives_into_the_resolve_ref() {
let d = "a".repeat(64);
assert_eq!(
plan_lookup(&format!("zc://alice/mini@sha256:{d}")).unwrap(),
Lookup::Resolve(format!("zc://alice/mini@sha256:{d}"))
);
}
#[test]
fn a_uuid_reference_never_goes_to_resolve() {
let id = "e629e662-1d32-4e52-88e9-b0e83416c852";
assert_eq!(
plan_lookup(&format!("zc://{id}")).unwrap(),
Lookup::Direct(id.to_string())
);
}
#[test]
fn a_bare_uuid_is_accepted_too() {
let id = "e629e662-1d32-4e52-88e9-b0e83416c852";
assert_eq!(plan_lookup(id).unwrap(), Lookup::Direct(id.to_string()));
}
#[test]
fn nonsense_is_rejected_with_the_expected_shapes_named() {
let err = plan_lookup("https://example.com/x").unwrap_err();
assert!(err.contains("zc://<owner>/<name>"), "{err}");
}
#[test]
fn parses_a_reference_with_an_output_directory() {
let args: Vec<String> = ["zc://a/b", "-o", "./data"]
.iter()
.map(|s| s.to_string())
.collect();
let got = parse_get_args(&args).unwrap();
assert_eq!(got.reference, "zc://a/b");
assert_eq!(got.out.as_deref(), Some("./data"));
}
#[test]
fn an_api_url_flag_overrides_the_default() {
let args: Vec<String> = ["zc://a/b", "--api-url", "http://localhost:8000"]
.iter()
.map(|s| s.to_string())
.collect();
assert_eq!(
parse_get_args(&args).unwrap().api_url.as_deref(),
Some("http://localhost:8000")
);
}
#[test]
fn a_missing_reference_is_an_error_not_a_download_of_nothing() {
assert!(parse_get_args(&[]).is_err());
}
#[test]
fn a_dangling_output_flag_is_an_error() {
let args = vec!["zc://a/b".to_string(), "-o".to_string()];
assert!(parse_get_args(&args).is_err());
}
#[test]
fn an_unknown_flag_is_refused_rather_than_ignored() {
let args = vec!["zc://a/b".to_string(), "--recursive".to_string()];
assert!(parse_get_args(&args).is_err());
}
#[test]
fn sha256_matches_a_known_vector() {
assert_eq!(
sha256_hex(b"abc"),
"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"
);
}
#[test]
fn a_matching_hash_verifies() {
let f = DatasetFile {
path: "d.csv".into(),
sha256: sha256_hex(b"hello"),
size_bytes: 5,
};
assert!(verify(&f, &sha256_hex(b"hello")).is_ok());
}
#[test]
fn a_corrupted_download_is_rejected() {
let f = DatasetFile {
path: "d.csv".into(),
sha256: sha256_hex(b"hello"),
size_bytes: 5,
};
let err = verify(&f, &sha256_hex(b"hello!")).unwrap_err();
assert!(err.contains("sha256 mismatch"), "{err}");
}
#[test]
fn an_uppercase_published_hash_still_matches() {
let f = DatasetFile {
path: "d.csv".into(),
sha256: sha256_hex(b"hello").to_uppercase(),
size_bytes: 5,
};
assert!(verify(&f, &sha256_hex(b"hello")).is_ok());
}
#[test]
fn a_nested_path_lands_under_the_output_directory() {
let dir = Path::new("/tmp/out");
assert_eq!(
safe_destination(dir, "train/part-0.csv").unwrap(),
PathBuf::from("/tmp/out/train/part-0.csv")
);
}
#[test]
fn a_traversing_path_cannot_climb_out() {
let dir = Path::new("/tmp/out");
assert_eq!(
safe_destination(dir, "../../etc/passwd").unwrap(),
PathBuf::from("/tmp/out/etc/passwd")
);
}
#[test]
fn an_absolute_path_is_reparented_not_honoured() {
let dir = Path::new("/tmp/out");
assert_eq!(
safe_destination(dir, "/etc/passwd").unwrap(),
PathBuf::from("/tmp/out/etc/passwd")
);
}
#[test]
fn a_path_that_is_only_dots_yields_nothing_to_write() {
assert!(safe_destination(Path::new("/tmp/out"), "../..").is_none());
}
#[test]
fn files_are_read_out_of_a_version_body() {
let body = r#"{"files":[{"path":"d.csv","sha256":"ab","size_bytes":12}]}"#;
assert_eq!(
parse_files(body).unwrap(),
vec![DatasetFile {
path: "d.csv".into(),
sha256: "ab".into(),
size_bytes: 12
}]
);
}
#[test]
fn a_body_without_files_is_an_error() {
assert!(parse_files(r#"{"id":"x"}"#).is_err());
}
#[test]
fn the_reference_is_encoded_for_a_query_string() {
assert_eq!(urlencode("zc://alice/mini"), "zc%3A%2F%2Falice%2Fmini");
}
#[test]
fn the_404_message_covers_missing_and_foreign_private_alike() {
let msg = not_found("zc://a/b");
assert!(msg.contains("not found"), "{msg}");
assert!(msg.contains("private"), "{msg}");
}
#[test]
fn an_unknown_subcommand_is_refused() {
assert_eq!(run(&["frobnicate".to_string()]), 2);
}
#[test]
fn help_is_not_an_error() {
assert_eq!(run(&["--help".to_string()]), 0);
}
}