use std::path::{Path, PathBuf};
use std::process::Command;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Request {
pub url: String,
pub headers: Vec<(String, String)>,
pub dest: Option<PathBuf>,
pub resume: bool,
pub connect_timeout_secs: u32,
pub retries: u32,
}
impl Request {
pub fn get(url: impl Into<String>) -> Request {
Request {
url: url.into(),
headers: Vec::new(),
dest: None,
resume: false,
connect_timeout_secs: 20,
retries: 3,
}
}
pub fn header(mut self, k: impl Into<String>, v: impl Into<String>) -> Request {
self.headers.push((k.into(), v.into()));
self
}
pub fn to_file(mut self, dest: impl Into<PathBuf>) -> Request {
self.dest = Some(dest.into());
self
}
pub fn resuming(mut self) -> Request {
self.resume = true;
self
}
pub fn argv(&self) -> Vec<String> {
let mut v: Vec<String> = vec![
"--fail".into(),
"--location".into(),
"--silent".into(),
"--show-error".into(),
"--connect-timeout".into(),
self.connect_timeout_secs.to_string(),
"--retry".into(),
self.retries.to_string(),
"--retry-connrefused".into(),
];
for (k, val) in &self.headers {
v.push("--header".into());
v.push(format!("{k}: {val}"));
}
if let Some(dest) = &self.dest {
if self.resume {
v.push("--continue-at".into());
v.push("-".into());
}
v.push("--output".into());
v.push(dest.display().to_string());
}
v.push("--".into());
v.push(self.url.clone());
v
}
}
#[derive(Debug)]
pub struct CurlError(pub String);
impl std::fmt::Display for CurlError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.0)
}
}
impl std::error::Error for CurlError {}
pub fn curl_available() -> bool {
Command::new("curl")
.arg("--version")
.stdin(std::process::Stdio::null())
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.status()
.map(|s| s.success())
.unwrap_or(false)
}
pub fn explain_failure(code: Option<i32>, stderr: &str) -> String {
let said = stderr.trim();
let known = match code {
Some(6) => Some("could not resolve the host — DNS, or no network from this box"),
Some(7) => Some("could not connect — the host is reachable but refused us"),
Some(22) => Some("the server returned an error status (see below)"),
Some(28) => Some("timed out before the transfer began"),
Some(23) => Some("could not write the file — check the disk and permissions"),
Some(35) | Some(60) => Some("TLS failed — a proxy or a clock skew will do this"),
Some(33) => Some("the server refused a resumed range; retry without resume"),
_ => None,
};
match (known, said.is_empty()) {
(Some(k), true) => format!("curl: {k}"),
(Some(k), false) => format!("curl: {k}\n {said}"),
(None, true) => format!("curl exited {code:?} and said nothing"),
(None, false) => format!("curl exited {code:?}: {said}"),
}
}
pub fn fetch_string(req: &Request) -> Result<String, CurlError> {
if !curl_available() {
return Err(CurlError(
"curl is not on PATH — hearth uses it for registry downloads (TLS, \
redirects, resume). Install curl, or pre-place the blobs yourself."
.into(),
));
}
let out = Command::new("curl")
.args(req.argv())
.stdin(std::process::Stdio::null())
.output()
.map_err(|e| CurlError(format!("could not run curl: {e}")))?;
if !out.status.success() {
return Err(CurlError(format!(
"{}\n url: {}",
explain_failure(out.status.code(), &String::from_utf8_lossy(&out.stderr)),
req.url
)));
}
Ok(String::from_utf8_lossy(&out.stdout).to_string())
}
pub fn fetch_file(req: &Request, show_progress: bool) -> Result<u64, CurlError> {
let dest = req
.dest
.clone()
.ok_or_else(|| CurlError("fetch_file needs a destination".into()))?;
if !curl_available() {
return Err(CurlError(
"curl is not on PATH — hearth uses it for registry downloads (TLS, \
redirects, resume). Install curl, or pre-place the blobs yourself."
.into(),
));
}
if let Some(parent) = dest.parent() {
std::fs::create_dir_all(parent)
.map_err(|e| CurlError(format!("could not create {}: {e}", parent.display())))?;
}
let mut argv = req.argv();
if show_progress {
if let Some(i) = argv.iter().position(|a| a == "--silent") {
argv[i] = "--progress-bar".into();
}
}
let status = Command::new("curl")
.args(&argv)
.stdin(std::process::Stdio::null())
.stderr(std::process::Stdio::inherit())
.status()
.map_err(|e| CurlError(format!("could not run curl: {e}")))?;
if !status.success() {
return Err(CurlError(format!(
"{}\n url: {}",
explain_failure(status.code(), ""),
req.url
)));
}
file_len(&dest)
.map_err(|e| CurlError(format!("{} vanished after download: {e}", dest.display())))
}
fn file_len(p: &Path) -> std::io::Result<u64> {
Ok(std::fs::metadata(p)?.len())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_404_must_not_be_written_to_disk_as_a_model() {
let argv = Request::get("https://example.invalid/blob")
.to_file("/tmp/x")
.argv();
assert!(argv.contains(&"--fail".to_string()));
}
#[test]
fn redirects_are_followed_because_huggingface_always_redirects() {
let argv = Request::get("https://huggingface.co/x").argv();
assert!(argv.contains(&"--location".to_string()));
}
#[test]
fn headers_are_passed_as_one_argument_each() {
let argv = Request::get("https://r/x")
.header(
"Accept",
"application/vnd.docker.distribution.manifest.v2+json",
)
.argv();
let i = argv.iter().position(|a| a == "--header").expect("--header");
assert_eq!(
argv[i + 1],
"Accept: application/vnd.docker.distribution.manifest.v2+json"
);
}
#[test]
fn resume_is_only_added_when_asked_and_only_with_a_destination() {
let plain = Request::get("https://r/x").to_file("/tmp/x").argv();
assert!(!plain.contains(&"--continue-at".to_string()));
let resumed = Request::get("https://r/x")
.to_file("/tmp/x")
.resuming()
.argv();
let i = resumed.iter().position(|a| a == "--continue-at").unwrap();
assert_eq!(resumed[i + 1], "-");
let no_dest = Request {
resume: true,
..Request::get("https://r/x")
};
assert!(!no_dest.argv().contains(&"--continue-at".to_string()));
}
#[test]
fn the_url_is_after_a_double_dash_so_a_leading_dash_is_not_a_flag() {
let argv = Request::get("-oops").argv();
let i = argv.iter().position(|a| a == "--").expect("--");
assert_eq!(argv[i + 1], "-oops");
assert_eq!(i + 2, argv.len(), "the url is last");
}
#[test]
fn retry_is_configured_but_no_max_time_is() {
let argv = Request::get("https://r/x").argv();
assert!(argv.contains(&"--retry".to_string()));
assert!(argv.contains(&"--retry-connrefused".to_string()));
assert!(
!argv.iter().any(|a| a == "--max-time"),
"a slow model is slow, not broken"
);
}
#[test]
fn failures_are_explained_by_cause_not_just_by_number() {
assert!(explain_failure(Some(6), "").contains("resolve"));
assert!(explain_failure(Some(22), "404").contains("error status"));
assert!(explain_failure(Some(22), "404").contains("404"));
assert!(explain_failure(Some(23), "").contains("disk"));
assert!(explain_failure(Some(33), "").contains("resume"));
let odd = explain_failure(Some(99), "something odd");
assert!(odd.contains("99") && odd.contains("something odd"));
assert!(explain_failure(None, "").contains("said nothing"));
}
}