pub mod curl;
pub mod registry;
pub mod runtime;
use std::path::{Path, PathBuf};
use hearth_core::sha256;
use hearth_resolve::Reference;
use hearth_store::{EventRef, Spine, Transition};
pub use registry::{Blob, Fetched};
#[derive(Debug, Clone)]
pub struct PullConfig {
pub blobs_dir: PathBuf,
pub progress: bool,
pub verify_existing: bool,
}
impl Default for PullConfig {
fn default() -> Self {
PullConfig {
blobs_dir: PathBuf::from("./blobs"),
progress: true,
verify_existing: false,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Pulled {
pub model: String,
pub source: String,
pub weights_path: PathBuf,
pub bytes: u64,
pub already_had_it: bool,
}
pub fn blob_path(blobs_dir: &Path, digest: &str) -> PathBuf {
blobs_dir.join(format!("sha256-{}", sha256::normalize(digest)))
}
pub fn human_bytes(n: u64) -> String {
const GIB: f64 = 1024.0 * 1024.0 * 1024.0;
const MIB: f64 = 1024.0 * 1024.0;
let f = n as f64;
if f >= GIB {
format!("{:.2} GiB", f / GIB)
} else if f >= MIB {
format!("{:.1} MiB", f / MIB)
} else {
format!("{n} B")
}
}
fn digest_file_loud(path: &Path, label: &str, progress: bool) -> Result<String, String> {
use std::io::Write;
let started = std::time::Instant::now();
let mut last = std::time::Instant::now();
let digest = sha256::hex_digest_file_with_progress(path, |done, total| {
if !progress {
return;
}
if last.elapsed() < std::time::Duration::from_millis(500) && done < total {
return;
}
last = std::time::Instant::now();
let secs = started.elapsed().as_secs_f64().max(0.001);
let rate = (done as f64 / secs) as u64;
let pct = if total > 0 {
format!("{:5.1}%", done as f64 / total as f64 * 100.0)
} else {
" ? ".to_string()
};
let _ = write!(
std::io::stderr(),
"\rhearth: verifying {label} {pct} ({} / {}) {}/s ",
human_bytes(done),
human_bytes(total),
human_bytes(rate)
);
let _ = std::io::stderr().flush();
})
.map_err(|e| format!("could not hash {}: {e}", path.display()))?;
if progress {
let secs = started.elapsed().as_secs_f64();
let len = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0);
let rate = if secs > 0.0 {
(len as f64 / secs) as u64
} else {
0
};
let _ = writeln!(
std::io::stderr(),
"\rhearth: verified {label} — {} in {:.1}s ({}/s) ",
human_bytes(len),
secs,
human_bytes(rate)
);
}
Ok(digest)
}
pub fn fetch_blob(blob: &Blob, cfg: &PullConfig) -> Result<(PathBuf, u64, bool), String> {
match &blob.digest {
Some(digest) => fetch_blob_verified(blob, digest, cfg),
None => fetch_blob_self_verified(blob, cfg),
}
}
fn fetch_blob_verified(
blob: &Blob,
digest: &str,
cfg: &PullConfig,
) -> Result<(PathBuf, u64, bool), String> {
let final_path = blob_path(&cfg.blobs_dir, digest);
if final_path.exists() {
let len = std::fs::metadata(&final_path).map(|m| m.len()).unwrap_or(0);
if cfg.verify_existing {
let actual = digest_file_loud(&final_path, &blob.name, cfg.progress)?;
if !sha256::matches(digest, &actual) {
eprintln!(
"hearth: {} does not match its own name — refetching",
final_path.display()
);
let _ = std::fs::remove_file(&final_path);
} else {
return Ok((final_path, len, true));
}
} else if blob.size_bytes == 0 || len == blob.size_bytes {
return Ok((final_path, len, true));
} else {
eprintln!(
"hearth: {} is {len} bytes, expected {} — refetching",
final_path.display(),
blob.size_bytes
);
let _ = std::fs::remove_file(&final_path);
}
}
let partial = final_path.with_extension("partial");
download_to(&partial, blob, cfg)?;
let actual = digest_file_loud(&partial, &blob.name, cfg.progress)?;
if !sha256::matches(digest, &actual) {
let _ = std::fs::remove_file(&partial);
return Err(format!(
"digest mismatch for {}\n expected sha256:{}\n got sha256:{}\n \
the file has been deleted; a resumed transfer over a proxy that \
rewrote a range will do this, so a plain retry is worth one attempt",
blob.name,
sha256::normalize(digest),
actual
));
}
let len = std::fs::metadata(&partial).map(|m| m.len()).unwrap_or(0);
std::fs::rename(&partial, &final_path)
.map_err(|e| format!("could not commit {}: {e}", final_path.display()))?;
Ok((final_path, len, false))
}
fn fetch_blob_self_verified(blob: &Blob, cfg: &PullConfig) -> Result<(PathBuf, u64, bool), String> {
std::fs::create_dir_all(&cfg.blobs_dir).map_err(|e| e.to_string())?;
let safe_name = blob
.name
.chars()
.map(|c| {
if c.is_ascii_alphanumeric() || c == '.' || c == '-' {
c
} else {
'_'
}
})
.collect::<String>();
let partial = cfg
.blobs_dir
.join(format!("{safe_name}.unverified.partial"));
download_to(&partial, blob, cfg)?;
let actual = digest_file_loud(&partial, &blob.name, cfg.progress)?;
let final_path = blob_path(&cfg.blobs_dir, &actual);
if final_path.exists() {
let _ = std::fs::remove_file(&partial);
let len = std::fs::metadata(&final_path).map(|m| m.len()).unwrap_or(0);
return Ok((final_path, len, true));
}
let len = std::fs::metadata(&partial).map(|m| m.len()).unwrap_or(0);
std::fs::rename(&partial, &final_path)
.map_err(|e| format!("could not commit {}: {e}", final_path.display()))?;
eprintln!(
"hearth: {} has no published digest — stored as sha256:{actual}, \
self-consistent only (not checked against a source, because the \
source published none)",
blob.name
);
Ok((final_path, len, false))
}
fn download_to(dest: &Path, blob: &Blob, cfg: &PullConfig) -> Result<(), String> {
let req = {
let mut r = curl::Request::get(&blob.url).to_file(dest);
for (k, v) in &blob.headers {
r = r.header(k, v);
}
if dest.exists() && std::fs::metadata(dest).map(|m| m.len()).unwrap_or(0) > 0 {
r = r.resuming();
}
r
};
curl::fetch_file(&req, cfg.progress).map_err(|e| e.0)?;
Ok(())
}
pub fn pull(reference: &str, cfg: &PullConfig, spine: &Spine) -> Result<Pulled, String> {
let reference = Reference::parse(reference).map_err(|e| e.0)?;
let model = reference.display_name();
let source = registry::source_string(&reference);
if let Reference::Local { path } = &reference {
let p = PathBuf::from(path);
if !p.exists() {
return Err(format!("no file at {path}"));
}
let bytes = std::fs::metadata(&p).map(|m| m.len()).unwrap_or(0);
return Ok(Pulled {
model,
source,
weights_path: p,
bytes,
already_had_it: true,
});
}
let started = spine
.record(
&model,
&Transition::PullStarted {
source: source.clone(),
},
&[],
)
.map_err(|e| format!("could not record the pull: {e}"))?;
if cfg.progress {
eprintln!("hearth: resolving {source} …");
}
let fetched = registry::resolve_blobs(&reference).map_err(|e| {
let _ = spine.record(
&model,
&Transition::PullStarted {
source: format!("{source} — FAILED: {e}"),
},
std::slice::from_ref(&started),
);
e
})?;
let n = fetched.blobs.len();
if cfg.progress {
let total: u64 = fetched.blobs.iter().map(|b| b.size_bytes).sum();
eprintln!(
"hearth: {n} file(s), {} to fetch — then a sha256 verify of each",
human_bytes(total)
);
}
let mut weights: Option<(PathBuf, u64)> = None;
let mut cached_all = true;
for (i, blob) in fetched.blobs.iter().enumerate() {
if cfg.progress {
eprintln!(
"hearth: [{}/{n}] {} ({})",
i + 1,
blob.name,
if blob.size_bytes > 0 {
human_bytes(blob.size_bytes)
} else {
"size unknown".to_string()
}
);
}
let (path, len, cached) = fetch_blob(blob, cfg)?;
if cfg.progress && cached {
eprintln!("hearth: [{}/{n}] already on disk, skipped", i + 1);
}
if !cached {
cached_all = false;
}
if blob.is_weights {
weights = Some((path, len));
}
}
let (weights_path, bytes) = weights.ok_or_else(|| {
format!(
"{model} resolved with no weights layer — the registry returned \
{} blob(s), none of them the model",
fetched.blobs.len()
)
})?;
spine
.record(
&model,
&Transition::PullCompleted {
path: weights_path.display().to_string(),
size_bytes: bytes,
},
&[EventRef {
hash: started.hash,
seq: started.seq,
}],
)
.map_err(|e| format!("could not record completion: {e}"))?;
Ok(Pulled {
model,
source,
weights_path,
bytes,
already_had_it: cached_all,
})
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_blob_path_is_its_digest_so_presence_is_a_stat() {
let p = blob_path(Path::new("/b"), "sha256:ABC123");
assert_eq!(p, Path::new("/b/sha256-abc123"));
assert_eq!(p, blob_path(Path::new("/b"), "abc123"));
}
#[test]
fn a_local_reference_is_not_recorded_as_a_pull() {
let dir = std::env::temp_dir().join("hearth-pull-local");
std::fs::create_dir_all(&dir).unwrap();
let gguf = dir.join("muse.gguf");
std::fs::write(&gguf, b"weights").unwrap();
let spine = Spine::in_memory();
let out = pull(
&format!("file:{}", gguf.display()),
&PullConfig::default(),
&spine,
)
.unwrap();
assert!(out.already_had_it);
assert_eq!(out.bytes, 7);
assert!(
spine.latest(&out.model).is_none(),
"nothing was fetched, so claiming a pull would be a lie"
);
std::fs::remove_file(&gguf).ok();
}
#[test]
fn a_missing_local_file_is_refused_before_anything_is_recorded() {
let spine = Spine::in_memory();
let err = pull(
"file:/definitely/not/here.gguf",
&PullConfig::default(),
&spine,
)
.unwrap_err();
assert!(err.contains("no file at"), "{err}");
assert_eq!(spine.seq(), 0, "nothing should have been written");
}
#[test]
fn a_digest_mismatch_deletes_the_file_rather_than_caching_corruption() {
let dir = std::env::temp_dir().join("hearth-pull-mismatch");
std::fs::create_dir_all(&dir).unwrap();
let served = dir.join("served.bin");
std::fs::write(&served, b"not what we asked for").unwrap();
let blobs = dir.join("blobs");
let cfg = PullConfig {
blobs_dir: blobs.clone(),
progress: false,
verify_existing: false,
};
let blob = Blob {
name: "weights".into(),
url: format!("file://{}", served.display()),
digest: Some(
"sha256:0000000000000000000000000000000000000000000000000000000000000000".into(),
),
size_bytes: 0,
headers: vec![],
is_weights: true,
};
let err = fetch_blob(&blob, &cfg).unwrap_err();
assert!(err.contains("digest mismatch"), "{err}");
let digest = blob.digest.as_deref().unwrap();
assert!(
!blob_path(&blobs, digest).exists(),
"a blob that failed verification must not be left on disk"
);
assert!(
!blob_path(&blobs, digest).with_extension("partial").exists(),
"and neither must the partial"
);
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn a_matching_digest_commits_the_file_under_its_digest() {
let dir = std::env::temp_dir().join("hearth-pull-match");
std::fs::create_dir_all(&dir).unwrap();
let body = b"the actual weights, honestly";
let served = dir.join("served.bin");
std::fs::write(&served, body).unwrap();
let blobs = dir.join("blobs");
let cfg = PullConfig {
blobs_dir: blobs.clone(),
progress: false,
verify_existing: false,
};
let blob = Blob {
name: "weights".into(),
url: format!("file://{}", served.display()),
digest: Some(sha256::hex_digest(body)),
size_bytes: body.len() as u64,
headers: vec![],
is_weights: true,
};
let (path, len, cached) = fetch_blob(&blob, &cfg).unwrap();
assert!(!cached);
assert_eq!(len, body.len() as u64);
assert_eq!(std::fs::read(&path).unwrap(), body);
let (_, _, cached2) = fetch_blob(&blob, &cfg).unwrap();
assert!(cached2, "the filename IS the digest — presence is enough");
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn a_cached_blob_of_the_wrong_size_is_refetched() {
let dir = std::env::temp_dir().join("hearth-pull-truncated");
std::fs::create_dir_all(&dir).unwrap();
let body = b"complete contents here";
let served = dir.join("served.bin");
std::fs::write(&served, body).unwrap();
let blobs = dir.join("blobs");
std::fs::create_dir_all(&blobs).unwrap();
let digest = sha256::hex_digest(body);
std::fs::write(blob_path(&blobs, &digest), b"trunc").unwrap();
let cfg = PullConfig {
blobs_dir: blobs.clone(),
progress: false,
verify_existing: false,
};
let blob = Blob {
name: "weights".into(),
url: format!("file://{}", served.display()),
digest: Some(digest),
size_bytes: body.len() as u64,
headers: vec![],
is_weights: true,
};
let (path, len, cached) = fetch_blob(&blob, &cfg).unwrap();
assert!(!cached, "the size did not match, so it was not trusted");
assert_eq!(len, body.len() as u64);
assert_eq!(std::fs::read(&path).unwrap(), body);
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn no_digest_still_lands_the_file_under_its_own_computed_sha256() {
let dir = std::env::temp_dir().join("hearth-pull-no-digest");
std::fs::create_dir_all(&dir).unwrap();
let body = b"weights nobody published a digest for";
let served = dir.join("served.bin");
std::fs::write(&served, body).unwrap();
let blobs = dir.join("blobs");
let cfg = PullConfig {
blobs_dir: blobs.clone(),
progress: false,
verify_existing: false,
};
let blob = Blob {
name: "model.gguf".into(),
url: format!("file://{}", served.display()),
digest: None,
size_bytes: 0,
headers: vec![],
is_weights: true,
};
let (path, len, cached) = fetch_blob(&blob, &cfg).unwrap();
assert!(!cached);
assert_eq!(len, body.len() as u64);
assert_eq!(std::fs::read(&path).unwrap(), body);
let expected_name = format!("sha256-{}", sha256::hex_digest(body));
assert_eq!(
path.file_name().unwrap().to_str().unwrap(),
expected_name,
"the final path must be the digest hearth computed, not a guess"
);
assert!(
!blobs.join("model.gguf.unverified.partial").exists(),
"the working file must not survive under its temporary name"
);
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn a_second_no_digest_pull_of_identical_content_is_recognised_as_cached() {
let dir = std::env::temp_dir().join("hearth-pull-no-digest-dup");
std::fs::create_dir_all(&dir).unwrap();
let body = b"identical weights served from two places";
let served = dir.join("served.bin");
std::fs::write(&served, body).unwrap();
let blobs = dir.join("blobs");
let cfg = PullConfig {
blobs_dir: blobs.clone(),
progress: false,
verify_existing: false,
};
let blob = Blob {
name: "model.gguf".into(),
url: format!("file://{}", served.display()),
digest: None,
size_bytes: 0,
headers: vec![],
is_weights: true,
};
let (first_path, ..) = fetch_blob(&blob, &cfg).unwrap();
let (second_path, _, cached) = fetch_blob(&blob, &cfg).unwrap();
assert!(
cached,
"identical content must be recognised, not re-stored"
);
assert_eq!(first_path, second_path);
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn an_unsafe_blob_name_does_not_escape_the_blobs_directory() {
let dir = std::env::temp_dir().join("hearth-pull-unsafe-name");
std::fs::create_dir_all(&dir).unwrap();
let body = b"contents";
let served = dir.join("served.bin");
std::fs::write(&served, body).unwrap();
let blobs = dir.join("blobs");
let cfg = PullConfig {
blobs_dir: blobs.clone(),
progress: false,
verify_existing: false,
};
let blob = Blob {
name: "../../etc/model.gguf".into(),
url: format!("file://{}", served.display()),
digest: None,
size_bytes: 0,
headers: vec![],
is_weights: true,
};
let (path, ..) = fetch_blob(&blob, &cfg).unwrap();
assert!(
path.starts_with(&blobs),
"the final path must stay under blobs_dir, got {}",
path.display()
);
std::fs::remove_dir_all(&dir).ok();
}
}