pub mod curl;
pub mod registry;
pub mod runtime;
use std::path::{Path, PathBuf};
use hearth_core::sha256;
use hearth_resolve::Reference;
use hearth_store::{EventRef, Spine, Transition};
pub use registry::{Blob, Fetched};
#[derive(Debug, Clone)]
pub struct PullConfig {
pub blobs_dir: PathBuf,
pub progress: bool,
pub verify_existing: bool,
}
impl Default for PullConfig {
fn default() -> Self {
PullConfig {
blobs_dir: PathBuf::from("./blobs"),
progress: true,
verify_existing: false,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Pulled {
pub model: String,
pub source: String,
pub weights_path: PathBuf,
pub bytes: u64,
pub already_had_it: bool,
}
pub fn blob_path(blobs_dir: &Path, digest: &str) -> PathBuf {
blobs_dir.join(format!("sha256-{}", sha256::normalize(digest)))
}
pub fn fetch_blob(blob: &Blob, cfg: &PullConfig) -> Result<(PathBuf, u64, bool), String> {
let final_path = blob_path(&cfg.blobs_dir, &blob.digest);
if final_path.exists() {
let len = std::fs::metadata(&final_path).map(|m| m.len()).unwrap_or(0);
if cfg.verify_existing {
let actual = sha256::hex_digest_file(&final_path)
.map_err(|e| format!("could not read {}: {e}", final_path.display()))?;
if !sha256::matches(&blob.digest, &actual) {
eprintln!(
"hearth: {} does not match its own name — refetching",
final_path.display()
);
let _ = std::fs::remove_file(&final_path);
} else {
return Ok((final_path, len, true));
}
} else if blob.size_bytes == 0 || len == blob.size_bytes {
return Ok((final_path, len, true));
} else {
eprintln!(
"hearth: {} is {len} bytes, expected {} — refetching",
final_path.display(),
blob.size_bytes
);
let _ = std::fs::remove_file(&final_path);
}
}
let partial = final_path.with_extension("partial");
let req = {
let mut r = curl::Request::get(&blob.url).to_file(&partial);
for (k, v) in &blob.headers {
r = r.header(k, v);
}
if partial.exists() && std::fs::metadata(&partial).map(|m| m.len()).unwrap_or(0) > 0 {
r = r.resuming();
}
r
};
curl::fetch_file(&req, cfg.progress).map_err(|e| e.0)?;
let actual = sha256::hex_digest_file(&partial)
.map_err(|e| format!("could not hash {}: {e}", partial.display()))?;
if !sha256::matches(&blob.digest, &actual) {
let _ = std::fs::remove_file(&partial);
return Err(format!(
"digest mismatch for {}\n expected sha256:{}\n got sha256:{}\n \
the file has been deleted; a resumed transfer over a proxy that \
rewrote a range will do this, so a plain retry is worth one attempt",
blob.name,
sha256::normalize(&blob.digest),
actual
));
}
let len = std::fs::metadata(&partial).map(|m| m.len()).unwrap_or(0);
std::fs::rename(&partial, &final_path)
.map_err(|e| format!("could not commit {}: {e}", final_path.display()))?;
Ok((final_path, len, false))
}
pub fn pull(reference: &str, cfg: &PullConfig, spine: &Spine) -> Result<Pulled, String> {
let reference = Reference::parse(reference).map_err(|e| e.0)?;
let model = reference.display_name();
let source = registry::source_string(&reference);
if let Reference::Local { path } = &reference {
let p = PathBuf::from(path);
if !p.exists() {
return Err(format!("no file at {path}"));
}
let bytes = std::fs::metadata(&p).map(|m| m.len()).unwrap_or(0);
return Ok(Pulled {
model,
source,
weights_path: p,
bytes,
already_had_it: true,
});
}
let started = spine
.record(
&model,
&Transition::PullStarted {
source: source.clone(),
},
&[],
)
.map_err(|e| format!("could not record the pull: {e}"))?;
let fetched = registry::resolve_blobs(&reference).map_err(|e| {
let _ = spine.record(
&model,
&Transition::PullStarted {
source: format!("{source} — FAILED: {e}"),
},
std::slice::from_ref(&started),
);
e
})?;
let mut weights: Option<(PathBuf, u64)> = None;
let mut cached_all = true;
for blob in &fetched.blobs {
let (path, len, cached) = fetch_blob(blob, cfg)?;
if !cached {
cached_all = false;
}
if blob.is_weights {
weights = Some((path, len));
}
}
let (weights_path, bytes) = weights.ok_or_else(|| {
format!(
"{model} resolved with no weights layer — the registry returned \
{} blob(s), none of them the model",
fetched.blobs.len()
)
})?;
spine
.record(
&model,
&Transition::PullCompleted {
path: weights_path.display().to_string(),
size_bytes: bytes,
},
&[EventRef {
hash: started.hash,
seq: started.seq,
}],
)
.map_err(|e| format!("could not record completion: {e}"))?;
Ok(Pulled {
model,
source,
weights_path,
bytes,
already_had_it: cached_all,
})
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_blob_path_is_its_digest_so_presence_is_a_stat() {
let p = blob_path(Path::new("/b"), "sha256:ABC123");
assert_eq!(p, Path::new("/b/sha256-abc123"));
assert_eq!(p, blob_path(Path::new("/b"), "abc123"));
}
#[test]
fn a_local_reference_is_not_recorded_as_a_pull() {
let dir = std::env::temp_dir().join("hearth-pull-local");
std::fs::create_dir_all(&dir).unwrap();
let gguf = dir.join("muse.gguf");
std::fs::write(&gguf, b"weights").unwrap();
let spine = Spine::in_memory();
let out = pull(
&format!("file:{}", gguf.display()),
&PullConfig::default(),
&spine,
)
.unwrap();
assert!(out.already_had_it);
assert_eq!(out.bytes, 7);
assert!(
spine.latest(&out.model).is_none(),
"nothing was fetched, so claiming a pull would be a lie"
);
std::fs::remove_file(&gguf).ok();
}
#[test]
fn a_missing_local_file_is_refused_before_anything_is_recorded() {
let spine = Spine::in_memory();
let err = pull(
"file:/definitely/not/here.gguf",
&PullConfig::default(),
&spine,
)
.unwrap_err();
assert!(err.contains("no file at"), "{err}");
assert_eq!(spine.seq(), 0, "nothing should have been written");
}
#[test]
fn a_digest_mismatch_deletes_the_file_rather_than_caching_corruption() {
let dir = std::env::temp_dir().join("hearth-pull-mismatch");
std::fs::create_dir_all(&dir).unwrap();
let served = dir.join("served.bin");
std::fs::write(&served, b"not what we asked for").unwrap();
let blobs = dir.join("blobs");
let cfg = PullConfig {
blobs_dir: blobs.clone(),
progress: false,
verify_existing: false,
};
let blob = Blob {
name: "weights".into(),
url: format!("file://{}", served.display()),
digest: "sha256:0000000000000000000000000000000000000000000000000000000000000000"
.into(),
size_bytes: 0,
headers: vec![],
is_weights: true,
};
let err = fetch_blob(&blob, &cfg).unwrap_err();
assert!(err.contains("digest mismatch"), "{err}");
assert!(
!blob_path(&blobs, &blob.digest).exists(),
"a blob that failed verification must not be left on disk"
);
assert!(
!blob_path(&blobs, &blob.digest)
.with_extension("partial")
.exists(),
"and neither must the partial"
);
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn a_matching_digest_commits_the_file_under_its_digest() {
let dir = std::env::temp_dir().join("hearth-pull-match");
std::fs::create_dir_all(&dir).unwrap();
let body = b"the actual weights, honestly";
let served = dir.join("served.bin");
std::fs::write(&served, body).unwrap();
let blobs = dir.join("blobs");
let cfg = PullConfig {
blobs_dir: blobs.clone(),
progress: false,
verify_existing: false,
};
let blob = Blob {
name: "weights".into(),
url: format!("file://{}", served.display()),
digest: sha256::hex_digest(body),
size_bytes: body.len() as u64,
headers: vec![],
is_weights: true,
};
let (path, len, cached) = fetch_blob(&blob, &cfg).unwrap();
assert!(!cached);
assert_eq!(len, body.len() as u64);
assert_eq!(std::fs::read(&path).unwrap(), body);
let (_, _, cached2) = fetch_blob(&blob, &cfg).unwrap();
assert!(cached2, "the filename IS the digest — presence is enough");
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn a_cached_blob_of_the_wrong_size_is_refetched() {
let dir = std::env::temp_dir().join("hearth-pull-truncated");
std::fs::create_dir_all(&dir).unwrap();
let body = b"complete contents here";
let served = dir.join("served.bin");
std::fs::write(&served, body).unwrap();
let blobs = dir.join("blobs");
std::fs::create_dir_all(&blobs).unwrap();
let digest = sha256::hex_digest(body);
std::fs::write(blob_path(&blobs, &digest), b"trunc").unwrap();
let cfg = PullConfig {
blobs_dir: blobs.clone(),
progress: false,
verify_existing: false,
};
let blob = Blob {
name: "weights".into(),
url: format!("file://{}", served.display()),
digest,
size_bytes: body.len() as u64,
headers: vec![],
is_weights: true,
};
let (path, len, cached) = fetch_blob(&blob, &cfg).unwrap();
assert!(!cached, "the size did not match, so it was not trusted");
assert_eq!(len, body.len() as u64);
assert_eq!(std::fs::read(&path).unwrap(), body);
std::fs::remove_dir_all(&dir).ok();
}
}