1pub mod curl;
25pub mod registry;
26
27use std::path::{Path, PathBuf};
28
29use hearth_core::sha256;
30use hearth_resolve::Reference;
31use hearth_store::{EventRef, Spine, Transition};
32
33pub use registry::{Blob, Fetched};
34
35#[derive(Debug, Clone)]
37pub struct PullConfig {
38 pub blobs_dir: PathBuf,
40 pub progress: bool,
44 pub verify_existing: bool,
48}
49
50impl Default for PullConfig {
51 fn default() -> Self {
52 PullConfig {
53 blobs_dir: PathBuf::from("./blobs"),
54 progress: true,
55 verify_existing: false,
56 }
57 }
58}
59
60#[derive(Debug, Clone, PartialEq, Eq)]
62pub struct Pulled {
63 pub model: String,
64 pub source: String,
66 pub weights_path: PathBuf,
68 pub bytes: u64,
69 pub already_had_it: bool,
72}
73
74pub fn blob_path(blobs_dir: &Path, digest: &str) -> PathBuf {
77 blobs_dir.join(format!("sha256-{}", sha256::normalize(digest)))
78}
79
80pub fn fetch_blob(blob: &Blob, cfg: &PullConfig) -> Result<(PathBuf, u64, bool), String> {
87 let final_path = blob_path(&cfg.blobs_dir, &blob.digest);
88
89 if final_path.exists() {
90 let len = std::fs::metadata(&final_path).map(|m| m.len()).unwrap_or(0);
91 if cfg.verify_existing {
92 let actual = sha256::hex_digest_file(&final_path)
93 .map_err(|e| format!("could not read {}: {e}", final_path.display()))?;
94 if !sha256::matches(&blob.digest, &actual) {
95 eprintln!(
98 "hearth: {} does not match its own name — refetching",
99 final_path.display()
100 );
101 let _ = std::fs::remove_file(&final_path);
102 } else {
103 return Ok((final_path, len, true));
104 }
105 } else if blob.size_bytes == 0 || len == blob.size_bytes {
106 return Ok((final_path, len, true));
107 } else {
108 eprintln!(
111 "hearth: {} is {len} bytes, expected {} — refetching",
112 final_path.display(),
113 blob.size_bytes
114 );
115 let _ = std::fs::remove_file(&final_path);
116 }
117 }
118
119 let partial = final_path.with_extension("partial");
120 let req = {
121 let mut r = curl::Request::get(&blob.url).to_file(&partial);
122 for (k, v) in &blob.headers {
123 r = r.header(k, v);
124 }
125 if partial.exists() && std::fs::metadata(&partial).map(|m| m.len()).unwrap_or(0) > 0 {
128 r = r.resuming();
129 }
130 r
131 };
132
133 curl::fetch_file(&req, cfg.progress).map_err(|e| e.0)?;
134
135 let actual = sha256::hex_digest_file(&partial)
136 .map_err(|e| format!("could not hash {}: {e}", partial.display()))?;
137 if !sha256::matches(&blob.digest, &actual) {
138 let _ = std::fs::remove_file(&partial);
141 return Err(format!(
142 "digest mismatch for {}\n expected sha256:{}\n got sha256:{}\n \
143 the file has been deleted; a resumed transfer over a proxy that \
144 rewrote a range will do this, so a plain retry is worth one attempt",
145 blob.name,
146 sha256::normalize(&blob.digest),
147 actual
148 ));
149 }
150
151 let len = std::fs::metadata(&partial).map(|m| m.len()).unwrap_or(0);
152 std::fs::rename(&partial, &final_path)
153 .map_err(|e| format!("could not commit {}: {e}", final_path.display()))?;
154 Ok((final_path, len, false))
155}
156
157pub fn pull(reference: &str, cfg: &PullConfig, spine: &Spine) -> Result<Pulled, String> {
164 let reference = Reference::parse(reference).map_err(|e| e.0)?;
165 let model = reference.display_name();
166 let source = registry::source_string(&reference);
167
168 if let Reference::Local { path } = &reference {
170 let p = PathBuf::from(path);
171 if !p.exists() {
172 return Err(format!("no file at {path}"));
173 }
174 let bytes = std::fs::metadata(&p).map(|m| m.len()).unwrap_or(0);
175 return Ok(Pulled {
176 model,
177 source,
178 weights_path: p,
179 bytes,
180 already_had_it: true,
181 });
182 }
183
184 let started = spine
185 .record(
186 &model,
187 &Transition::PullStarted {
188 source: source.clone(),
189 },
190 &[],
191 )
192 .map_err(|e| format!("could not record the pull: {e}"))?;
193
194 let fetched = registry::resolve_blobs(&reference).map_err(|e| {
195 let _ = spine.record(
198 &model,
199 &Transition::PullStarted {
200 source: format!("{source} — FAILED: {e}"),
201 },
202 std::slice::from_ref(&started),
203 );
204 e
205 })?;
206
207 let mut weights: Option<(PathBuf, u64)> = None;
208 let mut cached_all = true;
209 for blob in &fetched.blobs {
210 let (path, len, cached) = fetch_blob(blob, cfg)?;
211 if !cached {
212 cached_all = false;
213 }
214 if blob.is_weights {
215 weights = Some((path, len));
216 }
217 }
218
219 let (weights_path, bytes) = weights.ok_or_else(|| {
220 format!(
221 "{model} resolved with no weights layer — the registry returned \
222 {} blob(s), none of them the model",
223 fetched.blobs.len()
224 )
225 })?;
226
227 spine
228 .record(
229 &model,
230 &Transition::PullCompleted {
231 path: weights_path.display().to_string(),
232 size_bytes: bytes,
233 },
234 &[EventRef {
235 hash: started.hash,
236 seq: started.seq,
237 }],
238 )
239 .map_err(|e| format!("could not record completion: {e}"))?;
240
241 Ok(Pulled {
242 model,
243 source,
244 weights_path,
245 bytes,
246 already_had_it: cached_all,
247 })
248}
249
250#[cfg(test)]
251mod tests {
252 use super::*;
253
254 #[test]
255 fn a_blob_path_is_its_digest_so_presence_is_a_stat() {
256 let p = blob_path(Path::new("/b"), "sha256:ABC123");
257 assert_eq!(p, Path::new("/b/sha256-abc123"));
258 assert_eq!(p, blob_path(Path::new("/b"), "abc123"));
261 }
262
263 #[test]
264 fn a_local_reference_is_not_recorded_as_a_pull() {
265 let dir = std::env::temp_dir().join("hearth-pull-local");
266 std::fs::create_dir_all(&dir).unwrap();
267 let gguf = dir.join("muse.gguf");
268 std::fs::write(&gguf, b"weights").unwrap();
269
270 let spine = Spine::in_memory();
271 let out = pull(
272 &format!("file:{}", gguf.display()),
273 &PullConfig::default(),
274 &spine,
275 )
276 .unwrap();
277
278 assert!(out.already_had_it);
279 assert_eq!(out.bytes, 7);
280 assert!(
281 spine.latest(&out.model).is_none(),
282 "nothing was fetched, so claiming a pull would be a lie"
283 );
284 std::fs::remove_file(&gguf).ok();
285 }
286
287 #[test]
288 fn a_missing_local_file_is_refused_before_anything_is_recorded() {
289 let spine = Spine::in_memory();
290 let err = pull(
291 "file:/definitely/not/here.gguf",
292 &PullConfig::default(),
293 &spine,
294 )
295 .unwrap_err();
296 assert!(err.contains("no file at"), "{err}");
297 assert_eq!(spine.seq(), 0, "nothing should have been written");
298 }
299
300 #[test]
301 fn a_digest_mismatch_deletes_the_file_rather_than_caching_corruption() {
302 let dir = std::env::temp_dir().join("hearth-pull-mismatch");
305 std::fs::create_dir_all(&dir).unwrap();
306 let served = dir.join("served.bin");
307 std::fs::write(&served, b"not what we asked for").unwrap();
308
309 let blobs = dir.join("blobs");
310 let cfg = PullConfig {
311 blobs_dir: blobs.clone(),
312 progress: false,
313 verify_existing: false,
314 };
315 let blob = Blob {
316 name: "weights".into(),
317 url: format!("file://{}", served.display()),
318 digest: "sha256:0000000000000000000000000000000000000000000000000000000000000000"
319 .into(),
320 size_bytes: 0,
321 headers: vec![],
322 is_weights: true,
323 };
324
325 let err = fetch_blob(&blob, &cfg).unwrap_err();
326 assert!(err.contains("digest mismatch"), "{err}");
327 assert!(
328 !blob_path(&blobs, &blob.digest).exists(),
329 "a blob that failed verification must not be left on disk"
330 );
331 assert!(
332 !blob_path(&blobs, &blob.digest)
333 .with_extension("partial")
334 .exists(),
335 "and neither must the partial"
336 );
337 std::fs::remove_dir_all(&dir).ok();
338 }
339
340 #[test]
341 fn a_matching_digest_commits_the_file_under_its_digest() {
342 let dir = std::env::temp_dir().join("hearth-pull-match");
343 std::fs::create_dir_all(&dir).unwrap();
344 let body = b"the actual weights, honestly";
345 let served = dir.join("served.bin");
346 std::fs::write(&served, body).unwrap();
347
348 let blobs = dir.join("blobs");
349 let cfg = PullConfig {
350 blobs_dir: blobs.clone(),
351 progress: false,
352 verify_existing: false,
353 };
354 let blob = Blob {
355 name: "weights".into(),
356 url: format!("file://{}", served.display()),
357 digest: sha256::hex_digest(body),
358 size_bytes: body.len() as u64,
359 headers: vec![],
360 is_weights: true,
361 };
362
363 let (path, len, cached) = fetch_blob(&blob, &cfg).unwrap();
364 assert!(!cached);
365 assert_eq!(len, body.len() as u64);
366 assert_eq!(std::fs::read(&path).unwrap(), body);
367
368 let (_, _, cached2) = fetch_blob(&blob, &cfg).unwrap();
370 assert!(cached2, "the filename IS the digest — presence is enough");
371
372 std::fs::remove_dir_all(&dir).ok();
373 }
374
375 #[test]
376 fn a_cached_blob_of_the_wrong_size_is_refetched() {
377 let dir = std::env::temp_dir().join("hearth-pull-truncated");
378 std::fs::create_dir_all(&dir).unwrap();
379 let body = b"complete contents here";
380 let served = dir.join("served.bin");
381 std::fs::write(&served, body).unwrap();
382
383 let blobs = dir.join("blobs");
384 std::fs::create_dir_all(&blobs).unwrap();
385 let digest = sha256::hex_digest(body);
386 std::fs::write(blob_path(&blobs, &digest), b"trunc").unwrap();
389
390 let cfg = PullConfig {
391 blobs_dir: blobs.clone(),
392 progress: false,
393 verify_existing: false,
394 };
395 let blob = Blob {
396 name: "weights".into(),
397 url: format!("file://{}", served.display()),
398 digest,
399 size_bytes: body.len() as u64,
400 headers: vec![],
401 is_weights: true,
402 };
403 let (path, len, cached) = fetch_blob(&blob, &cfg).unwrap();
404 assert!(!cached, "the size did not match, so it was not trusted");
405 assert_eq!(len, body.len() as u64);
406 assert_eq!(std::fs::read(&path).unwrap(), body);
407 std::fs::remove_dir_all(&dir).ok();
408 }
409}