1pub mod curl;
25pub mod registry;
26pub mod runtime;
27
28use std::path::{Path, PathBuf};
29
30use hearth_core::sha256;
31use hearth_resolve::Reference;
32use hearth_store::{EventRef, Spine, Transition};
33
34pub use registry::{Blob, Fetched};
35
36#[derive(Debug, Clone)]
38pub struct PullConfig {
39 pub blobs_dir: PathBuf,
41 pub progress: bool,
45 pub verify_existing: bool,
49}
50
51impl Default for PullConfig {
52 fn default() -> Self {
53 PullConfig {
54 blobs_dir: PathBuf::from("./blobs"),
55 progress: true,
56 verify_existing: false,
57 }
58 }
59}
60
61#[derive(Debug, Clone, PartialEq, Eq)]
63pub struct Pulled {
64 pub model: String,
65 pub source: String,
67 pub weights_path: PathBuf,
69 pub bytes: u64,
70 pub already_had_it: bool,
73}
74
75pub fn blob_path(blobs_dir: &Path, digest: &str) -> PathBuf {
78 blobs_dir.join(format!("sha256-{}", sha256::normalize(digest)))
79}
80
81pub fn human_bytes(n: u64) -> String {
83 const GIB: f64 = 1024.0 * 1024.0 * 1024.0;
84 const MIB: f64 = 1024.0 * 1024.0;
85 let f = n as f64;
86 if f >= GIB {
87 format!("{:.2} GiB", f / GIB)
88 } else if f >= MIB {
89 format!("{:.1} MiB", f / MIB)
90 } else {
91 format!("{n} B")
92 }
93}
94
95fn digest_file_loud(path: &Path, label: &str, progress: bool) -> Result<String, String> {
106 use std::io::Write;
107 let started = std::time::Instant::now();
108 let mut last = std::time::Instant::now();
109 let digest = sha256::hex_digest_file_with_progress(path, |done, total| {
110 if !progress {
111 return;
112 }
113 if last.elapsed() < std::time::Duration::from_millis(500) && done < total {
114 return;
115 }
116 last = std::time::Instant::now();
117 let secs = started.elapsed().as_secs_f64().max(0.001);
118 let rate = (done as f64 / secs) as u64;
119 let pct = if total > 0 {
120 format!("{:5.1}%", done as f64 / total as f64 * 100.0)
121 } else {
122 " ? ".to_string()
123 };
124 let _ = write!(
126 std::io::stderr(),
127 "\rhearth: verifying {label} {pct} ({} / {}) {}/s ",
128 human_bytes(done),
129 human_bytes(total),
130 human_bytes(rate)
131 );
132 let _ = std::io::stderr().flush();
133 })
134 .map_err(|e| format!("could not hash {}: {e}", path.display()))?;
135 if progress {
136 let secs = started.elapsed().as_secs_f64();
137 let len = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0);
138 let rate = if secs > 0.0 {
139 (len as f64 / secs) as u64
140 } else {
141 0
142 };
143 let _ = writeln!(
144 std::io::stderr(),
145 "\rhearth: verified {label} — {} in {:.1}s ({}/s) ",
146 human_bytes(len),
147 secs,
148 human_bytes(rate)
149 );
150 }
151 Ok(digest)
152}
153
154pub fn fetch_blob(blob: &Blob, cfg: &PullConfig) -> Result<(PathBuf, u64, bool), String> {
161 match &blob.digest {
162 Some(digest) => fetch_blob_verified(blob, digest, cfg),
163 None => fetch_blob_self_verified(blob, cfg),
164 }
165}
166
167fn fetch_blob_verified(
170 blob: &Blob,
171 digest: &str,
172 cfg: &PullConfig,
173) -> Result<(PathBuf, u64, bool), String> {
174 let final_path = blob_path(&cfg.blobs_dir, digest);
175
176 if final_path.exists() {
177 let len = std::fs::metadata(&final_path).map(|m| m.len()).unwrap_or(0);
178 if cfg.verify_existing {
179 let actual = digest_file_loud(&final_path, &blob.name, cfg.progress)?;
180 if !sha256::matches(digest, &actual) {
181 eprintln!(
184 "hearth: {} does not match its own name — refetching",
185 final_path.display()
186 );
187 let _ = std::fs::remove_file(&final_path);
188 } else {
189 return Ok((final_path, len, true));
190 }
191 } else if blob.size_bytes == 0 || len == blob.size_bytes {
192 return Ok((final_path, len, true));
193 } else {
194 eprintln!(
197 "hearth: {} is {len} bytes, expected {} — refetching",
198 final_path.display(),
199 blob.size_bytes
200 );
201 let _ = std::fs::remove_file(&final_path);
202 }
203 }
204
205 let partial = final_path.with_extension("partial");
206 download_to(&partial, blob, cfg)?;
207
208 let actual = digest_file_loud(&partial, &blob.name, cfg.progress)?;
209 if !sha256::matches(digest, &actual) {
210 let _ = std::fs::remove_file(&partial);
213 return Err(format!(
214 "digest mismatch for {}\n expected sha256:{}\n got sha256:{}\n \
215 the file has been deleted; a resumed transfer over a proxy that \
216 rewrote a range will do this, so a plain retry is worth one attempt",
217 blob.name,
218 sha256::normalize(digest),
219 actual
220 ));
221 }
222
223 let len = std::fs::metadata(&partial).map(|m| m.len()).unwrap_or(0);
224 std::fs::rename(&partial, &final_path)
225 .map_err(|e| format!("could not commit {}: {e}", final_path.display()))?;
226 Ok((final_path, len, false))
227}
228
229fn fetch_blob_self_verified(blob: &Blob, cfg: &PullConfig) -> Result<(PathBuf, u64, bool), String> {
247 std::fs::create_dir_all(&cfg.blobs_dir).map_err(|e| e.to_string())?;
248 let safe_name = blob
249 .name
250 .chars()
251 .map(|c| {
252 if c.is_ascii_alphanumeric() || c == '.' || c == '-' {
253 c
254 } else {
255 '_'
256 }
257 })
258 .collect::<String>();
259 let partial = cfg
260 .blobs_dir
261 .join(format!("{safe_name}.unverified.partial"));
262
263 download_to(&partial, blob, cfg)?;
264
265 let actual = digest_file_loud(&partial, &blob.name, cfg.progress)?;
266 let final_path = blob_path(&cfg.blobs_dir, &actual);
267
268 if final_path.exists() {
269 let _ = std::fs::remove_file(&partial);
272 let len = std::fs::metadata(&final_path).map(|m| m.len()).unwrap_or(0);
273 return Ok((final_path, len, true));
274 }
275
276 let len = std::fs::metadata(&partial).map(|m| m.len()).unwrap_or(0);
277 std::fs::rename(&partial, &final_path)
278 .map_err(|e| format!("could not commit {}: {e}", final_path.display()))?;
279 eprintln!(
280 "hearth: {} has no published digest — stored as sha256:{actual}, \
281 self-consistent only (not checked against a source, because the \
282 source published none)",
283 blob.name
284 );
285 Ok((final_path, len, false))
286}
287
288fn download_to(dest: &Path, blob: &Blob, cfg: &PullConfig) -> Result<(), String> {
292 let req = {
293 let mut r = curl::Request::get(&blob.url).to_file(dest);
294 for (k, v) in &blob.headers {
295 r = r.header(k, v);
296 }
297 if dest.exists() && std::fs::metadata(dest).map(|m| m.len()).unwrap_or(0) > 0 {
300 r = r.resuming();
301 }
302 r
303 };
304 curl::fetch_file(&req, cfg.progress).map_err(|e| e.0)?;
305 Ok(())
306}
307
308pub fn pull(reference: &str, cfg: &PullConfig, spine: &Spine) -> Result<Pulled, String> {
315 let reference = Reference::parse(reference).map_err(|e| e.0)?;
316 let model = reference.display_name();
317 let source = registry::source_string(&reference);
318
319 if let Reference::Local { path } = &reference {
321 let p = PathBuf::from(path);
322 if !p.exists() {
323 return Err(format!("no file at {path}"));
324 }
325 let bytes = std::fs::metadata(&p).map(|m| m.len()).unwrap_or(0);
326 return Ok(Pulled {
327 model,
328 source,
329 weights_path: p,
330 bytes,
331 already_had_it: true,
332 });
333 }
334
335 let started = spine
336 .record(
337 &model,
338 &Transition::PullStarted {
339 source: source.clone(),
340 },
341 &[],
342 )
343 .map_err(|e| format!("could not record the pull: {e}"))?;
344
345 if cfg.progress {
349 eprintln!("hearth: resolving {source} …");
350 }
351
352 let fetched = registry::resolve_blobs(&reference).map_err(|e| {
353 let _ = spine.record(
356 &model,
357 &Transition::PullStarted {
358 source: format!("{source} — FAILED: {e}"),
359 },
360 std::slice::from_ref(&started),
361 );
362 e
363 })?;
364
365 let n = fetched.blobs.len();
370 if cfg.progress {
371 let total: u64 = fetched.blobs.iter().map(|b| b.size_bytes).sum();
372 eprintln!(
373 "hearth: {n} file(s), {} to fetch — then a sha256 verify of each",
374 human_bytes(total)
375 );
376 }
377
378 let mut weights: Option<(PathBuf, u64)> = None;
379 let mut cached_all = true;
380 for (i, blob) in fetched.blobs.iter().enumerate() {
381 if cfg.progress {
382 eprintln!(
383 "hearth: [{}/{n}] {} ({})",
384 i + 1,
385 blob.name,
386 if blob.size_bytes > 0 {
387 human_bytes(blob.size_bytes)
388 } else {
389 "size unknown".to_string()
390 }
391 );
392 }
393 let (path, len, cached) = fetch_blob(blob, cfg)?;
394 if cfg.progress && cached {
395 eprintln!("hearth: [{}/{n}] already on disk, skipped", i + 1);
396 }
397 if !cached {
398 cached_all = false;
399 }
400 if blob.is_weights {
401 weights = Some((path, len));
402 }
403 }
404
405 let (weights_path, bytes) = weights.ok_or_else(|| {
406 format!(
407 "{model} resolved with no weights layer — the registry returned \
408 {} blob(s), none of them the model",
409 fetched.blobs.len()
410 )
411 })?;
412
413 spine
414 .record(
415 &model,
416 &Transition::PullCompleted {
417 path: weights_path.display().to_string(),
418 size_bytes: bytes,
419 },
420 &[EventRef {
421 hash: started.hash,
422 seq: started.seq,
423 }],
424 )
425 .map_err(|e| format!("could not record completion: {e}"))?;
426
427 Ok(Pulled {
428 model,
429 source,
430 weights_path,
431 bytes,
432 already_had_it: cached_all,
433 })
434}
435
436#[cfg(test)]
437mod tests {
438 use super::*;
439
440 #[test]
441 fn a_blob_path_is_its_digest_so_presence_is_a_stat() {
442 let p = blob_path(Path::new("/b"), "sha256:ABC123");
443 assert_eq!(p, Path::new("/b/sha256-abc123"));
444 assert_eq!(p, blob_path(Path::new("/b"), "abc123"));
447 }
448
449 #[test]
450 fn a_local_reference_is_not_recorded_as_a_pull() {
451 let dir = std::env::temp_dir().join("hearth-pull-local");
452 std::fs::create_dir_all(&dir).unwrap();
453 let gguf = dir.join("muse.gguf");
454 std::fs::write(&gguf, b"weights").unwrap();
455
456 let spine = Spine::in_memory();
457 let out = pull(
458 &format!("file:{}", gguf.display()),
459 &PullConfig::default(),
460 &spine,
461 )
462 .unwrap();
463
464 assert!(out.already_had_it);
465 assert_eq!(out.bytes, 7);
466 assert!(
467 spine.latest(&out.model).is_none(),
468 "nothing was fetched, so claiming a pull would be a lie"
469 );
470 std::fs::remove_file(&gguf).ok();
471 }
472
473 #[test]
474 fn a_missing_local_file_is_refused_before_anything_is_recorded() {
475 let spine = Spine::in_memory();
476 let err = pull(
477 "file:/definitely/not/here.gguf",
478 &PullConfig::default(),
479 &spine,
480 )
481 .unwrap_err();
482 assert!(err.contains("no file at"), "{err}");
483 assert_eq!(spine.seq(), 0, "nothing should have been written");
484 }
485
486 #[test]
487 fn a_digest_mismatch_deletes_the_file_rather_than_caching_corruption() {
488 let dir = std::env::temp_dir().join("hearth-pull-mismatch");
491 std::fs::create_dir_all(&dir).unwrap();
492 let served = dir.join("served.bin");
493 std::fs::write(&served, b"not what we asked for").unwrap();
494
495 let blobs = dir.join("blobs");
496 let cfg = PullConfig {
497 blobs_dir: blobs.clone(),
498 progress: false,
499 verify_existing: false,
500 };
501 let blob = Blob {
502 name: "weights".into(),
503 url: format!("file://{}", served.display()),
504 digest: Some(
505 "sha256:0000000000000000000000000000000000000000000000000000000000000000".into(),
506 ),
507 size_bytes: 0,
508 headers: vec![],
509 is_weights: true,
510 };
511
512 let err = fetch_blob(&blob, &cfg).unwrap_err();
513 assert!(err.contains("digest mismatch"), "{err}");
514 let digest = blob.digest.as_deref().unwrap();
515 assert!(
516 !blob_path(&blobs, digest).exists(),
517 "a blob that failed verification must not be left on disk"
518 );
519 assert!(
520 !blob_path(&blobs, digest).with_extension("partial").exists(),
521 "and neither must the partial"
522 );
523 std::fs::remove_dir_all(&dir).ok();
524 }
525
526 #[test]
527 fn a_matching_digest_commits_the_file_under_its_digest() {
528 let dir = std::env::temp_dir().join("hearth-pull-match");
529 std::fs::create_dir_all(&dir).unwrap();
530 let body = b"the actual weights, honestly";
531 let served = dir.join("served.bin");
532 std::fs::write(&served, body).unwrap();
533
534 let blobs = dir.join("blobs");
535 let cfg = PullConfig {
536 blobs_dir: blobs.clone(),
537 progress: false,
538 verify_existing: false,
539 };
540 let blob = Blob {
541 name: "weights".into(),
542 url: format!("file://{}", served.display()),
543 digest: Some(sha256::hex_digest(body)),
544 size_bytes: body.len() as u64,
545 headers: vec![],
546 is_weights: true,
547 };
548
549 let (path, len, cached) = fetch_blob(&blob, &cfg).unwrap();
550 assert!(!cached);
551 assert_eq!(len, body.len() as u64);
552 assert_eq!(std::fs::read(&path).unwrap(), body);
553
554 let (_, _, cached2) = fetch_blob(&blob, &cfg).unwrap();
556 assert!(cached2, "the filename IS the digest — presence is enough");
557
558 std::fs::remove_dir_all(&dir).ok();
559 }
560
561 #[test]
562 fn a_cached_blob_of_the_wrong_size_is_refetched() {
563 let dir = std::env::temp_dir().join("hearth-pull-truncated");
564 std::fs::create_dir_all(&dir).unwrap();
565 let body = b"complete contents here";
566 let served = dir.join("served.bin");
567 std::fs::write(&served, body).unwrap();
568
569 let blobs = dir.join("blobs");
570 std::fs::create_dir_all(&blobs).unwrap();
571 let digest = sha256::hex_digest(body);
572 std::fs::write(blob_path(&blobs, &digest), b"trunc").unwrap();
575
576 let cfg = PullConfig {
577 blobs_dir: blobs.clone(),
578 progress: false,
579 verify_existing: false,
580 };
581 let blob = Blob {
582 name: "weights".into(),
583 url: format!("file://{}", served.display()),
584 digest: Some(digest),
585 size_bytes: body.len() as u64,
586 headers: vec![],
587 is_weights: true,
588 };
589 let (path, len, cached) = fetch_blob(&blob, &cfg).unwrap();
590 assert!(!cached, "the size did not match, so it was not trusted");
591 assert_eq!(len, body.len() as u64);
592 assert_eq!(std::fs::read(&path).unwrap(), body);
593 std::fs::remove_dir_all(&dir).ok();
594 }
595
596 #[test]
599 fn no_digest_still_lands_the_file_under_its_own_computed_sha256() {
600 let dir = std::env::temp_dir().join("hearth-pull-no-digest");
605 std::fs::create_dir_all(&dir).unwrap();
606 let body = b"weights nobody published a digest for";
607 let served = dir.join("served.bin");
608 std::fs::write(&served, body).unwrap();
609
610 let blobs = dir.join("blobs");
611 let cfg = PullConfig {
612 blobs_dir: blobs.clone(),
613 progress: false,
614 verify_existing: false,
615 };
616 let blob = Blob {
617 name: "model.gguf".into(),
618 url: format!("file://{}", served.display()),
619 digest: None,
620 size_bytes: 0,
621 headers: vec![],
622 is_weights: true,
623 };
624
625 let (path, len, cached) = fetch_blob(&blob, &cfg).unwrap();
626 assert!(!cached);
627 assert_eq!(len, body.len() as u64);
628 assert_eq!(std::fs::read(&path).unwrap(), body);
629
630 let expected_name = format!("sha256-{}", sha256::hex_digest(body));
631 assert_eq!(
632 path.file_name().unwrap().to_str().unwrap(),
633 expected_name,
634 "the final path must be the digest hearth computed, not a guess"
635 );
636 assert!(
637 !blobs.join("model.gguf.unverified.partial").exists(),
638 "the working file must not survive under its temporary name"
639 );
640
641 std::fs::remove_dir_all(&dir).ok();
642 }
643
644 #[test]
645 fn a_second_no_digest_pull_of_identical_content_is_recognised_as_cached() {
646 let dir = std::env::temp_dir().join("hearth-pull-no-digest-dup");
652 std::fs::create_dir_all(&dir).unwrap();
653 let body = b"identical weights served from two places";
654 let served = dir.join("served.bin");
655 std::fs::write(&served, body).unwrap();
656
657 let blobs = dir.join("blobs");
658 let cfg = PullConfig {
659 blobs_dir: blobs.clone(),
660 progress: false,
661 verify_existing: false,
662 };
663 let blob = Blob {
664 name: "model.gguf".into(),
665 url: format!("file://{}", served.display()),
666 digest: None,
667 size_bytes: 0,
668 headers: vec![],
669 is_weights: true,
670 };
671
672 let (first_path, ..) = fetch_blob(&blob, &cfg).unwrap();
673 let (second_path, _, cached) = fetch_blob(&blob, &cfg).unwrap();
674 assert!(
675 cached,
676 "identical content must be recognised, not re-stored"
677 );
678 assert_eq!(first_path, second_path);
679
680 std::fs::remove_dir_all(&dir).ok();
681 }
682
683 #[test]
684 fn an_unsafe_blob_name_does_not_escape_the_blobs_directory() {
685 let dir = std::env::temp_dir().join("hearth-pull-unsafe-name");
689 std::fs::create_dir_all(&dir).unwrap();
690 let body = b"contents";
691 let served = dir.join("served.bin");
692 std::fs::write(&served, body).unwrap();
693
694 let blobs = dir.join("blobs");
695 let cfg = PullConfig {
696 blobs_dir: blobs.clone(),
697 progress: false,
698 verify_existing: false,
699 };
700 let blob = Blob {
701 name: "../../etc/model.gguf".into(),
702 url: format!("file://{}", served.display()),
703 digest: None,
704 size_bytes: 0,
705 headers: vec![],
706 is_weights: true,
707 };
708
709 let (path, ..) = fetch_blob(&blob, &cfg).unwrap();
710 assert!(
711 path.starts_with(&blobs),
712 "the final path must stay under blobs_dir, got {}",
713 path.display()
714 );
715
716 std::fs::remove_dir_all(&dir).ok();
717 }
718}