1pub mod curl;
25pub mod registry;
26pub mod runtime;
27
28use std::path::{Path, PathBuf};
29
30use hearth_core::sha256;
31use hearth_resolve::Reference;
32use hearth_store::{EventRef, Spine, Transition};
33
34pub use registry::{Blob, Fetched};
35
36#[derive(Debug, Clone)]
38pub struct PullConfig {
39 pub blobs_dir: PathBuf,
41 pub progress: bool,
45 pub verify_existing: bool,
49}
50
51impl Default for PullConfig {
52 fn default() -> Self {
53 PullConfig {
54 blobs_dir: PathBuf::from("./blobs"),
55 progress: true,
56 verify_existing: false,
57 }
58 }
59}
60
61#[derive(Debug, Clone, PartialEq, Eq)]
63pub struct Pulled {
64 pub model: String,
65 pub source: String,
67 pub weights_path: PathBuf,
69 pub bytes: u64,
70 pub already_had_it: bool,
73}
74
75pub fn blob_path(blobs_dir: &Path, digest: &str) -> PathBuf {
78 blobs_dir.join(format!("sha256-{}", sha256::normalize(digest)))
79}
80
81pub fn fetch_blob(blob: &Blob, cfg: &PullConfig) -> Result<(PathBuf, u64, bool), String> {
88 match &blob.digest {
89 Some(digest) => fetch_blob_verified(blob, digest, cfg),
90 None => fetch_blob_self_verified(blob, cfg),
91 }
92}
93
94fn fetch_blob_verified(
97 blob: &Blob,
98 digest: &str,
99 cfg: &PullConfig,
100) -> Result<(PathBuf, u64, bool), String> {
101 let final_path = blob_path(&cfg.blobs_dir, digest);
102
103 if final_path.exists() {
104 let len = std::fs::metadata(&final_path).map(|m| m.len()).unwrap_or(0);
105 if cfg.verify_existing {
106 let actual = sha256::hex_digest_file(&final_path)
107 .map_err(|e| format!("could not read {}: {e}", final_path.display()))?;
108 if !sha256::matches(digest, &actual) {
109 eprintln!(
112 "hearth: {} does not match its own name — refetching",
113 final_path.display()
114 );
115 let _ = std::fs::remove_file(&final_path);
116 } else {
117 return Ok((final_path, len, true));
118 }
119 } else if blob.size_bytes == 0 || len == blob.size_bytes {
120 return Ok((final_path, len, true));
121 } else {
122 eprintln!(
125 "hearth: {} is {len} bytes, expected {} — refetching",
126 final_path.display(),
127 blob.size_bytes
128 );
129 let _ = std::fs::remove_file(&final_path);
130 }
131 }
132
133 let partial = final_path.with_extension("partial");
134 download_to(&partial, blob, cfg)?;
135
136 let actual = sha256::hex_digest_file(&partial)
137 .map_err(|e| format!("could not hash {}: {e}", partial.display()))?;
138 if !sha256::matches(digest, &actual) {
139 let _ = std::fs::remove_file(&partial);
142 return Err(format!(
143 "digest mismatch for {}\n expected sha256:{}\n got sha256:{}\n \
144 the file has been deleted; a resumed transfer over a proxy that \
145 rewrote a range will do this, so a plain retry is worth one attempt",
146 blob.name,
147 sha256::normalize(digest),
148 actual
149 ));
150 }
151
152 let len = std::fs::metadata(&partial).map(|m| m.len()).unwrap_or(0);
153 std::fs::rename(&partial, &final_path)
154 .map_err(|e| format!("could not commit {}: {e}", final_path.display()))?;
155 Ok((final_path, len, false))
156}
157
158fn fetch_blob_self_verified(blob: &Blob, cfg: &PullConfig) -> Result<(PathBuf, u64, bool), String> {
176 std::fs::create_dir_all(&cfg.blobs_dir).map_err(|e| e.to_string())?;
177 let safe_name = blob
178 .name
179 .chars()
180 .map(|c| {
181 if c.is_ascii_alphanumeric() || c == '.' || c == '-' {
182 c
183 } else {
184 '_'
185 }
186 })
187 .collect::<String>();
188 let partial = cfg
189 .blobs_dir
190 .join(format!("{safe_name}.unverified.partial"));
191
192 download_to(&partial, blob, cfg)?;
193
194 let actual = sha256::hex_digest_file(&partial)
195 .map_err(|e| format!("could not hash {}: {e}", partial.display()))?;
196 let final_path = blob_path(&cfg.blobs_dir, &actual);
197
198 if final_path.exists() {
199 let _ = std::fs::remove_file(&partial);
202 let len = std::fs::metadata(&final_path).map(|m| m.len()).unwrap_or(0);
203 return Ok((final_path, len, true));
204 }
205
206 let len = std::fs::metadata(&partial).map(|m| m.len()).unwrap_or(0);
207 std::fs::rename(&partial, &final_path)
208 .map_err(|e| format!("could not commit {}: {e}", final_path.display()))?;
209 eprintln!(
210 "hearth: {} has no published digest — stored as sha256:{actual}, \
211 self-consistent only (not checked against a source, because the \
212 source published none)",
213 blob.name
214 );
215 Ok((final_path, len, false))
216}
217
218fn download_to(dest: &Path, blob: &Blob, cfg: &PullConfig) -> Result<(), String> {
222 let req = {
223 let mut r = curl::Request::get(&blob.url).to_file(dest);
224 for (k, v) in &blob.headers {
225 r = r.header(k, v);
226 }
227 if dest.exists() && std::fs::metadata(dest).map(|m| m.len()).unwrap_or(0) > 0 {
230 r = r.resuming();
231 }
232 r
233 };
234 curl::fetch_file(&req, cfg.progress).map_err(|e| e.0)?;
235 Ok(())
236}
237
238pub fn pull(reference: &str, cfg: &PullConfig, spine: &Spine) -> Result<Pulled, String> {
245 let reference = Reference::parse(reference).map_err(|e| e.0)?;
246 let model = reference.display_name();
247 let source = registry::source_string(&reference);
248
249 if let Reference::Local { path } = &reference {
251 let p = PathBuf::from(path);
252 if !p.exists() {
253 return Err(format!("no file at {path}"));
254 }
255 let bytes = std::fs::metadata(&p).map(|m| m.len()).unwrap_or(0);
256 return Ok(Pulled {
257 model,
258 source,
259 weights_path: p,
260 bytes,
261 already_had_it: true,
262 });
263 }
264
265 let started = spine
266 .record(
267 &model,
268 &Transition::PullStarted {
269 source: source.clone(),
270 },
271 &[],
272 )
273 .map_err(|e| format!("could not record the pull: {e}"))?;
274
275 let fetched = registry::resolve_blobs(&reference).map_err(|e| {
276 let _ = spine.record(
279 &model,
280 &Transition::PullStarted {
281 source: format!("{source} — FAILED: {e}"),
282 },
283 std::slice::from_ref(&started),
284 );
285 e
286 })?;
287
288 let mut weights: Option<(PathBuf, u64)> = None;
289 let mut cached_all = true;
290 for blob in &fetched.blobs {
291 let (path, len, cached) = fetch_blob(blob, cfg)?;
292 if !cached {
293 cached_all = false;
294 }
295 if blob.is_weights {
296 weights = Some((path, len));
297 }
298 }
299
300 let (weights_path, bytes) = weights.ok_or_else(|| {
301 format!(
302 "{model} resolved with no weights layer — the registry returned \
303 {} blob(s), none of them the model",
304 fetched.blobs.len()
305 )
306 })?;
307
308 spine
309 .record(
310 &model,
311 &Transition::PullCompleted {
312 path: weights_path.display().to_string(),
313 size_bytes: bytes,
314 },
315 &[EventRef {
316 hash: started.hash,
317 seq: started.seq,
318 }],
319 )
320 .map_err(|e| format!("could not record completion: {e}"))?;
321
322 Ok(Pulled {
323 model,
324 source,
325 weights_path,
326 bytes,
327 already_had_it: cached_all,
328 })
329}
330
331#[cfg(test)]
332mod tests {
333 use super::*;
334
335 #[test]
336 fn a_blob_path_is_its_digest_so_presence_is_a_stat() {
337 let p = blob_path(Path::new("/b"), "sha256:ABC123");
338 assert_eq!(p, Path::new("/b/sha256-abc123"));
339 assert_eq!(p, blob_path(Path::new("/b"), "abc123"));
342 }
343
344 #[test]
345 fn a_local_reference_is_not_recorded_as_a_pull() {
346 let dir = std::env::temp_dir().join("hearth-pull-local");
347 std::fs::create_dir_all(&dir).unwrap();
348 let gguf = dir.join("muse.gguf");
349 std::fs::write(&gguf, b"weights").unwrap();
350
351 let spine = Spine::in_memory();
352 let out = pull(
353 &format!("file:{}", gguf.display()),
354 &PullConfig::default(),
355 &spine,
356 )
357 .unwrap();
358
359 assert!(out.already_had_it);
360 assert_eq!(out.bytes, 7);
361 assert!(
362 spine.latest(&out.model).is_none(),
363 "nothing was fetched, so claiming a pull would be a lie"
364 );
365 std::fs::remove_file(&gguf).ok();
366 }
367
368 #[test]
369 fn a_missing_local_file_is_refused_before_anything_is_recorded() {
370 let spine = Spine::in_memory();
371 let err = pull(
372 "file:/definitely/not/here.gguf",
373 &PullConfig::default(),
374 &spine,
375 )
376 .unwrap_err();
377 assert!(err.contains("no file at"), "{err}");
378 assert_eq!(spine.seq(), 0, "nothing should have been written");
379 }
380
381 #[test]
382 fn a_digest_mismatch_deletes_the_file_rather_than_caching_corruption() {
383 let dir = std::env::temp_dir().join("hearth-pull-mismatch");
386 std::fs::create_dir_all(&dir).unwrap();
387 let served = dir.join("served.bin");
388 std::fs::write(&served, b"not what we asked for").unwrap();
389
390 let blobs = dir.join("blobs");
391 let cfg = PullConfig {
392 blobs_dir: blobs.clone(),
393 progress: false,
394 verify_existing: false,
395 };
396 let blob = Blob {
397 name: "weights".into(),
398 url: format!("file://{}", served.display()),
399 digest: Some(
400 "sha256:0000000000000000000000000000000000000000000000000000000000000000".into(),
401 ),
402 size_bytes: 0,
403 headers: vec![],
404 is_weights: true,
405 };
406
407 let err = fetch_blob(&blob, &cfg).unwrap_err();
408 assert!(err.contains("digest mismatch"), "{err}");
409 let digest = blob.digest.as_deref().unwrap();
410 assert!(
411 !blob_path(&blobs, digest).exists(),
412 "a blob that failed verification must not be left on disk"
413 );
414 assert!(
415 !blob_path(&blobs, digest).with_extension("partial").exists(),
416 "and neither must the partial"
417 );
418 std::fs::remove_dir_all(&dir).ok();
419 }
420
421 #[test]
422 fn a_matching_digest_commits_the_file_under_its_digest() {
423 let dir = std::env::temp_dir().join("hearth-pull-match");
424 std::fs::create_dir_all(&dir).unwrap();
425 let body = b"the actual weights, honestly";
426 let served = dir.join("served.bin");
427 std::fs::write(&served, body).unwrap();
428
429 let blobs = dir.join("blobs");
430 let cfg = PullConfig {
431 blobs_dir: blobs.clone(),
432 progress: false,
433 verify_existing: false,
434 };
435 let blob = Blob {
436 name: "weights".into(),
437 url: format!("file://{}", served.display()),
438 digest: Some(sha256::hex_digest(body)),
439 size_bytes: body.len() as u64,
440 headers: vec![],
441 is_weights: true,
442 };
443
444 let (path, len, cached) = fetch_blob(&blob, &cfg).unwrap();
445 assert!(!cached);
446 assert_eq!(len, body.len() as u64);
447 assert_eq!(std::fs::read(&path).unwrap(), body);
448
449 let (_, _, cached2) = fetch_blob(&blob, &cfg).unwrap();
451 assert!(cached2, "the filename IS the digest — presence is enough");
452
453 std::fs::remove_dir_all(&dir).ok();
454 }
455
456 #[test]
457 fn a_cached_blob_of_the_wrong_size_is_refetched() {
458 let dir = std::env::temp_dir().join("hearth-pull-truncated");
459 std::fs::create_dir_all(&dir).unwrap();
460 let body = b"complete contents here";
461 let served = dir.join("served.bin");
462 std::fs::write(&served, body).unwrap();
463
464 let blobs = dir.join("blobs");
465 std::fs::create_dir_all(&blobs).unwrap();
466 let digest = sha256::hex_digest(body);
467 std::fs::write(blob_path(&blobs, &digest), b"trunc").unwrap();
470
471 let cfg = PullConfig {
472 blobs_dir: blobs.clone(),
473 progress: false,
474 verify_existing: false,
475 };
476 let blob = Blob {
477 name: "weights".into(),
478 url: format!("file://{}", served.display()),
479 digest: Some(digest),
480 size_bytes: body.len() as u64,
481 headers: vec![],
482 is_weights: true,
483 };
484 let (path, len, cached) = fetch_blob(&blob, &cfg).unwrap();
485 assert!(!cached, "the size did not match, so it was not trusted");
486 assert_eq!(len, body.len() as u64);
487 assert_eq!(std::fs::read(&path).unwrap(), body);
488 std::fs::remove_dir_all(&dir).ok();
489 }
490
491 #[test]
494 fn no_digest_still_lands_the_file_under_its_own_computed_sha256() {
495 let dir = std::env::temp_dir().join("hearth-pull-no-digest");
500 std::fs::create_dir_all(&dir).unwrap();
501 let body = b"weights nobody published a digest for";
502 let served = dir.join("served.bin");
503 std::fs::write(&served, body).unwrap();
504
505 let blobs = dir.join("blobs");
506 let cfg = PullConfig {
507 blobs_dir: blobs.clone(),
508 progress: false,
509 verify_existing: false,
510 };
511 let blob = Blob {
512 name: "model.gguf".into(),
513 url: format!("file://{}", served.display()),
514 digest: None,
515 size_bytes: 0,
516 headers: vec![],
517 is_weights: true,
518 };
519
520 let (path, len, cached) = fetch_blob(&blob, &cfg).unwrap();
521 assert!(!cached);
522 assert_eq!(len, body.len() as u64);
523 assert_eq!(std::fs::read(&path).unwrap(), body);
524
525 let expected_name = format!("sha256-{}", sha256::hex_digest(body));
526 assert_eq!(
527 path.file_name().unwrap().to_str().unwrap(),
528 expected_name,
529 "the final path must be the digest hearth computed, not a guess"
530 );
531 assert!(
532 !blobs.join("model.gguf.unverified.partial").exists(),
533 "the working file must not survive under its temporary name"
534 );
535
536 std::fs::remove_dir_all(&dir).ok();
537 }
538
539 #[test]
540 fn a_second_no_digest_pull_of_identical_content_is_recognised_as_cached() {
541 let dir = std::env::temp_dir().join("hearth-pull-no-digest-dup");
547 std::fs::create_dir_all(&dir).unwrap();
548 let body = b"identical weights served from two places";
549 let served = dir.join("served.bin");
550 std::fs::write(&served, body).unwrap();
551
552 let blobs = dir.join("blobs");
553 let cfg = PullConfig {
554 blobs_dir: blobs.clone(),
555 progress: false,
556 verify_existing: false,
557 };
558 let blob = Blob {
559 name: "model.gguf".into(),
560 url: format!("file://{}", served.display()),
561 digest: None,
562 size_bytes: 0,
563 headers: vec![],
564 is_weights: true,
565 };
566
567 let (first_path, ..) = fetch_blob(&blob, &cfg).unwrap();
568 let (second_path, _, cached) = fetch_blob(&blob, &cfg).unwrap();
569 assert!(
570 cached,
571 "identical content must be recognised, not re-stored"
572 );
573 assert_eq!(first_path, second_path);
574
575 std::fs::remove_dir_all(&dir).ok();
576 }
577
578 #[test]
579 fn an_unsafe_blob_name_does_not_escape_the_blobs_directory() {
580 let dir = std::env::temp_dir().join("hearth-pull-unsafe-name");
584 std::fs::create_dir_all(&dir).unwrap();
585 let body = b"contents";
586 let served = dir.join("served.bin");
587 std::fs::write(&served, body).unwrap();
588
589 let blobs = dir.join("blobs");
590 let cfg = PullConfig {
591 blobs_dir: blobs.clone(),
592 progress: false,
593 verify_existing: false,
594 };
595 let blob = Blob {
596 name: "../../etc/model.gguf".into(),
597 url: format!("file://{}", served.display()),
598 digest: None,
599 size_bytes: 0,
600 headers: vec![],
601 is_weights: true,
602 };
603
604 let (path, ..) = fetch_blob(&blob, &cfg).unwrap();
605 assert!(
606 path.starts_with(&blobs),
607 "the final path must stay under blobs_dir, got {}",
608 path.display()
609 );
610
611 std::fs::remove_dir_all(&dir).ok();
612 }
613}