Skip to main content

hearth_pull/
lib.rs

1//! `hearth pull` — get the bytes, prove they are the right bytes, and write
2//! down where they came from.
3//!
4//! This is the half of the aggregator that was missing. [`hearth_resolve`] is
5//! deliberately network-free: it turns a reference into blob URLs, a chosen
6//! quantization and a byte count, with `serde` and `serde_json` as its entire
7//! dependency list. That is what lets the VRAM planner refuse a model *before*
8//! forty gigabytes move. This crate is what moves them.
9//!
10//! Three properties, and the third is the one nobody else offers:
11//!
12//! 1. **Digest-verified.** Every blob is hashed and compared against the digest
13//!    the registry published. A download that does not match is deleted, not
14//!    kept — a corrupt GGUF fails much later as an unexplainable tokenizer
15//!    error, three layers away from the truth.
16//! 2. **Resumable.** A 40 GiB transfer that dies at 39 GiB resumes. Restarting
17//!    is not a recovery strategy, it is a way to never finish on a bad link.
18//! 3. **Recorded.** `PullStarted` and `PullCompleted` land in the spine with
19//!    the resolved source, so months later "where did this file come from" is a
20//!    query — `hearth why <model>` — instead of an archaeology project. Those
21//!    two transitions already existed in [`hearth_store`]; the history was
22//!    ready before the downloader was.
23
24pub mod curl;
25pub mod registry;
26pub mod runtime;
27
28use std::path::{Path, PathBuf};
29
30use hearth_core::sha256;
31use hearth_resolve::Reference;
32use hearth_store::{EventRef, Spine, Transition};
33
34pub use registry::{Blob, Fetched};
35
36/// Where blobs live, and how loud to be about it.
37#[derive(Debug, Clone)]
38pub struct PullConfig {
39    /// Content-addressed blob directory, e.g. `~/.hearth/blobs`.
40    pub blobs_dir: PathBuf,
41    /// Draw curl's progress meter. A silent twenty-minute pause is
42    /// indistinguishable from a hang, and a downloader that looks hung gets
43    /// killed halfway.
44    pub progress: bool,
45    /// Re-verify a blob that is already on disk instead of trusting its name.
46    /// Off by default: re-hashing 40 GiB on every start is a real cost, and the
47    /// filename IS the digest. On when you have reason to distrust the disk.
48    pub verify_existing: bool,
49}
50
51impl Default for PullConfig {
52    fn default() -> Self {
53        PullConfig {
54            blobs_dir: PathBuf::from("./blobs"),
55            progress: true,
56            verify_existing: false,
57        }
58    }
59}
60
61/// What a pull did.
62#[derive(Debug, Clone, PartialEq, Eq)]
63pub struct Pulled {
64    pub model: String,
65    /// Resolved origin, as recorded in the spine.
66    pub source: String,
67    /// The weights blob on disk.
68    pub weights_path: PathBuf,
69    pub bytes: u64,
70    /// True when everything was already present and verified — so a second
71    /// `pull` of the same model is honest about having done nothing.
72    pub already_had_it: bool,
73}
74
75/// Content-addressed path for a digest. The filename IS the digest, which is
76/// what makes "do we already have this" a `stat` rather than a re-hash.
77pub fn blob_path(blobs_dir: &Path, digest: &str) -> PathBuf {
78    blobs_dir.join(format!("sha256-{}", sha256::normalize(digest)))
79}
80
81/// Fetch one blob to its content-addressed home, verifying the digest.
82///
83/// Downloads to a `.partial` sibling and renames only after the digest
84/// matches. Without that, an interrupted transfer leaves a file whose *name*
85/// claims a digest its *contents* do not have — and every later run trusts the
86/// name. The rename is the commit.
87pub fn fetch_blob(blob: &Blob, cfg: &PullConfig) -> Result<(PathBuf, u64, bool), String> {
88    match &blob.digest {
89        Some(digest) => fetch_blob_verified(blob, digest, cfg),
90        None => fetch_blob_self_verified(blob, cfg),
91    }
92}
93
94/// The original path: a digest is already known, so the final home is known
95/// before a single byte moves, and every byte is checked against it.
96fn fetch_blob_verified(
97    blob: &Blob,
98    digest: &str,
99    cfg: &PullConfig,
100) -> Result<(PathBuf, u64, bool), String> {
101    let final_path = blob_path(&cfg.blobs_dir, digest);
102
103    if final_path.exists() {
104        let len = std::fs::metadata(&final_path).map(|m| m.len()).unwrap_or(0);
105        if cfg.verify_existing {
106            let actual = sha256::hex_digest_file(&final_path)
107                .map_err(|e| format!("could not read {}: {e}", final_path.display()))?;
108            if !sha256::matches(digest, &actual) {
109                // On disk under a digest it does not have. Say so loudly and
110                // re-fetch rather than quietly serving the wrong weights.
111                eprintln!(
112                    "hearth: {} does not match its own name — refetching",
113                    final_path.display()
114                );
115                let _ = std::fs::remove_file(&final_path);
116            } else {
117                return Ok((final_path, len, true));
118            }
119        } else if blob.size_bytes == 0 || len == blob.size_bytes {
120            return Ok((final_path, len, true));
121        } else {
122            // Right name, wrong size: a previous run was interrupted after the
123            // rename, or something truncated it. Not trustworthy.
124            eprintln!(
125                "hearth: {} is {len} bytes, expected {} — refetching",
126                final_path.display(),
127                blob.size_bytes
128            );
129            let _ = std::fs::remove_file(&final_path);
130        }
131    }
132
133    let partial = final_path.with_extension("partial");
134    download_to(&partial, blob, cfg)?;
135
136    let actual = sha256::hex_digest_file(&partial)
137        .map_err(|e| format!("could not hash {}: {e}", partial.display()))?;
138    if !sha256::matches(digest, &actual) {
139        // Delete it. Keeping a blob that failed verification is how a corrupt
140        // download becomes a permanent, cached, corrupt download.
141        let _ = std::fs::remove_file(&partial);
142        return Err(format!(
143            "digest mismatch for {}\n  expected sha256:{}\n  got      sha256:{}\n  \
144             the file has been deleted; a resumed transfer over a proxy that \
145             rewrote a range will do this, so a plain retry is worth one attempt",
146            blob.name,
147            sha256::normalize(digest),
148            actual
149        ));
150    }
151
152    let len = std::fs::metadata(&partial).map(|m| m.len()).unwrap_or(0);
153    std::fs::rename(&partial, &final_path)
154        .map_err(|e| format!("could not commit {}: {e}", final_path.display()))?;
155    Ok((final_path, len, false))
156}
157
158/// No digest was published to check against — a bare URL, or a HuggingFace
159/// file the tree listing did not mark as LFS.
160///
161/// The final home cannot be known before downloading, because the digest
162/// that names it is the thing being discovered. So this downloads to a
163/// TEMPORARY name (never a content-addressed one — a partial file must never
164/// sit under the name its finished content will claim), hashes what actually
165/// arrived, and only then computes the path a verified blob would have had
166/// from the start. If that path already exists, this exact content was
167/// already fetched under some other name, and the new download is discarded
168/// as a duplicate rather than kept as a second copy of the same bytes.
169///
170/// This is self-consistency, not verification: the file matches its own
171/// name by construction, which is trivially true of every file and proves
172/// nothing about whether the content is what the operator meant to fetch.
173/// Callers that need real verification pin a digest — the URL scheme's
174/// `#sha256:HEX` fragment exists for exactly that.
175fn fetch_blob_self_verified(blob: &Blob, cfg: &PullConfig) -> Result<(PathBuf, u64, bool), String> {
176    std::fs::create_dir_all(&cfg.blobs_dir).map_err(|e| e.to_string())?;
177    let safe_name = blob
178        .name
179        .chars()
180        .map(|c| {
181            if c.is_ascii_alphanumeric() || c == '.' || c == '-' {
182                c
183            } else {
184                '_'
185            }
186        })
187        .collect::<String>();
188    let partial = cfg
189        .blobs_dir
190        .join(format!("{safe_name}.unverified.partial"));
191
192    download_to(&partial, blob, cfg)?;
193
194    let actual = sha256::hex_digest_file(&partial)
195        .map_err(|e| format!("could not hash {}: {e}", partial.display()))?;
196    let final_path = blob_path(&cfg.blobs_dir, &actual);
197
198    if final_path.exists() {
199        // Already have this exact content under its rightful name — this
200        // download was redundant, not wrong.
201        let _ = std::fs::remove_file(&partial);
202        let len = std::fs::metadata(&final_path).map(|m| m.len()).unwrap_or(0);
203        return Ok((final_path, len, true));
204    }
205
206    let len = std::fs::metadata(&partial).map(|m| m.len()).unwrap_or(0);
207    std::fs::rename(&partial, &final_path)
208        .map_err(|e| format!("could not commit {}: {e}", final_path.display()))?;
209    eprintln!(
210        "hearth: {} has no published digest — stored as sha256:{actual}, \
211         self-consistent only (not checked against a source, because the \
212         source published none)",
213        blob.name
214    );
215    Ok((final_path, len, false))
216}
217
218/// Download one blob to `dest`, resuming a partial transfer if one is
219/// already there. Shared by both verification paths — they differ in what
220/// happens to the bytes afterward, not in how the bytes arrive.
221fn download_to(dest: &Path, blob: &Blob, cfg: &PullConfig) -> Result<(), String> {
222    let req = {
223        let mut r = curl::Request::get(&blob.url).to_file(dest);
224        for (k, v) in &blob.headers {
225            r = r.header(k, v);
226        }
227        // Resume only if there is something to resume; `-C -` on a missing file
228        // is fine, but on a zero-byte one some servers answer 416.
229        if dest.exists() && std::fs::metadata(dest).map(|m| m.len()).unwrap_or(0) > 0 {
230            r = r.resuming();
231        }
232        r
233    };
234    curl::fetch_file(&req, cfg.progress).map_err(|e| e.0)?;
235    Ok(())
236}
237
238/// Pull a model reference, recording the whole thing in the spine.
239///
240/// The recording is not decoration. `PullStarted` goes in *before* any bytes
241/// move, so a pull that dies halfway leaves evidence that it was attempted and
242/// from where — the alternative is a half-empty blob directory and no idea
243/// which model it belongs to.
244pub fn pull(reference: &str, cfg: &PullConfig, spine: &Spine) -> Result<Pulled, String> {
245    let reference = Reference::parse(reference).map_err(|e| e.0)?;
246    let model = reference.display_name();
247    let source = registry::source_string(&reference);
248
249    // A local file is already here. Recording a pull for it would be a lie.
250    if let Reference::Local { path } = &reference {
251        let p = PathBuf::from(path);
252        if !p.exists() {
253            return Err(format!("no file at {path}"));
254        }
255        let bytes = std::fs::metadata(&p).map(|m| m.len()).unwrap_or(0);
256        return Ok(Pulled {
257            model,
258            source,
259            weights_path: p,
260            bytes,
261            already_had_it: true,
262        });
263    }
264
265    let started = spine
266        .record(
267            &model,
268            &Transition::PullStarted {
269                source: source.clone(),
270            },
271            &[],
272        )
273        .map_err(|e| format!("could not record the pull: {e}"))?;
274
275    let fetched = registry::resolve_blobs(&reference).map_err(|e| {
276        // The failure is worth recording too — "we tried and the registry said
277        // no" is a different fact from "nobody ever tried".
278        let _ = spine.record(
279            &model,
280            &Transition::PullStarted {
281                source: format!("{source} — FAILED: {e}"),
282            },
283            std::slice::from_ref(&started),
284        );
285        e
286    })?;
287
288    let mut weights: Option<(PathBuf, u64)> = None;
289    let mut cached_all = true;
290    for blob in &fetched.blobs {
291        let (path, len, cached) = fetch_blob(blob, cfg)?;
292        if !cached {
293            cached_all = false;
294        }
295        if blob.is_weights {
296            weights = Some((path, len));
297        }
298    }
299
300    let (weights_path, bytes) = weights.ok_or_else(|| {
301        format!(
302            "{model} resolved with no weights layer — the registry returned \
303             {} blob(s), none of them the model",
304            fetched.blobs.len()
305        )
306    })?;
307
308    spine
309        .record(
310            &model,
311            &Transition::PullCompleted {
312                path: weights_path.display().to_string(),
313                size_bytes: bytes,
314            },
315            &[EventRef {
316                hash: started.hash,
317                seq: started.seq,
318            }],
319        )
320        .map_err(|e| format!("could not record completion: {e}"))?;
321
322    Ok(Pulled {
323        model,
324        source,
325        weights_path,
326        bytes,
327        already_had_it: cached_all,
328    })
329}
330
331#[cfg(test)]
332mod tests {
333    use super::*;
334
335    #[test]
336    fn a_blob_path_is_its_digest_so_presence_is_a_stat() {
337        let p = blob_path(Path::new("/b"), "sha256:ABC123");
338        assert_eq!(p, Path::new("/b/sha256-abc123"));
339        // Both spellings of the same digest land on the same file, or a cache
340        // would miss every time the registry changed its formatting.
341        assert_eq!(p, blob_path(Path::new("/b"), "abc123"));
342    }
343
344    #[test]
345    fn a_local_reference_is_not_recorded_as_a_pull() {
346        let dir = std::env::temp_dir().join("hearth-pull-local");
347        std::fs::create_dir_all(&dir).unwrap();
348        let gguf = dir.join("muse.gguf");
349        std::fs::write(&gguf, b"weights").unwrap();
350
351        let spine = Spine::in_memory();
352        let out = pull(
353            &format!("file:{}", gguf.display()),
354            &PullConfig::default(),
355            &spine,
356        )
357        .unwrap();
358
359        assert!(out.already_had_it);
360        assert_eq!(out.bytes, 7);
361        assert!(
362            spine.latest(&out.model).is_none(),
363            "nothing was fetched, so claiming a pull would be a lie"
364        );
365        std::fs::remove_file(&gguf).ok();
366    }
367
368    #[test]
369    fn a_missing_local_file_is_refused_before_anything_is_recorded() {
370        let spine = Spine::in_memory();
371        let err = pull(
372            "file:/definitely/not/here.gguf",
373            &PullConfig::default(),
374            &spine,
375        )
376        .unwrap_err();
377        assert!(err.contains("no file at"), "{err}");
378        assert_eq!(spine.seq(), 0, "nothing should have been written");
379    }
380
381    #[test]
382    fn a_digest_mismatch_deletes_the_file_rather_than_caching_corruption() {
383        // The blob is served by a local file:// URL whose contents do not match
384        // the digest we claim. curl handles file:// so this needs no network.
385        let dir = std::env::temp_dir().join("hearth-pull-mismatch");
386        std::fs::create_dir_all(&dir).unwrap();
387        let served = dir.join("served.bin");
388        std::fs::write(&served, b"not what we asked for").unwrap();
389
390        let blobs = dir.join("blobs");
391        let cfg = PullConfig {
392            blobs_dir: blobs.clone(),
393            progress: false,
394            verify_existing: false,
395        };
396        let blob = Blob {
397            name: "weights".into(),
398            url: format!("file://{}", served.display()),
399            digest: Some(
400                "sha256:0000000000000000000000000000000000000000000000000000000000000000".into(),
401            ),
402            size_bytes: 0,
403            headers: vec![],
404            is_weights: true,
405        };
406
407        let err = fetch_blob(&blob, &cfg).unwrap_err();
408        assert!(err.contains("digest mismatch"), "{err}");
409        let digest = blob.digest.as_deref().unwrap();
410        assert!(
411            !blob_path(&blobs, digest).exists(),
412            "a blob that failed verification must not be left on disk"
413        );
414        assert!(
415            !blob_path(&blobs, digest).with_extension("partial").exists(),
416            "and neither must the partial"
417        );
418        std::fs::remove_dir_all(&dir).ok();
419    }
420
421    #[test]
422    fn a_matching_digest_commits_the_file_under_its_digest() {
423        let dir = std::env::temp_dir().join("hearth-pull-match");
424        std::fs::create_dir_all(&dir).unwrap();
425        let body = b"the actual weights, honestly";
426        let served = dir.join("served.bin");
427        std::fs::write(&served, body).unwrap();
428
429        let blobs = dir.join("blobs");
430        let cfg = PullConfig {
431            blobs_dir: blobs.clone(),
432            progress: false,
433            verify_existing: false,
434        };
435        let blob = Blob {
436            name: "weights".into(),
437            url: format!("file://{}", served.display()),
438            digest: Some(sha256::hex_digest(body)),
439            size_bytes: body.len() as u64,
440            headers: vec![],
441            is_weights: true,
442        };
443
444        let (path, len, cached) = fetch_blob(&blob, &cfg).unwrap();
445        assert!(!cached);
446        assert_eq!(len, body.len() as u64);
447        assert_eq!(std::fs::read(&path).unwrap(), body);
448
449        // Second call is a stat, not a download.
450        let (_, _, cached2) = fetch_blob(&blob, &cfg).unwrap();
451        assert!(cached2, "the filename IS the digest — presence is enough");
452
453        std::fs::remove_dir_all(&dir).ok();
454    }
455
456    #[test]
457    fn a_cached_blob_of_the_wrong_size_is_refetched() {
458        let dir = std::env::temp_dir().join("hearth-pull-truncated");
459        std::fs::create_dir_all(&dir).unwrap();
460        let body = b"complete contents here";
461        let served = dir.join("served.bin");
462        std::fs::write(&served, body).unwrap();
463
464        let blobs = dir.join("blobs");
465        std::fs::create_dir_all(&blobs).unwrap();
466        let digest = sha256::hex_digest(body);
467        // A truncated file already sitting under the right name — what an
468        // interrupted run leaves behind if it renames before verifying.
469        std::fs::write(blob_path(&blobs, &digest), b"trunc").unwrap();
470
471        let cfg = PullConfig {
472            blobs_dir: blobs.clone(),
473            progress: false,
474            verify_existing: false,
475        };
476        let blob = Blob {
477            name: "weights".into(),
478            url: format!("file://{}", served.display()),
479            digest: Some(digest),
480            size_bytes: body.len() as u64,
481            headers: vec![],
482            is_weights: true,
483        };
484        let (path, len, cached) = fetch_blob(&blob, &cfg).unwrap();
485        assert!(!cached, "the size did not match, so it was not trusted");
486        assert_eq!(len, body.len() as u64);
487        assert_eq!(std::fs::read(&path).unwrap(), body);
488        std::fs::remove_dir_all(&dir).ok();
489    }
490
491    // ---- fetch_blob_self_verified: no published digest to check against ----
492
493    #[test]
494    fn no_digest_still_lands_the_file_under_its_own_computed_sha256() {
495        // A bare URL pull: nothing published a digest, so `blob.digest` is
496        // `None`. The file must still end up content-addressed — hearth
497        // computes the digest itself rather than trusting a caller-supplied
498        // filename for the final path.
499        let dir = std::env::temp_dir().join("hearth-pull-no-digest");
500        std::fs::create_dir_all(&dir).unwrap();
501        let body = b"weights nobody published a digest for";
502        let served = dir.join("served.bin");
503        std::fs::write(&served, body).unwrap();
504
505        let blobs = dir.join("blobs");
506        let cfg = PullConfig {
507            blobs_dir: blobs.clone(),
508            progress: false,
509            verify_existing: false,
510        };
511        let blob = Blob {
512            name: "model.gguf".into(),
513            url: format!("file://{}", served.display()),
514            digest: None,
515            size_bytes: 0,
516            headers: vec![],
517            is_weights: true,
518        };
519
520        let (path, len, cached) = fetch_blob(&blob, &cfg).unwrap();
521        assert!(!cached);
522        assert_eq!(len, body.len() as u64);
523        assert_eq!(std::fs::read(&path).unwrap(), body);
524
525        let expected_name = format!("sha256-{}", sha256::hex_digest(body));
526        assert_eq!(
527            path.file_name().unwrap().to_str().unwrap(),
528            expected_name,
529            "the final path must be the digest hearth computed, not a guess"
530        );
531        assert!(
532            !blobs.join("model.gguf.unverified.partial").exists(),
533            "the working file must not survive under its temporary name"
534        );
535
536        std::fs::remove_dir_all(&dir).ok();
537    }
538
539    #[test]
540    fn a_second_no_digest_pull_of_identical_content_is_recognised_as_cached() {
541        // Two different URLs (or the same URL fetched twice under different
542        // callers) that happen to serve byte-identical content must land on
543        // the SAME blob, not two copies — that is the entire point of
544        // content addressing, and the no-digest path has to preserve it even
545        // though it does not know the address until after downloading.
546        let dir = std::env::temp_dir().join("hearth-pull-no-digest-dup");
547        std::fs::create_dir_all(&dir).unwrap();
548        let body = b"identical weights served from two places";
549        let served = dir.join("served.bin");
550        std::fs::write(&served, body).unwrap();
551
552        let blobs = dir.join("blobs");
553        let cfg = PullConfig {
554            blobs_dir: blobs.clone(),
555            progress: false,
556            verify_existing: false,
557        };
558        let blob = Blob {
559            name: "model.gguf".into(),
560            url: format!("file://{}", served.display()),
561            digest: None,
562            size_bytes: 0,
563            headers: vec![],
564            is_weights: true,
565        };
566
567        let (first_path, ..) = fetch_blob(&blob, &cfg).unwrap();
568        let (second_path, _, cached) = fetch_blob(&blob, &cfg).unwrap();
569        assert!(
570            cached,
571            "identical content must be recognised, not re-stored"
572        );
573        assert_eq!(first_path, second_path);
574
575        std::fs::remove_dir_all(&dir).ok();
576    }
577
578    #[test]
579    fn an_unsafe_blob_name_does_not_escape_the_blobs_directory() {
580        // blob.name comes from a URL path or a registry listing — untrusted
581        // input. A name containing "../" must not let the temporary working
582        // file land outside blobs_dir.
583        let dir = std::env::temp_dir().join("hearth-pull-unsafe-name");
584        std::fs::create_dir_all(&dir).unwrap();
585        let body = b"contents";
586        let served = dir.join("served.bin");
587        std::fs::write(&served, body).unwrap();
588
589        let blobs = dir.join("blobs");
590        let cfg = PullConfig {
591            blobs_dir: blobs.clone(),
592            progress: false,
593            verify_existing: false,
594        };
595        let blob = Blob {
596            name: "../../etc/model.gguf".into(),
597            url: format!("file://{}", served.display()),
598            digest: None,
599            size_bytes: 0,
600            headers: vec![],
601            is_weights: true,
602        };
603
604        let (path, ..) = fetch_blob(&blob, &cfg).unwrap();
605        assert!(
606            path.starts_with(&blobs),
607            "the final path must stay under blobs_dir, got {}",
608            path.display()
609        );
610
611        std::fs::remove_dir_all(&dir).ok();
612    }
613}