hf_fetch_model/cache.rs
1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! `HuggingFace` cache directory resolution, model family scanning, disk usage,
4//! and integrity verification.
5//!
6//! [`hf_cache_dir()`] locates the local HF cache. [`list_cached_families()`]
7//! scans downloaded models and groups them by `model_type`.
8//! [`cache_summary()`] provides per-repo size totals,
9//! [`cache_repo_usage()`] returns per-file disk usage for a single repo, and
10//! [`verify_cache()`] re-checks `SHA256` digests of cached files against
11//! `HuggingFace` LFS metadata.
12
13use std::collections::BTreeMap;
14use std::path::{Path, PathBuf};
15
16use serde::{Deserialize, Serialize};
17
18use crate::error::FetchError;
19
20/// Filename of the per-repo `hf-fm` snapshot sidecar.
21///
22/// Lives at `{cache_root}/models--{org}--{name}/.hf-fm-snapshot.json`.
23/// Written by `download` (recording the active `--preset` / `--filter` /
24/// `--exclude`) and consumed by `status` (to distinguish files deliberately
25/// skipped via the preset's glob list from files that are genuinely missing).
26pub const SNAPSHOT_FILENAME: &str = ".hf-fm-snapshot.json";
27
28/// Schema version of the on-disk [`Snapshot`] file. Bumped on incompatible changes.
29pub const SNAPSHOT_VERSION: u32 = 1;
30
31/// On-disk record of the arguments that produced a cached repository.
32///
33/// Persisted by `hf-fm download` as a small JSON file at the repository's
34/// cache root, alongside `refs/`, `blobs/`, and `snapshots/`. Read back by
35/// `hf-fm status` so files that don't match the recorded preset can be
36/// reported as [`FileStatus::Excluded`] instead of [`FileStatus::Missing`].
37#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
38pub struct Snapshot {
39 /// Schema version. Equals [`SNAPSHOT_VERSION`] for newly-written files.
40 pub version: u32,
41 /// Git revision at download time (resolved commit SHA or branch name).
42 pub revision: String,
43 /// The `--preset` value used at download time, if any. One of
44 /// `"safetensors"`, `"gguf"`, `"npz"`, `"pth"`, `"config-only"`.
45 pub preset: Option<String>,
46 /// `--filter` glob patterns used at download time. Reserved for a later
47 /// patch that adds `status --filter`; not yet consumed by `status`.
48 pub filter: Vec<String>,
49 /// `--exclude` glob patterns used at download time. Reserved for a later
50 /// patch that adds `status --exclude`; not yet consumed by `status`.
51 pub exclude: Vec<String>,
52}
53
54/// Returns the absolute path of the [`SNAPSHOT_FILENAME`] sidecar for a given
55/// repository cache directory.
56#[must_use]
57pub fn snapshot_path(repo_dir: &Path) -> PathBuf {
58 repo_dir.join(SNAPSHOT_FILENAME)
59}
60
61/// Reads the per-repo [`Snapshot`] sidecar if it exists.
62///
63/// A missing sidecar is not an error — older caches (downloaded before this
64/// feature) simply return `Ok(None)`.
65///
66/// # Errors
67///
68/// Returns [`FetchError::Io`] if the file exists but
69/// cannot be read.
70/// Returns [`FetchError::InvalidArgument`]
71/// if the file is present but its JSON cannot be parsed (e.g. corruption or
72/// a future schema version this binary cannot understand).
73pub fn read_snapshot(repo_dir: &Path) -> Result<Option<Snapshot>, FetchError> {
74 let path = snapshot_path(repo_dir);
75 let bytes = match std::fs::read(&path) {
76 Ok(b) => b,
77 Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None),
78 Err(e) => return Err(FetchError::Io { path, source: e }),
79 };
80 let snapshot: Snapshot = serde_json::from_slice(&bytes).map_err(|e| {
81 FetchError::InvalidArgument(format!("failed to parse snapshot {}: {e}", path.display()))
82 })?;
83 Ok(Some(snapshot))
84}
85
86/// Writes the [`Snapshot`] sidecar for a repository, atomically (write to a
87/// `.tmp` sibling, then rename).
88///
89/// Overwrites any previously-written sidecar — the design is intentionally
90/// last-download-wins.
91///
92/// # Errors
93///
94/// Returns [`FetchError::Io`] on any filesystem
95/// failure (parent directory missing, no write permission, rename across
96/// filesystems, etc.).
97pub fn write_snapshot(repo_dir: &Path, snapshot: &Snapshot) -> Result<(), FetchError> {
98 let path = snapshot_path(repo_dir);
99 let tmp = path.with_extension("json.tmp");
100 let bytes = serde_json::to_vec_pretty(snapshot)
101 .map_err(|e| FetchError::InvalidArgument(format!("failed to serialize snapshot: {e}")))?;
102 std::fs::write(&tmp, bytes).map_err(|e| FetchError::Io {
103 path: tmp.clone(),
104 source: e,
105 })?;
106 std::fs::rename(&tmp, &path).map_err(|e| FetchError::Io {
107 path: path.clone(),
108 source: e,
109 })?;
110 Ok(())
111}
112
113/// Reconstructs a repo ID from a `models--org--name` directory name.
114///
115/// Returns `None` if the directory name does not start with `models--`.
116fn repo_id_from_folder_name(dir_name: &str) -> Option<String> {
117 let repo_part = dir_name.strip_prefix("models--")?;
118
119 // Reconstruct repo_id: replace first "--" with "/".
120 let repo_id = match repo_part.find("--") {
121 Some(pos) => {
122 let (org, name_with_sep) = repo_part.split_at(pos);
123 let name = name_with_sep.get(2..).unwrap_or_default();
124 format!("{org}/{name}")
125 }
126 None => repo_part.to_string(),
127 };
128
129 Some(repo_id)
130}
131
132/// Returns the `HuggingFace` Hub cache directory.
133///
134/// Resolution order:
135/// 1. `HF_HOME` environment variable + `/hub`
136/// 2. `~/.cache/huggingface/hub/` (via [`dirs::home_dir()`])
137///
138/// # Errors
139///
140/// Returns [`FetchError::Io`] if the home directory cannot be determined.
141pub fn hf_cache_dir() -> Result<PathBuf, FetchError> {
142 if let Ok(home) = std::env::var("HF_HOME") {
143 let mut path = PathBuf::from(home);
144 path.push("hub");
145 return Ok(path);
146 }
147
148 let home = dirs::home_dir().ok_or_else(|| FetchError::Io {
149 path: PathBuf::from("~"),
150 source: std::io::Error::new(std::io::ErrorKind::NotFound, "home directory not found"),
151 })?;
152
153 let mut path = home;
154 path.push(".cache");
155 path.push("huggingface");
156 path.push("hub");
157 Ok(path)
158}
159
160/// One repository inside a cached family, with optional quantization label.
161///
162/// Returned by [`list_cached_families`] so callers can render a quant column
163/// alongside the repo ID without re-reading every snapshot's `config.json`
164/// in a second pass.
165#[derive(Debug, Clone, PartialEq, Eq)]
166pub struct FamilyEntry {
167 /// The repository identifier (e.g., `"meta-llama/Llama-3.2-1B"`).
168 pub repo_id: String,
169 /// Quantization method as reported by `quantization_config.quant_method`
170 /// in the cached `config.json`. Falls back to `"gguf"` when any cached
171 /// file in the newest snapshot directory has a `.gguf` extension.
172 /// `None` for full-precision repos.
173 pub quant_method: Option<String>,
174}
175
176/// Scans the local HF cache for downloaded models and groups them by `model_type`.
177///
178/// Looks for `config.json` files inside model snapshot directories:
179/// `<cache>/models--<org>--<name>/snapshots/*/config.json`
180///
181/// Returns a map from `model_type` (e.g., `"llama"`) to a sorted list of
182/// [`FamilyEntry`] values, each pairing a repo ID with its quantization
183/// label (if any).
184///
185/// Models without a `model_type` field in their `config.json` are skipped.
186///
187/// # Errors
188///
189/// Returns [`FetchError::Io`] if the cache directory cannot be read.
190pub fn list_cached_families() -> Result<BTreeMap<String, Vec<FamilyEntry>>, FetchError> {
191 let cache_dir = hf_cache_dir()?;
192
193 if !cache_dir.exists() {
194 return Ok(BTreeMap::new());
195 }
196
197 let entries = std::fs::read_dir(&cache_dir).map_err(|e| FetchError::Io {
198 path: cache_dir.clone(),
199 source: e,
200 })?;
201
202 let mut families: BTreeMap<String, Vec<FamilyEntry>> = BTreeMap::new();
203
204 for entry in entries {
205 let Ok(entry) = entry else { continue };
206
207 let dir_name = entry.file_name();
208 // BORROW: explicit .to_string_lossy() for OsString → str conversion
209 let dir_str = dir_name.to_string_lossy();
210
211 let Some(repo_id) = repo_id_from_folder_name(&dir_str) else {
212 continue;
213 };
214
215 // Find the newest snapshot's config.json
216 let snapshots_dir = crate::cache_layout::snapshots_dir(&entry.path());
217 if !snapshots_dir.exists() {
218 continue;
219 }
220
221 if let Some((model_type, quant_method)) = find_family_info_in_snapshots(&snapshots_dir) {
222 families.entry(model_type).or_default().push(FamilyEntry {
223 repo_id,
224 quant_method,
225 });
226 }
227 }
228
229 // Sort repo lists within each family for stable output
230 for entries in families.values_mut() {
231 entries.sort_by(|a, b| a.repo_id.cmp(&b.repo_id));
232 }
233
234 Ok(families)
235}
236
237/// Searches snapshot directories for a `config.json` containing `model_type`,
238/// and reports an accompanying quantization label when one can be inferred.
239///
240/// Returns the first `(model_type, quant_method)` pair found. The quant
241/// label comes from `quantization_config.quant_method` in `config.json`,
242/// or falls back to `Some("gguf".to_owned())` when any sibling file in the
243/// same snapshot directory has a `.gguf` extension. Returns `None` if no
244/// snapshot yields a parseable `model_type`.
245fn find_family_info_in_snapshots(
246 snapshots_dir: &std::path::Path,
247) -> Option<(String, Option<String>)> {
248 let snapshots = std::fs::read_dir(snapshots_dir).ok()?;
249
250 for snap_entry in snapshots {
251 let Ok(snap_entry) = snap_entry else { continue };
252 let snap_path = snap_entry.path();
253 let config_path = snap_path.join("config.json");
254
255 if !config_path.exists() {
256 continue;
257 }
258
259 if let Some(model_type) = extract_model_type(&config_path) {
260 let quant_method = extract_quant_method(&config_path)
261 .or_else(|| snapshot_has_gguf(&snap_path).then(|| "gguf".to_owned())); // BORROW: explicit .to_owned()
262 return Some((model_type, quant_method));
263 }
264 }
265
266 None
267}
268
269/// Reads a `config.json` file and extracts the `model_type` field.
270fn extract_model_type(config_path: &std::path::Path) -> Option<String> {
271 let contents = std::fs::read_to_string(config_path).ok()?;
272 // BORROW: explicit .as_str() instead of Deref coercion
273 let value: serde_json::Value = serde_json::from_str(contents.as_str()).ok()?;
274 // BORROW: explicit .as_str() on serde_json Value
275 value.get("model_type")?.as_str().map(String::from)
276}
277
278/// Reads a `config.json` file and extracts the transformers-standard
279/// `quantization_config.quant_method` field, if present.
280///
281/// Returns `None` when the file is unreadable, malformed, or contains no
282/// quantization config (the repo is treated as full-precision in that case;
283/// the GGUF filename fallback is applied separately by the caller).
284fn extract_quant_method(config_path: &std::path::Path) -> Option<String> {
285 let contents = std::fs::read_to_string(config_path).ok()?;
286 // BORROW: explicit .as_str() instead of Deref coercion
287 let value: serde_json::Value = serde_json::from_str(contents.as_str()).ok()?;
288 // BORROW: explicit .as_str() on serde_json Value
289 value
290 .get("quantization_config")?
291 .get("quant_method")?
292 .as_str()
293 .map(String::from)
294}
295
296/// Returns `true` if any file in the snapshot directory has a `.gguf` extension.
297///
298/// Used as a fallback quant label when `config.json` carries no
299/// `quantization_config` (GGUF repos typically lack a transformers-style
300/// `config.json` `quantization_config` block). Extension comparison is
301/// case-insensitive via `OsStr::eq_ignore_ascii_case`.
302fn snapshot_has_gguf(snapshot_dir: &std::path::Path) -> bool {
303 let Ok(entries) = std::fs::read_dir(snapshot_dir) else {
304 return false;
305 };
306 for entry in entries.flatten() {
307 let path = entry.path();
308 if path
309 .extension()
310 .is_some_and(|ext| ext.eq_ignore_ascii_case("gguf"))
311 {
312 return true;
313 }
314 }
315 false
316}
317
318/// Status of a single file in the cache.
319#[derive(Debug, Clone)]
320#[non_exhaustive]
321pub enum FileStatus {
322 /// File is fully downloaded (local size matches expected size, or no expected size known).
323 Complete {
324 /// Local file size in bytes.
325 local_size: u64,
326 },
327 /// File exists but is smaller than expected (interrupted download),
328 /// or the file's own `blobs/<sha256>.chunked.part` temp blob exists
329 /// (keyed on the file's LFS `sha256` — per-file attribution).
330 Partial {
331 /// Local file size in bytes.
332 local_size: u64,
333 /// Expected file size in bytes.
334 expected_size: u64,
335 },
336 /// File is not present in the cache.
337 Missing {
338 /// Expected file size in bytes (0 if unknown).
339 expected_size: u64,
340 },
341 /// File is on the Hub but was deliberately not requested at download time
342 /// (or is now filtered out by `status --preset <P>`). Distinguished from
343 /// [`FileStatus::Missing`] so the user does not chase a "fix" for an
344 /// intentional skip.
345 Excluded {
346 /// Expected file size in bytes (0 if unknown).
347 expected_size: u64,
348 },
349}
350
351/// Cache status report for a repository.
352#[derive(Debug, Clone)]
353pub struct RepoStatus {
354 /// The repository identifier.
355 pub repo_id: String,
356 /// The resolved commit hash (if available).
357 pub commit_hash: Option<String>,
358 /// The cache directory for this repo.
359 pub cache_path: PathBuf,
360 /// Per-file status, sorted by filename.
361 pub files: Vec<(String, FileStatus)>,
362}
363
364impl RepoStatus {
365 /// Number of fully downloaded files.
366 #[must_use]
367 pub fn complete_count(&self) -> usize {
368 self.files
369 .iter()
370 .filter(|(_, s)| matches!(s, FileStatus::Complete { .. }))
371 .count()
372 }
373
374 /// Number of partially downloaded files.
375 #[must_use]
376 pub fn partial_count(&self) -> usize {
377 self.files
378 .iter()
379 .filter(|(_, s)| matches!(s, FileStatus::Partial { .. }))
380 .count()
381 }
382
383 /// Number of missing files.
384 #[must_use]
385 pub fn missing_count(&self) -> usize {
386 self.files
387 .iter()
388 .filter(|(_, s)| matches!(s, FileStatus::Missing { .. }))
389 .count()
390 }
391
392 /// Number of files deliberately excluded by the active preset / filter.
393 ///
394 /// Always `0` when `status` is invoked without a preset (whether from
395 /// CLI or sidecar) — the `Excluded` variant requires an active filter.
396 #[must_use]
397 pub fn excluded_count(&self) -> usize {
398 self.files
399 .iter()
400 .filter(|(_, s)| matches!(s, FileStatus::Excluded { .. }))
401 .count()
402 }
403}
404
405/// Inspects the local cache for a repository and compares against the remote file list.
406///
407/// When `preset_globs` is `Some`, files that do **not** match any of the
408/// supplied glob patterns and are absent locally are classified as
409/// [`FileStatus::Excluded`] instead of [`FileStatus::Missing`]. Files that
410/// **do** match the preset and are absent are still [`FileStatus::Missing`].
411/// Files that are present locally classify as `Complete` / `Partial`
412/// regardless of whether they match (the user has them — by what route is
413/// not status's concern).
414///
415/// # Arguments
416///
417/// * `repo_id` — The repository identifier (e.g., `"RWKV/RWKV7-Goose-World3-1.5B-HF"`).
418/// * `token` — Optional authentication token.
419/// * `revision` — Optional revision (defaults to `"main"`).
420/// * `preset_globs` — Optional include-glob list (typically returned by
421/// [`crate::config::preset_globs`]). When supplied, governs the
422/// `Excluded` distinction. When `None`, no `Excluded` entries are produced.
423///
424/// # Notes
425///
426/// Partial-download detection is per-file: a file absent from the snapshot
427/// directory is reported [`FileStatus::Partial`] only when its own
428/// chunked-download temp blob (`blobs/<sha256>.chunked.part`, keyed on the
429/// file's LFS `sha256`) exists on disk, and `local_size` is that temp
430/// blob's current byte count. Non-LFS files carry no `sha256`, so an
431/// interrupted non-chunked download reports [`FileStatus::Missing`] until
432/// it finalizes.
433///
434/// # Errors
435///
436/// Returns [`FetchError::Http`] if the API request fails.
437/// Returns [`FetchError::Io`] if the cache directory cannot be read.
438/// Returns [`FetchError::InvalidPattern`]
439/// if any of the supplied `preset_globs` patterns fails to compile.
440pub async fn repo_status(
441 repo_id: &str,
442 token: Option<&str>,
443 revision: Option<&str>,
444 preset_globs: Option<&[&str]>,
445) -> Result<RepoStatus, FetchError> {
446 let revision = revision.unwrap_or("main");
447 let cache_dir = hf_cache_dir()?;
448 let repo_dir = crate::cache_layout::repo_dir(&cache_dir, repo_id);
449
450 // Read commit hash from refs file if available.
451 let commit_hash = read_ref(&repo_dir, revision);
452
453 // Fetch remote file list with sizes.
454 let client = crate::chunked::build_client(token)?;
455 let remote_files =
456 crate::repo::list_repo_files_with_metadata(repo_id, token, Some(revision), &client).await?;
457
458 // Determine snapshot directory.
459 // BORROW: explicit .as_deref() for Option<String> → Option<&str>
460 let snapshot_dir = commit_hash
461 .as_deref()
462 .map(|hash| crate::cache_layout::snapshot_dir(&repo_dir, hash));
463
464 // Compile preset globs once, outside the per-file loop. `compile_glob_patterns`
465 // already returns `Ok(None)` for empty slices, so an empty `Some(&[])`
466 // collapses to the "no filter" case below.
467 let preset_globset: Option<globset::GlobSet> = if let Some(patterns) = preset_globs {
468 // BORROW: explicit .to_string() — compile_glob_patterns expects &[String]
469 let owned: Vec<String> = patterns.iter().map(|s| (*s).to_string()).collect();
470 crate::compile_glob_patterns(&owned)?
471 } else {
472 None
473 };
474
475 // Cross-reference remote files against local state.
476 let mut files: Vec<(String, FileStatus)> = Vec::with_capacity(remote_files.len());
477
478 for remote in &remote_files {
479 let expected_size = remote.size.unwrap_or(0);
480
481 let local_path = snapshot_dir
482 .as_ref()
483 // BORROW: explicit .as_str() for path construction
484 .map(|dir| dir.join(remote.filename.as_str()));
485
486 // Size of the snapshot copy when it exists; `None` when the file is
487 // absent (or no snapshot directory is known for this revision).
488 let snapshot_size = local_path.as_ref().and_then(|path| {
489 path.exists()
490 .then(|| std::fs::metadata(path).map_or(0, |m| m.len()))
491 });
492
493 let status = if let Some(local_size) = snapshot_size {
494 if expected_size > 0 && local_size < expected_size {
495 FileStatus::Partial {
496 local_size,
497 expected_size,
498 }
499 } else {
500 FileStatus::Complete { local_size }
501 }
502 } else if let Some(part_size) =
503 // BORROW: explicit .as_deref() for Option<String> → Option<&str>
504 partial_chunk_size(&repo_dir, remote.sha256.as_deref())
505 {
506 // This file's own chunked-download temp blob is on disk: attribute
507 // exactly its byte count to this file (keyed on the LFS sha256).
508 FileStatus::Partial {
509 local_size: part_size,
510 expected_size,
511 }
512 } else if preset_globset
513 .as_ref()
514 // BORROW: explicit .as_str() — globset matches against &str
515 .is_some_and(|gs| !gs.is_match(remote.filename.as_str()))
516 {
517 // File is absent AND the active preset deliberately excludes it.
518 FileStatus::Excluded { expected_size }
519 } else {
520 FileStatus::Missing { expected_size }
521 };
522
523 // BORROW: explicit .clone() for owned String
524 files.push((remote.filename.clone(), status));
525 }
526
527 files.sort_by(|(a, _), (b, _)| a.cmp(b));
528
529 // BORROW: explicit .to_owned() for &str → owned String field
530 Ok(RepoStatus {
531 repo_id: repo_id.to_owned(),
532 commit_hash,
533 cache_path: repo_dir,
534 files,
535 })
536}
537
538/// Summary of a single cached model (local-only, no API calls).
539#[derive(Debug, Clone)]
540pub struct CachedModelSummary {
541 /// The repository identifier (e.g., `"RWKV/RWKV7-Goose-World3-1.5B-HF"`).
542 pub repo_id: String,
543 /// Number of files in the snapshot directory.
544 pub file_count: usize,
545 /// Total size on disk in bytes.
546 pub total_size: u64,
547 /// Whether there are incomplete `.chunked.part` temp files.
548 pub has_partial: bool,
549 /// Most recent modification time among files in the snapshot directory.
550 ///
551 /// `None` if no files were found or all metadata reads failed.
552 pub last_modified: Option<std::time::SystemTime>,
553}
554
555/// Scans the entire HF cache and returns a summary for each cached model.
556///
557/// This is a local-only operation (no API calls). It lists all `models--*`
558/// directories and counts files + sizes in each snapshot.
559///
560/// # Errors
561///
562/// Returns [`FetchError::Io`] if the cache directory cannot be read.
563pub fn cache_summary() -> Result<Vec<CachedModelSummary>, FetchError> {
564 let cache_dir = hf_cache_dir()?;
565
566 if !cache_dir.exists() {
567 return Ok(Vec::new());
568 }
569
570 let entries = std::fs::read_dir(&cache_dir).map_err(|e| FetchError::Io {
571 path: cache_dir.clone(),
572 source: e,
573 })?;
574
575 let mut summaries: Vec<CachedModelSummary> = Vec::new();
576
577 for entry in entries {
578 let Ok(entry) = entry else { continue };
579 let dir_name = entry.file_name();
580 // BORROW: explicit .to_string_lossy() for OsString → str conversion
581 let dir_str = dir_name.to_string_lossy();
582
583 let Some(repo_id) = repo_id_from_folder_name(&dir_str) else {
584 continue;
585 };
586
587 let repo_dir = entry.path();
588
589 // Count files and total size in snapshots.
590 let (file_count, total_size, last_modified) = count_snapshot_files(&repo_dir);
591
592 // Check for partial downloads.
593 let has_partial = find_partial_blob_size(&crate::cache_layout::blobs_dir(&repo_dir)) > 0;
594
595 summaries.push(CachedModelSummary {
596 repo_id,
597 file_count,
598 total_size,
599 has_partial,
600 last_modified,
601 });
602 }
603
604 summaries.sort_by(|a, b| a.repo_id.cmp(&b.repo_id));
605
606 Ok(summaries)
607}
608
609/// Returns the file count and total size for a single cached repo.
610///
611/// Avoids scanning the entire cache when only one repo's metrics are needed
612/// (e.g., for the `cache delete` preview).
613///
614/// # Errors
615///
616/// Returns [`FetchError::Io`] if the cache directory cannot be determined.
617pub fn repo_disk_usage(repo_id: &str) -> Result<(usize, u64), FetchError> {
618 let cache_dir = hf_cache_dir()?;
619 let repo_dir = crate::cache_layout::repo_dir(&cache_dir, repo_id);
620 let (file_count, total_size, _) = count_snapshot_files(&repo_dir);
621 Ok((file_count, total_size))
622}
623
624/// Checks whether a single cached repo has `.chunked.part` temp files.
625///
626/// Avoids scanning the entire cache when only one repo's partial status
627/// is needed (e.g., for the `du <REPO>` partial-download hint).
628///
629/// # Errors
630///
631/// Returns [`FetchError::Io`] if the cache directory cannot be determined.
632pub fn repo_has_partial(repo_id: &str) -> Result<bool, FetchError> {
633 let cache_dir = hf_cache_dir()?;
634 let repo_dir = crate::cache_layout::repo_dir(&cache_dir, repo_id);
635 let blobs_dir = crate::cache_layout::blobs_dir(&repo_dir);
636 Ok(find_partial_blob_size(&blobs_dir) > 0)
637}
638
639/// Counts files, total size, and most recent modification time across all
640/// snapshot directories for a repo.
641fn count_snapshot_files(repo_dir: &Path) -> (usize, u64, Option<std::time::SystemTime>) {
642 let snapshots_dir = crate::cache_layout::snapshots_dir(repo_dir);
643 let Ok(snapshots) = std::fs::read_dir(snapshots_dir) else {
644 return (0, 0, None);
645 };
646
647 let mut file_count: usize = 0;
648 let mut total_size: u64 = 0;
649 let mut latest: Option<std::time::SystemTime> = None;
650
651 for snap_entry in snapshots {
652 let Ok(snap_entry) = snap_entry else { continue };
653 let snap_path = snap_entry.path();
654 if !snap_path.is_dir() {
655 continue;
656 }
657 count_files_recursive(&snap_path, &mut file_count, &mut total_size, &mut latest);
658 }
659
660 (file_count, total_size, latest)
661}
662
663/// Recursively counts files, accumulates sizes, and tracks the most recent
664/// modification time in a directory.
665fn count_files_recursive(
666 dir: &Path,
667 count: &mut usize,
668 total: &mut u64,
669 latest: &mut Option<std::time::SystemTime>,
670) {
671 let Ok(entries) = std::fs::read_dir(dir) else {
672 return;
673 };
674
675 for entry in entries {
676 let Ok(entry) = entry else { continue };
677 let path = entry.path();
678 if path.is_dir() {
679 count_files_recursive(&path, count, total, latest);
680 } else if let Ok(meta) = entry.metadata() {
681 *count += 1;
682 *total += meta.len();
683 if let Ok(modified) = meta.modified() {
684 match *latest {
685 Some(current) if modified <= current => {} // EXPLICIT: current mtime is more recent, keep it
686 _ => *latest = Some(modified),
687 }
688 }
689 } else {
690 *count += 1;
691 }
692 }
693}
694
695/// Reads the commit hash from a refs file, if it exists.
696///
697/// Looks for `<repo_dir>/refs/<revision>` and returns the trimmed contents
698/// (a commit hash) or `None` if the file does not exist or is empty.
699#[must_use]
700pub fn read_ref(repo_dir: &Path, revision: &str) -> Option<String> {
701 let ref_path = crate::cache_layout::ref_path(repo_dir, revision);
702 std::fs::read_to_string(ref_path)
703 .ok()
704 // BORROW: explicit .to_owned() to convert trimmed &str → owned String
705 .map(|s| s.trim().to_owned())
706 .filter(|s| !s.is_empty())
707}
708
709/// Returns the truly-downloaded byte count of a file's own chunked-download
710/// temp blob.
711///
712/// Looks up `{repo_dir}/blobs/<sha256>.chunked.part` — the chunked download
713/// path names its temp blob after the file's etag, which for LFS files is
714/// the content's `SHA256` (see [`crate::cache_layout::temp_blob_path`]) —
715/// so a specific file's partial is addressable without scanning the blobs
716/// directory. Returns `None` when the file has no known `sha256` (non-LFS
717/// file) or no temp blob exists on disk.
718///
719/// The chunked download preallocates the temp blob at its full size, so the
720/// blob's file length reads as `total_size` from the first byte onward.
721/// True progress lives in the `.chunked.part.state` resume sidecar's
722/// per-chunk `completed` offsets; this prefers their sum and falls back to
723/// the blob's file length when the sidecar is absent or unparseable
724/// (pre-v0.9.8 leftovers).
725fn partial_chunk_size(repo_dir: &Path, sha256: Option<&str>) -> Option<u64> {
726 let sha = sha256?;
727 let part_path = crate::cache_layout::temp_blob_path(repo_dir, sha);
728 let part_len = std::fs::metadata(part_path)
729 .ok()
730 .filter(std::fs::Metadata::is_file)
731 .map(|m| m.len())?;
732
733 let state_path = crate::cache_layout::temp_state_path(repo_dir, sha);
734 let sidecar_sum = std::fs::read_to_string(state_path)
735 .ok()
736 .and_then(|text| {
737 // BORROW: explicit .as_str() instead of Deref coercion
738 serde_json::from_str::<crate::chunked_state::ChunkedState>(text.as_str()).ok()
739 })
740 .map(|state| state.chunks.iter().map(|c| c.completed).sum::<u64>());
741
742 Some(sidecar_sum.unwrap_or(part_len))
743}
744
745/// Returns the size of the first `.chunked.part` file found in the blobs directory.
746fn find_partial_blob_size(blobs_dir: &Path) -> u64 {
747 let Ok(entries) = std::fs::read_dir(blobs_dir) else {
748 return 0;
749 };
750
751 for entry in entries {
752 let Ok(entry) = entry else { continue };
753 let name = entry.file_name();
754 // BORROW: explicit .to_string_lossy() for OsString → str conversion
755 if name.to_string_lossy().ends_with(".chunked.part") {
756 return entry.metadata().map_or(0, |m| m.len());
757 }
758 }
759
760 0
761}
762
763/// A `.chunked.part` temp file left by an interrupted chunked download.
764#[derive(Debug, Clone)]
765pub struct PartialFile {
766 /// The repository identifier (e.g., `"meta-llama/Llama-3.2-1B"`).
767 pub repo_id: String,
768 /// The `.chunked.part` filename (e.g., `"abc123def456.chunked.part"`).
769 pub filename: String,
770 /// Absolute path to the `.chunked.part` file.
771 pub path: PathBuf,
772 /// Size of the partial file in bytes.
773 pub size: u64,
774}
775
776impl PartialFile {
777 /// Returns sibling sidecar paths that should be removed alongside this
778 /// partial: the resume-state sidecar `{etag}.chunked.part.state` and
779 /// any orphan write-tmp `{etag}.chunked.part.state.tmp` left by an
780 /// interrupted atomic save.
781 ///
782 /// The paths are returned even when the underlying files do not exist
783 /// — callers (`run_cache_clean_partial`) attempt removal best-effort.
784 #[must_use]
785 pub fn sidecar_paths(&self) -> Vec<PathBuf> {
786 let Some(parent) = self.path.parent() else {
787 return Vec::new();
788 };
789 // String concat (mirrors `cache_layout::temp_state_path`'s
790 // rationale): the etag may itself contain periods, so
791 // `Path::with_extension` would truncate at the wrong boundary.
792 // BORROW: explicit .clone() for owned String → mutated copy
793 let mut state_name = self.filename.clone();
794 state_name.push_str(".state");
795 // BORROW: explicit .clone() for owned String → mutated copy
796 let mut tmp_name = self.filename.clone();
797 tmp_name.push_str(".state.tmp");
798 vec![parent.join(state_name), parent.join(tmp_name)]
799 }
800}
801
802/// Finds all `.chunked.part` temp files in the `HuggingFace` cache.
803///
804/// Walks `models--*/blobs/` directories and collects partial files.
805/// When `repo_filter` is `Some`, only the matching repo is scanned.
806///
807/// Returns an empty `Vec` if the cache directory does not exist.
808///
809/// # Errors
810///
811/// Returns [`FetchError::Io`] if the cache directory cannot be read.
812pub fn find_partial_files(repo_filter: Option<&str>) -> Result<Vec<PartialFile>, FetchError> {
813 let cache_dir = hf_cache_dir()?;
814
815 if !cache_dir.exists() {
816 return Ok(Vec::new());
817 }
818
819 let entries = std::fs::read_dir(&cache_dir).map_err(|e| FetchError::Io {
820 // BORROW: explicit .clone() for owned PathBuf
821 path: cache_dir.clone(),
822 source: e,
823 })?;
824
825 let mut partials: Vec<PartialFile> = Vec::new();
826
827 for entry in entries {
828 let Ok(entry) = entry else { continue };
829 let dir_name = entry.file_name();
830 // BORROW: explicit .to_string_lossy() for OsString → str conversion
831 let dir_str = dir_name.to_string_lossy();
832
833 let Some(repo_id) = repo_id_from_folder_name(&dir_str) else {
834 continue;
835 };
836
837 // Skip repos that don't match the filter.
838 // BORROW: explicit .as_str() instead of Deref coercion
839 if let Some(filter) = repo_filter {
840 if repo_id.as_str() != filter {
841 continue;
842 }
843 }
844
845 let blobs_dir = crate::cache_layout::blobs_dir(&entry.path());
846 let Ok(blob_entries) = std::fs::read_dir(&blobs_dir) else {
847 continue;
848 };
849
850 for blob_entry in blob_entries {
851 let Ok(blob_entry) = blob_entry else { continue };
852 let name = blob_entry.file_name();
853 // BORROW: explicit .to_string_lossy() for OsString → str conversion
854 let name_str = name.to_string_lossy();
855 if name_str.ends_with(".chunked.part") {
856 let size = blob_entry.metadata().map_or(0, |m| m.len());
857 partials.push(PartialFile {
858 // BORROW: explicit .clone() for owned String
859 repo_id: repo_id.clone(),
860 // BORROW: explicit .to_string() for Cow<str> → owned String
861 filename: name_str.to_string(),
862 path: blob_entry.path(),
863 size,
864 });
865 }
866 }
867 }
868
869 Ok(partials)
870}
871
872/// Per-file disk usage entry within a cached repository.
873#[derive(Debug, Clone)]
874pub struct CacheFileUsage {
875 /// Filename relative to the snapshot directory.
876 pub filename: String,
877 /// File size in bytes.
878 pub size: u64,
879}
880
881/// Returns per-file disk usage for a specific cached repository.
882///
883/// Walks the snapshot directories under
884/// `<cache_dir>/models--<org>--<name>/snapshots/` and collects each file's
885/// relative path and size. Results are sorted by size descending.
886///
887/// Returns an empty `Vec` if the repository is not cached.
888///
889/// # Errors
890///
891/// Returns [`FetchError::Io`] if the cache directory cannot be determined.
892pub fn cache_repo_usage(repo_id: &str) -> Result<Vec<CacheFileUsage>, FetchError> {
893 let cache_dir = hf_cache_dir()?;
894 let repo_dir = crate::cache_layout::repo_dir(&cache_dir, repo_id);
895
896 if !repo_dir.exists() {
897 return Ok(Vec::new());
898 }
899
900 let snapshots_dir = crate::cache_layout::snapshots_dir(&repo_dir);
901 let Ok(snapshots) = std::fs::read_dir(&snapshots_dir) else {
902 return Ok(Vec::new());
903 };
904
905 let mut files: Vec<CacheFileUsage> = Vec::new();
906
907 for snap_entry in snapshots {
908 let Ok(snap_entry) = snap_entry else { continue };
909 let snap_path = snap_entry.path();
910 if !snap_path.is_dir() {
911 continue;
912 }
913 collect_snapshot_files(&snap_path, "", &mut files);
914 }
915
916 files.sort_by_key(|f| std::cmp::Reverse(f.size));
917
918 Ok(files)
919}
920
921/// Recursively collects files from a snapshot directory into `CacheFileUsage` entries.
922///
923/// The `prefix` parameter tracks the relative path from the snapshot root,
924/// so that files in subdirectories get paths like `"tokenizer/vocab.json"`.
925fn collect_snapshot_files(dir: &Path, prefix: &str, files: &mut Vec<CacheFileUsage>) {
926 let Ok(entries) = std::fs::read_dir(dir) else {
927 return;
928 };
929
930 for entry in entries {
931 let Ok(entry) = entry else { continue };
932 let path = entry.path();
933 // BORROW: explicit .to_string_lossy() for OsString → str conversion
934 let name = entry.file_name().to_string_lossy().to_string();
935
936 if path.is_dir() {
937 let child_prefix = if prefix.is_empty() {
938 name
939 } else {
940 format!("{prefix}/{name}")
941 };
942 collect_snapshot_files(&path, &child_prefix, files);
943 } else {
944 let filename = if prefix.is_empty() {
945 name
946 } else {
947 format!("{prefix}/{name}")
948 };
949 let size = entry.metadata().map_or(0, |m| m.len());
950 files.push(CacheFileUsage { filename, size });
951 }
952 }
953}
954
955/// Verification status for a single cached file.
956#[non_exhaustive]
957#[derive(Debug, Clone)]
958pub enum VerifyStatus {
959 /// Local `SHA256` matches the expected hash from `HuggingFace` LFS metadata.
960 Ok,
961 /// Local `SHA256` does not match the expected hash — the cached file is
962 /// corrupted (bit rot, interrupted write, or upstream blob changed).
963 Mismatch {
964 /// Expected `SHA256` hex digest from `HuggingFace` LFS metadata.
965 expected: String,
966 /// Actual `SHA256` hex digest computed from the local file.
967 actual: String,
968 },
969 /// File has no LFS metadata (small git-stored file); verification skipped.
970 Skipped,
971 /// File is absent from the local snapshot directory.
972 Missing,
973}
974
975/// Result of verifying a single cached file against `HuggingFace` LFS metadata.
976#[derive(Debug, Clone)]
977pub struct FileVerification {
978 /// Filename within the repository.
979 pub filename: String,
980 /// File size in bytes — local size when the file is present, otherwise
981 /// the expected size from the API (or `0` when neither is known).
982 pub size: u64,
983 /// Verification result.
984 pub status: VerifyStatus,
985}
986
987/// Streaming progress event emitted by [`verify_cache_with_progress`] so
988/// callers can render per-file feedback during a long verification.
989///
990/// Events fire in this order:
991/// 1. [`VerifyEvent::Started`] — once, after the metadata fetch completes,
992/// before any per-file work begins. Carries the total file count and a
993/// pre-computed maximum filename length so callers can size display
994/// columns up-front.
995/// 2. For each file in alphabetical order:
996/// - [`VerifyEvent::FileStart`] — before the per-file `SHA256`
997/// computation kicks in.
998/// - [`VerifyEvent::FileComplete`] — when the per-file result is known,
999/// carrying the [`VerifyStatus`] outcome.
1000#[non_exhaustive]
1001#[derive(Debug)]
1002pub enum VerifyEvent<'a> {
1003 /// Fired once at the start of the run with summary stats useful for
1004 /// laying out a streamed table or progress display.
1005 Started {
1006 /// Total number of files that will be verified.
1007 total: usize,
1008 /// Maximum filename length across the verification list.
1009 max_filename_len: usize,
1010 },
1011 /// A file is about to be verified.
1012 FileStart {
1013 /// 1-based index of this file in the verification list.
1014 index: usize,
1015 /// Total number of files in the verification list.
1016 total: usize,
1017 /// Filename within the repository.
1018 filename: &'a str,
1019 /// File size in bytes (local size when present, else expected size).
1020 size: u64,
1021 /// `true` when the file has LFS metadata (a real `SHA256` computation
1022 /// is about to run); `false` when the file is git-stored and will be
1023 /// skipped near-instantly.
1024 has_lfs: bool,
1025 },
1026 /// A file's verification has completed.
1027 FileComplete {
1028 /// 1-based index of this file in the verification list.
1029 index: usize,
1030 /// Total number of files in the verification list.
1031 total: usize,
1032 /// Filename within the repository.
1033 filename: &'a str,
1034 /// File size in bytes (matches the `size` from the corresponding
1035 /// [`VerifyEvent::FileStart`]).
1036 size: u64,
1037 /// The per-file verification result.
1038 status: &'a VerifyStatus,
1039 },
1040}
1041
1042/// Verifies `SHA256` digests of cached files against `HuggingFace` LFS metadata.
1043///
1044/// Fetches the expected hashes from the `HuggingFace` API and, for each file
1045/// that has an LFS `SHA256`, reads the local cached file and compares.
1046///
1047/// Files without LFS metadata (small git-stored files such as `config.json`)
1048/// are reported as [`VerifyStatus::Skipped`]; files absent from the snapshot
1049/// directory are reported as [`VerifyStatus::Missing`]. Both are
1050/// non-failures — only [`VerifyStatus::Mismatch`] indicates a corrupted file.
1051///
1052/// `revision` defaults to `"main"` when `None`. Requires network access for
1053/// the metadata fetch; the per-file digest computation is local-only.
1054///
1055/// For long verifications (multi-GiB safetensors files), prefer
1056/// [`verify_cache_with_progress`] so a CLI / GUI can render a spinner or
1057/// progress bar while each file is hashed.
1058///
1059/// # Errors
1060///
1061/// Returns [`FetchError::Http`] if the `HuggingFace` API request fails.
1062/// Returns [`FetchError::Io`] when a local cached file is present but
1063/// cannot be read.
1064pub async fn verify_cache(
1065 repo_id: &str,
1066 token: Option<&str>,
1067 revision: Option<&str>,
1068) -> Result<Vec<FileVerification>, FetchError> {
1069 verify_cache_with_progress(repo_id, token, revision, |_| {}).await
1070}
1071
1072/// Same as [`verify_cache`] but emits [`VerifyEvent`]s through `on_event`
1073/// so callers can render streaming progress (e.g. a spinner per file).
1074///
1075/// The callback runs on the same task as the verification — keep it short.
1076/// Use interior mutability ([`std::cell::Cell`], [`std::cell::RefCell`]) if
1077/// you need to track state across events; the closure may capture by shared
1078/// reference because the API requires only [`Fn`].
1079///
1080/// Files are processed in alphabetical order by filename so that streamed
1081/// output remains stable across runs and matches the sort order of the
1082/// returned [`Vec<FileVerification>`].
1083///
1084/// # Errors
1085///
1086/// Same error conditions as [`verify_cache`].
1087pub async fn verify_cache_with_progress<F>(
1088 repo_id: &str,
1089 token: Option<&str>,
1090 revision: Option<&str>,
1091 on_event: F,
1092) -> Result<Vec<FileVerification>, FetchError>
1093where
1094 F: Fn(VerifyEvent<'_>),
1095{
1096 let revision = revision.unwrap_or("main");
1097 let cache_dir = hf_cache_dir()?;
1098 let repo_dir = crate::cache_layout::repo_dir(&cache_dir, repo_id);
1099
1100 let commit_hash = read_ref(&repo_dir, revision);
1101
1102 let client = crate::chunked::build_client(token)?;
1103 let mut remote_files =
1104 crate::repo::list_repo_files_with_metadata(repo_id, token, Some(revision), &client).await?;
1105
1106 // Sort up-front so streamed output is stable across runs and matches the
1107 // returned `Vec<FileVerification>`'s order.
1108 remote_files.sort_by(|a, b| a.filename.cmp(&b.filename));
1109
1110 // BORROW: explicit .as_deref() for Option<String> → Option<&str>
1111 let snapshot_dir = commit_hash
1112 .as_deref()
1113 .map(|hash| crate::cache_layout::snapshot_dir(&repo_dir, hash));
1114
1115 let total = remote_files.len();
1116 let max_filename_len = remote_files
1117 .iter()
1118 .map(|f| f.filename.len())
1119 .max()
1120 .unwrap_or(0);
1121
1122 on_event(VerifyEvent::Started {
1123 total,
1124 max_filename_len,
1125 });
1126
1127 let mut results: Vec<FileVerification> = Vec::with_capacity(total);
1128
1129 for (i, remote) in remote_files.iter().enumerate() {
1130 let index = i + 1;
1131 let local_path = snapshot_dir
1132 .as_ref()
1133 // BORROW: explicit .as_str() for path construction
1134 .map(|dir| dir.join(remote.filename.as_str()));
1135
1136 let exists = local_path.as_ref().is_some_and(|p| p.exists());
1137 let local_size = local_path
1138 .as_ref()
1139 .filter(|_| exists)
1140 .and_then(|p| std::fs::metadata(p).ok().map(|m| m.len()))
1141 .unwrap_or(0);
1142 let expected_size = remote.size.unwrap_or(0);
1143 let display_size = if exists { local_size } else { expected_size };
1144
1145 let has_lfs = remote.sha256.is_some();
1146 on_event(VerifyEvent::FileStart {
1147 index,
1148 total,
1149 // BORROW: explicit .as_str() for &String → &str argument
1150 filename: remote.filename.as_str(),
1151 size: display_size,
1152 has_lfs,
1153 });
1154
1155 let status = match (remote.sha256.as_deref(), local_path.as_deref(), exists) {
1156 (None, _, _) => VerifyStatus::Skipped,
1157 (Some(_), None, _) | (Some(_), Some(_), false) => VerifyStatus::Missing,
1158 (Some(expected), Some(path), true) => {
1159 // BORROW: explicit .as_str() for &String → &str argument
1160 match crate::checksum::verify_sha256(path, remote.filename.as_str(), expected).await
1161 {
1162 Ok(()) => VerifyStatus::Ok,
1163 Err(FetchError::Checksum {
1164 expected, actual, ..
1165 }) => VerifyStatus::Mismatch { expected, actual },
1166 Err(e) => return Err(e),
1167 }
1168 }
1169 };
1170
1171 on_event(VerifyEvent::FileComplete {
1172 index,
1173 total,
1174 // BORROW: explicit .as_str() for &String → &str argument
1175 filename: remote.filename.as_str(),
1176 size: display_size,
1177 status: &status,
1178 });
1179
1180 results.push(FileVerification {
1181 // BORROW: explicit .clone() for owned String
1182 filename: remote.filename.clone(),
1183 size: display_size,
1184 status,
1185 });
1186 }
1187
1188 Ok(results)
1189}
1190
1191#[cfg(test)]
1192mod tests {
1193 #![allow(
1194 clippy::panic,
1195 clippy::unwrap_used,
1196 clippy::expect_used,
1197 clippy::indexing_slicing
1198 )]
1199
1200 use super::*;
1201
1202 fn sample_partial(filename: &str) -> PartialFile {
1203 PartialFile {
1204 repo_id: "org/model".to_owned(),
1205 filename: filename.to_owned(),
1206 path: PathBuf::from("/tmp/models--org--model/blobs").join(filename),
1207 size: 1024,
1208 }
1209 }
1210
1211 #[test]
1212 fn sidecar_paths_returns_state_and_state_tmp() {
1213 let p = sample_partial("abc123.chunked.part");
1214 let sidecars = p.sidecar_paths();
1215
1216 assert_eq!(sidecars.len(), 2);
1217 assert_eq!(
1218 sidecars[0],
1219 PathBuf::from("/tmp/models--org--model/blobs/abc123.chunked.part.state")
1220 );
1221 assert_eq!(
1222 sidecars[1],
1223 PathBuf::from("/tmp/models--org--model/blobs/abc123.chunked.part.state.tmp")
1224 );
1225 }
1226
1227 #[test]
1228 fn sidecar_paths_handles_etag_with_periods() {
1229 // Same period-handling rationale as `cache_layout::temp_state_path`:
1230 // the etag may itself contain dots, so naive `Path::with_extension`
1231 // would chop at the wrong boundary.
1232 let p = sample_partial("abc.def.chunked.part");
1233 let sidecars = p.sidecar_paths();
1234
1235 assert_eq!(
1236 sidecars[0],
1237 PathBuf::from("/tmp/models--org--model/blobs/abc.def.chunked.part.state")
1238 );
1239 assert_eq!(
1240 sidecars[1],
1241 PathBuf::from("/tmp/models--org--model/blobs/abc.def.chunked.part.state.tmp")
1242 );
1243 }
1244
1245 #[test]
1246 fn snapshot_roundtrip_write_then_read_returns_equal_value() {
1247 // Use a freshly-created temp dir so the test is isolated from any
1248 // pre-existing cache state on the developer machine / CI runner.
1249 let tmp =
1250 std::env::temp_dir().join(format!("hf-fm-snapshot-roundtrip-{}", std::process::id()));
1251 std::fs::create_dir_all(&tmp).expect("create temp dir");
1252
1253 let original = Snapshot {
1254 version: SNAPSHOT_VERSION,
1255 revision: "main".to_owned(),
1256 preset: Some("safetensors".to_owned()),
1257 filter: vec!["*.json".to_owned()],
1258 exclude: vec!["*.md".to_owned()],
1259 };
1260
1261 write_snapshot(&tmp, &original).expect("write_snapshot");
1262 let round_tripped = read_snapshot(&tmp)
1263 .expect("read_snapshot")
1264 .expect("snapshot present");
1265
1266 assert_eq!(round_tripped, original);
1267
1268 // Cleanup
1269 let _ = std::fs::remove_file(snapshot_path(&tmp));
1270 let _ = std::fs::remove_dir(&tmp);
1271 }
1272
1273 #[test]
1274 fn snapshot_read_returns_none_when_absent() {
1275 let tmp =
1276 std::env::temp_dir().join(format!("hf-fm-snapshot-absent-{}", std::process::id()));
1277 std::fs::create_dir_all(&tmp).expect("create temp dir");
1278
1279 let result = read_snapshot(&tmp).expect("read_snapshot");
1280 assert!(result.is_none(), "expected None for absent sidecar");
1281
1282 let _ = std::fs::remove_dir(&tmp);
1283 }
1284
1285 #[test]
1286 fn partial_chunk_size_returns_own_chunk_size() {
1287 let tmp =
1288 std::env::temp_dir().join(format!("hf-fm-partial-own-chunk-{}", std::process::id()));
1289 let blobs = tmp.join("blobs");
1290 std::fs::create_dir_all(&blobs).expect("create blobs dir");
1291 let sha = "41246eed00000000000000000000000000000000000000000000000000000344";
1292 std::fs::write(blobs.join(format!("{sha}.chunked.part")), vec![0u8; 1234])
1293 .expect("write partial blob");
1294
1295 assert_eq!(partial_chunk_size(&tmp, Some(sha)), Some(1234));
1296
1297 let _ = std::fs::remove_dir_all(&tmp);
1298 }
1299
1300 #[test]
1301 fn partial_chunk_size_ignores_other_files_chunks() {
1302 // The regression this guards: a chunk belonging to ANOTHER file must
1303 // not be attributed to this one (the pre-v0.10.5 repo-level fallback
1304 // assigned the first chunk found to every absent file).
1305 let tmp =
1306 std::env::temp_dir().join(format!("hf-fm-partial-other-chunk-{}", std::process::id()));
1307 let blobs = tmp.join("blobs");
1308 std::fs::create_dir_all(&blobs).expect("create blobs dir");
1309 let other_sha = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
1310 std::fs::write(
1311 blobs.join(format!("{other_sha}.chunked.part")),
1312 vec![0u8; 999],
1313 )
1314 .expect("write partial blob");
1315
1316 let queried_sha = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb";
1317 assert_eq!(partial_chunk_size(&tmp, Some(queried_sha)), None);
1318
1319 let _ = std::fs::remove_dir_all(&tmp);
1320 }
1321
1322 #[test]
1323 fn partial_chunk_size_none_without_sha() {
1324 // Non-LFS files carry no sha256; the lookup must not guess.
1325 let tmp = std::env::temp_dir().join(format!("hf-fm-partial-no-sha-{}", std::process::id()));
1326 assert_eq!(partial_chunk_size(&tmp, None), None);
1327 }
1328
1329 #[test]
1330 fn partial_chunk_size_prefers_sidecar_completed_sum() {
1331 // The temp blob is preallocated at full size, so its file length is
1332 // total_size from the first byte; the truth is the sidecar's summed
1333 // per-chunk `completed` offsets.
1334 let tmp =
1335 std::env::temp_dir().join(format!("hf-fm-partial-sidecar-{}", std::process::id()));
1336 let blobs = tmp.join("blobs");
1337 std::fs::create_dir_all(&blobs).expect("create blobs dir");
1338 let sha = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc";
1339 // Preallocated blob: 4096 bytes on disk, only 300 truly downloaded.
1340 std::fs::write(blobs.join(format!("{sha}.chunked.part")), vec![0u8; 4096])
1341 .expect("write partial blob");
1342 let sidecar = format!(
1343 "{{\"schema_version\":1,\"etag\":\"{sha}\",\"total_size\":4096,\
1344 \"connections\":2,\"chunks\":[\
1345 {{\"idx\":0,\"start\":0,\"end\":2047,\"completed\":100}},\
1346 {{\"idx\":1,\"start\":2048,\"end\":4095,\"completed\":200}}]}}"
1347 );
1348 std::fs::write(blobs.join(format!("{sha}.chunked.part.state")), sidecar)
1349 .expect("write sidecar");
1350
1351 assert_eq!(partial_chunk_size(&tmp, Some(sha)), Some(300));
1352
1353 let _ = std::fs::remove_dir_all(&tmp);
1354 }
1355
1356 #[test]
1357 fn partial_chunk_size_falls_back_to_blob_len_on_corrupt_sidecar() {
1358 let tmp = std::env::temp_dir().join(format!(
1359 "hf-fm-partial-corrupt-sidecar-{}",
1360 std::process::id()
1361 ));
1362 let blobs = tmp.join("blobs");
1363 std::fs::create_dir_all(&blobs).expect("create blobs dir");
1364 let sha = "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd";
1365 std::fs::write(blobs.join(format!("{sha}.chunked.part")), vec![0u8; 2048])
1366 .expect("write partial blob");
1367 std::fs::write(
1368 blobs.join(format!("{sha}.chunked.part.state")),
1369 "not json at all",
1370 )
1371 .expect("write corrupt sidecar");
1372
1373 assert_eq!(partial_chunk_size(&tmp, Some(sha)), Some(2048));
1374
1375 let _ = std::fs::remove_dir_all(&tmp);
1376 }
1377}