hf_fetch_model/
repo.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! Repository file listing via the `HuggingFace` API.
4//!
5//! This module provides functions to list all files in a `HuggingFace` model
6//! repository, using the `hf-hub` crate's `info()` API and optionally
7//! fetching extended metadata (sizes and SHA256 hashes) via a direct HTTP call.
8
9use hf_hub::api::tokio::ApiRepo;
10use serde::Deserialize;
11
12use crate::error::FetchError;
13
14/// A file entry in a `HuggingFace` repository.
15#[derive(Debug, Clone)]
16pub struct RepoFile {
17    /// The relative path of the file within the repository.
18    pub filename: String,
19    /// File size in bytes (if known from API metadata).
20    pub size: Option<u64>,
21    /// SHA256 hex digest (if the file is stored in LFS).
22    pub sha256: Option<String>,
23}
24
25/// Lists all files in the given repository.
26///
27/// # Errors
28///
29/// Returns [`FetchError::Api`] if the `HuggingFace` API request fails.
30/// Returns [`FetchError::RepoNotFound`] if the repository does not exist.
31pub async fn list_repo_files(repo: &ApiRepo, repo_id: String) -> Result<Vec<RepoFile>, FetchError> {
32    let info = repo.info().await.map_err(|e| {
33        // BORROW: explicit .to_string() for error message inspection
34        let msg = e.to_string();
35        if msg.contains("404") {
36            FetchError::RepoNotFound { repo_id }
37        } else {
38            FetchError::Api(e)
39        }
40    })?;
41
42    let files = info
43        .siblings
44        .into_iter()
45        .map(|s| RepoFile {
46            filename: s.rfilename,
47            size: None,
48            sha256: None,
49        })
50        .collect();
51
52    Ok(files)
53}
54
55// --- Direct HF API metadata (for SHA256 and file sizes) ---
56
57/// Raw JSON sibling entry from the `HuggingFace` API.
58#[derive(Debug, Deserialize)]
59struct ApiSibling {
60    rfilename: String,
61    #[serde(default)]
62    size: Option<u64>,
63    #[serde(default)]
64    lfs: Option<ApiLfs>,
65}
66
67/// LFS metadata attached to a sibling entry.
68#[derive(Debug, Deserialize)]
69struct ApiLfs {
70    sha256: String,
71    size: u64,
72}
73
74/// Raw JSON response from `GET /api/models/{repo_id}`.
75#[derive(Debug, Deserialize)]
76struct ApiModelInfo {
77    siblings: Vec<ApiSibling>,
78    /// Commit SHA of the resolved revision, when present in the API response.
79    #[serde(default)]
80    sha: Option<String>,
81}
82
83/// Fetches extended file metadata (sizes and SHA256 hashes) via the `HuggingFace` REST API.
84///
85/// This makes a direct HTTP call to `https://huggingface.co/api/models/{repo_id}?blobs=true`
86/// to retrieve file sizes and LFS metadata that `hf-hub`'s `info()` does not expose.
87///
88/// Accepts a shared `reqwest::Client` to reuse TCP connections and TLS sessions
89/// across calls. Use [`chunked::build_client`](crate::chunked::build_client) to
90/// create a client with the standard connect timeout and auth headers.
91///
92/// # Errors
93///
94/// Returns [`FetchError::Http`] if the HTTP request fails.
95/// Returns [`FetchError::RepoNotFound`] if the repository does not exist.
96pub async fn list_repo_files_with_metadata(
97    repo_id: &str,
98    token: Option<&str>,
99    revision: Option<&str>,
100    client: &reqwest::Client,
101) -> Result<Vec<RepoFile>, FetchError> {
102    let (files, _commit) = list_repo_files_with_commit(repo_id, token, revision, client).await?;
103    Ok(files)
104}
105
106/// Fetches file metadata **and** the resolved commit SHA of the revision.
107///
108/// Same HTTP call as [`list_repo_files_with_metadata`], but also returns the
109/// `sha` field from the `HuggingFace` API response. Callers that need to show
110/// or pin the current revision (e.g. `inspect --list`) use this variant; all
111/// other callers should prefer [`list_repo_files_with_metadata`].
112///
113/// The commit SHA is `Option<String>` because the API has not always returned
114/// it on every endpoint variant; treat it as informational.
115///
116/// # Errors
117///
118/// Returns [`FetchError::Http`] if the HTTP request fails.
119/// Returns [`FetchError::RepoNotFound`] if the repository does not exist.
120pub async fn list_repo_files_with_commit(
121    repo_id: &str,
122    token: Option<&str>,
123    revision: Option<&str>,
124    client: &reqwest::Client,
125) -> Result<(Vec<RepoFile>, Option<String>), FetchError> {
126    let mut url = format!("https://huggingface.co/api/models/{repo_id}?blobs=true");
127    if let Some(rev) = revision {
128        url = format!("{url}&revision={rev}");
129    }
130
131    // BORROW: explicit .as_str() instead of Deref coercion
132    let mut request = client.get(url.as_str());
133    if let Some(t) = token {
134        request = request.bearer_auth(t);
135    }
136
137    let response = request
138        .send()
139        .await
140        .map_err(|e| FetchError::Http(e.to_string()))?;
141
142    if response.status() == reqwest::StatusCode::NOT_FOUND {
143        // BORROW: explicit .to_owned() for &str → owned String field
144        return Err(FetchError::RepoNotFound {
145            repo_id: repo_id.to_owned(),
146        });
147    }
148
149    if !response.status().is_success() {
150        return Err(FetchError::Http(format!(
151            "HF API returned status {}",
152            response.status()
153        )));
154    }
155
156    let info: ApiModelInfo = response
157        .json()
158        .await
159        .map_err(|e| FetchError::Http(e.to_string()))?;
160
161    let commit_sha = info.sha;
162    let files = info
163        .siblings
164        .into_iter()
165        .map(|s| {
166            let (size, sha256) = match s.lfs {
167                Some(lfs) => (Some(lfs.size), Some(lfs.sha256)),
168                None => (s.size, None),
169            };
170            RepoFile {
171                filename: s.rfilename,
172                size,
173                sha256,
174            }
175        })
176        .collect();
177
178    Ok((files, commit_sha))
179}
hf_fetch_model/repo.rs

hf_fetch_model/
repo.rs
