hf_hub/repository/
files.rs

1//! Repository file listing, metadata, download, and upload APIs.
2//!
3//! Common entry points:
4//!
5//! - Use [`HFRepository::list_tree`] to stream file and directory entries, or [`HFRepository::get_paths_info`] /
6//!   [`HFRepository::get_file_metadata`] for targeted lookups.
7//! - Use [`HFRepository::download_file`] to write one file to disk, [`HFRepository::download_file_stream`] to stream
8//!   bytes, [`HFRepository::download_file_to_bytes`] to collect a file into memory, and
9//!   [`HFRepository::snapshot_download`] to fetch a whole revision.
10//! - Use [`HFRepository::upload_file`] and [`HFRepository::upload_folder`] for convenience, or
11//!   [`HFRepository::create_commit`] when you need an explicit set of add/delete operations in one commit.
12//!
13//! The public types in this module are shared across the listing, download, and
14//! upload helpers implemented on [`HFRepository`].
15
16#[cfg(not(target_family = "wasm"))]
17use std::path::PathBuf;
18use std::pin::Pin;
19use std::sync::Arc;
20
21use bytes::Bytes;
22use futures::stream::Stream;
23use serde::{Deserialize, Serialize};
24
25#[allow(unused_imports)] // used by intra-doc links
26use super::HFRepository;
27use crate::constants;
28use crate::error::HFResult;
29
30/// LFS metadata attached to a repository file, when the file is stored in Git LFS.
31#[derive(Debug, Clone, Serialize, Deserialize)]
32pub struct BlobLfsInfo {
33    /// Original file size in bytes, when reported by the Hub.
34    pub size: Option<u64>,
35    /// SHA-256 object id of the LFS payload.
36    pub sha256: Option<String>,
37    /// Size in bytes of the LFS pointer file stored in git.
38    pub pointer_size: Option<u64>,
39}
40
41/// Summary of the last commit that touched a tree entry, included when expanded metadata is requested.
42#[derive(Debug, Clone, Serialize, Deserialize)]
43#[serde(rename_all = "camelCase")]
44pub struct LastCommitInfo {
45    /// Commit SHA, when available.
46    pub id: Option<String>,
47    /// Commit title/summary line.
48    pub title: Option<String>,
49    /// Commit timestamp in ISO 8601 format, when available.
50    pub date: Option<String>,
51}
52
53/// Security-scan summary for a file in a repository.
54///
55/// Populated on [`RepoTreeEntry::File`] entries when the listing was requested with `expand=true`.
56#[derive(Debug, Clone, Deserialize, Serialize)]
57#[serde(rename_all = "camelCase")]
58pub struct BlobSecurityInfo {
59    /// Status string reported by the scanner (e.g., `"safe"`, `"unsafe"`, `"suspicious"`). The
60    /// file is considered safe iff `status == "safe"`.
61    pub status: String,
62    /// Antivirus-scan details, when present.
63    #[serde(default)]
64    pub av_scan: Option<serde_json::Value>,
65    /// Pickle-import-scan details, when present.
66    #[serde(default)]
67    pub pickle_import_scan: Option<serde_json::Value>,
68}
69
70/// Metadata returned from a HEAD request on a file's resolve URL.
71///
72/// Produced by [`HFRepository::get_file_metadata`]
73/// and used internally by snapshot downloads.
74#[derive(Debug, Clone, Serialize, Deserialize)]
75pub struct FileMetadataInfo {
76    /// Path of the file within the repository.
77    pub filename: String,
78    /// ETag of the file content (normalized, with weak prefix and quotes stripped).
79    pub etag: String,
80    /// Commit hash the revision resolved to (from the `X-Repo-Commit` header).
81    pub commit_hash: String,
82    /// Xet content hash if the file is stored in Xet (from the `X-Xet-Hash` header).
83    pub xet_hash: Option<String>,
84    /// File size in bytes. Falls back to `0` if neither `X-Linked-Size` nor `Content-Length`
85    /// is present on the response.
86    pub file_size: u64,
87    /// Final URL the HEAD request resolved to after redirects (Hub URL or CDN). `None` when no
88    /// redirect was followed and the request URL itself was not preserved.
89    pub location: Option<String>,
90}
91
92/// File or directory entry returned by repository tree/listing APIs.
93#[derive(Debug, Clone, Deserialize, Serialize)]
94#[serde(tag = "type", rename_all = "lowercase")]
95pub enum RepoTreeEntry {
96    /// A file entry in the repository tree.
97    File {
98        /// Object id reported by the Hub for this entry.
99        oid: String,
100        /// File size in bytes.
101        size: u64,
102        /// Repository-relative path.
103        path: String,
104        /// LFS metadata, when the file is LFS-backed.
105        lfs: Option<BlobLfsInfo>,
106        /// Last-commit summary, only when expanded metadata is requested.
107        #[serde(default, rename = "lastCommit")]
108        last_commit: Option<LastCommitInfo>,
109        /// Xet content hash, when the file is Xet-backed.
110        #[serde(default, rename = "xetHash")]
111        xet_hash: Option<String>,
112        /// Security-scan summary for the file, only when expanded metadata is requested.
113        #[serde(default, rename = "securityFileStatus")]
114        security: Option<BlobSecurityInfo>,
115    },
116    /// A directory entry in the repository tree.
117    Directory {
118        /// Object id reported by the Hub for this entry.
119        oid: String,
120        /// Repository-relative path.
121        path: String,
122        /// Last-commit summary, only when expanded metadata is requested.
123        #[serde(default, rename = "lastCommit")]
124        last_commit: Option<LastCommitInfo>,
125    },
126}
127
128/// Response body returned after creating a commit.
129///
130/// Includes URLs for the commit and any PR that was opened, along with the commit OID
131/// when present. Returned by [`HFRepository::create_commit`]
132/// and related upload/delete helpers.
133#[derive(Debug, Clone, Deserialize)]
134#[serde(rename_all = "camelCase")]
135pub struct CommitInfo {
136    /// URL of the created commit on the Hub, when available.
137    #[serde(default)]
138    pub commit_url: Option<String>,
139    /// Commit message recorded for the operation.
140    #[serde(default)]
141    pub commit_message: Option<String>,
142    /// Commit description/body, when provided.
143    #[serde(default)]
144    pub commit_description: Option<String>,
145    /// Commit SHA, when returned by the API.
146    #[serde(default)]
147    pub commit_oid: Option<String>,
148    /// Pull-request URL, when `create_pr` was enabled and a PR was opened.
149    #[serde(default)]
150    pub pr_url: Option<String>,
151    /// Pull-request number, when `create_pr` was enabled and a PR was opened.
152    #[serde(default)]
153    pub pr_num: Option<u64>,
154}
155
156/// File mutation included in [`HFRepository::create_commit`].
157///
158/// Use the constructors ([`add_file`](Self::add_file), [`add_bytes`](Self::add_bytes),
159/// [`delete`](Self::delete)) — they read more linearly than the bare variants and
160/// pick the right [`AddSource`] for you. See [`AddSource`] for the trade-off
161/// between path-backed and in-memory uploads.
162///
163/// ```rust,no_run
164/// use hf_hub::repository::CommitOperation;
165///
166/// let ops = vec![
167///     CommitOperation::add_file("model.safetensors", "/local/path/model.safetensors"),
168///     CommitOperation::add_bytes("config.json", br#"{"vocab_size":50257}"#.as_slice()),
169///     CommitOperation::delete("old/checkpoint.bin"),
170/// ];
171/// # let _ = ops;
172/// ```
173#[derive(Debug, Clone)]
174pub enum CommitOperation {
175    /// Add or replace a file in the repository.
176    Add {
177        /// Destination path within the repository.
178        path_in_repo: String,
179        /// Source of the uploaded contents.
180        source: AddSource,
181    },
182    /// Delete a file from the repository.
183    Delete {
184        /// Repository-relative path to remove.
185        path_in_repo: String,
186    },
187}
188
189impl CommitOperation {
190    /// Add an operation backed by a local file path. Equivalent to
191    /// `Add { path_in_repo, source: AddSource::file(source) }`. Prefer this
192    /// for any non-trivial file size — see [`AddSource::File`] for the streaming
193    /// behavior and memory cost.
194    #[cfg(not(target_family = "wasm"))]
195    pub fn add_file(path_in_repo: impl Into<String>, source: impl Into<PathBuf>) -> Self {
196        CommitOperation::Add {
197            path_in_repo: path_in_repo.into(),
198            source: AddSource::file(source),
199        }
200    }
201
202    /// Add operation backed by in-memory bytes. Equivalent to
203    /// `Add { path_in_repo, source: AddSource::bytes(source) }`. The bytes are
204    /// retained in the operation and re-read at commit time — see
205    /// [`AddSource::Bytes`] for the memory cost.
206    pub fn add_bytes(path_in_repo: impl Into<String>, source: impl Into<Bytes>) -> Self {
207        CommitOperation::Add {
208            path_in_repo: path_in_repo.into(),
209            source: AddSource::bytes(source),
210        }
211    }
212
213    /// Delete operation for a repository path.
214    pub fn delete(path_in_repo: impl Into<String>) -> Self {
215        CommitOperation::Delete {
216            path_in_repo: path_in_repo.into(),
217        }
218    }
219}
220
221/// Stream of byte chunks produced by a [`StreamSource`].
222///
223/// `Send` on native targets; the bound is dropped on wasm, where streams
224/// typically wrap thread-bound JS values.
225#[cfg(not(target_family = "wasm"))]
226pub type SourceByteStream = Pin<Box<dyn Stream<Item = HFResult<Bytes>> + Send>>;
227#[cfg(target_family = "wasm")]
228pub type SourceByteStream = Pin<Box<dyn Stream<Item = HFResult<Bytes>>>>;
229
230/// Factory that produces a fresh [`SourceByteStream`] each time `open` is called.
231///
232/// The upload pipeline reads a source up to three times — a small "sample"
233/// prefix for the preupload classification, the full content for the LFS SHA-256
234/// hash, and the full content for the xet upload — so a one-shot stream isn't
235/// enough. Implementations must be cheap to re-open and produce identical bytes
236/// across opens.
237///
238/// The trait requires `Send + Sync` so that [`AddSource`] (and therefore
239/// `CommitOperation::Add { source }`) can be moved between tasks.
240pub trait StreamFactory: Send + Sync {
241    /// Produce a fresh stream over the source's bytes.
242    fn open(&self) -> SourceByteStream;
243}
244
245/// Source backed by a re-readable stream of byte chunks of known length.
246///
247/// Use this when the bytes are too large to materialize in memory at once.
248/// The upload pipeline only ever holds one chunk in flight at a time.
249#[derive(Clone)]
250pub struct StreamSource {
251    factory: Arc<dyn StreamFactory>,
252    size: u64,
253}
254
255impl StreamSource {
256    /// Construct a [`StreamSource`] from a factory and a known total byte length.
257    ///
258    /// `size` must equal the total number of bytes produced by `factory.open()`,
259    /// which must be the same across every invocation.
260    pub fn new(factory: Arc<dyn StreamFactory>, size: u64) -> Self {
261        Self { factory, size }
262    }
263
264    /// Total number of bytes the stream will produce.
265    pub fn size(&self) -> u64 {
266        self.size
267    }
268
269    /// Open a fresh byte stream from the source.
270    pub fn open(&self) -> SourceByteStream {
271        self.factory.open()
272    }
273}
274
275impl std::fmt::Debug for StreamSource {
276    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
277        f.debug_struct("StreamSource").field("size", &self.size).finish_non_exhaustive()
278    }
279}
280
281/// Source of content for a [`CommitOperation::Add`] operation.
282///
283/// Use [`File`](Self::File) for content already on disk, especially large files:
284/// the path is stored, but file contents are read when the commit runs and are
285/// streamed where possible. Use [`Bytes`](Self::Bytes) for small in-memory content;
286/// the buffer is owned by the operation and may be cloned for LFS/xet uploads.
287///
288/// `AddSource::File` is not a snapshot: the file must still exist, unchanged,
289/// when [`HFRepository::create_commit`] runs.
290#[derive(Debug, Clone)]
291pub enum AddSource {
292    /// Read file contents from this local path at commit time.
293    #[cfg(not(target_family = "wasm"))]
294    File(PathBuf),
295
296    /// In-memory contents used as the file body. Stored as [`bytes::Bytes`] so the
297    /// internal `.clone()` calls along the upload path are cheap refcount bumps
298    /// rather than full memory copies.
299    Bytes(Bytes),
300
301    /// Lazily-read byte stream of known total length. Lets the upload pipeline
302    /// process a source whose total size exceeds available memory: only one
303    /// chunk is held in flight at any time. See [`StreamSource`] for the
304    /// factory contract (must be re-readable to satisfy the sample, hash, and
305    /// upload passes).
306    Stream(StreamSource),
307}
308
309impl AddSource {
310    /// Construct an [`AddSource::File`] from a local file path.
311    #[cfg(not(target_family = "wasm"))]
312    pub fn file(path: impl Into<PathBuf>) -> Self {
313        AddSource::File(path.into())
314    }
315
316    /// Construct an [`AddSource::Bytes`] from in-memory contents.
317    ///
318    /// `bytes` accepts any type that converts to [`bytes::Bytes`]: `Vec<u8>` is
319    /// taken by value with no copy (via `Bytes::from`), `&'static [u8]` /
320    /// `&'static str` are wrapped without copying, and an existing [`Bytes`]
321    /// passes through unchanged.
322    pub fn bytes(bytes: impl Into<Bytes>) -> Self {
323        AddSource::Bytes(bytes.into())
324    }
325
326    /// Construct an [`AddSource::Stream`] from a [`StreamSource`].
327    ///
328    /// The total length must be known up-front and the factory must produce
329    /// identical bytes on every `open()` — see [`StreamSource::new`].
330    pub fn stream(source: StreamSource) -> Self {
331        AddSource::Stream(source)
332    }
333}
334
335pub(super) fn extract_etag(response: &reqwest::Response) -> Option<String> {
336    let headers = response.headers();
337    let raw = headers
338        .get(constants::HEADER_X_LINKED_ETAG)
339        .or_else(|| headers.get(reqwest::header::ETAG))
340        .and_then(|v| v.to_str().ok())?;
341    let normalized = raw.strip_prefix("W/").unwrap_or(raw);
342    Some(normalized.trim_matches('"').to_string())
343}
344
345pub(super) fn extract_commit_hash(response: &reqwest::Response) -> Option<String> {
346    response
347        .headers()
348        .get(constants::HEADER_X_REPO_COMMIT)
349        .and_then(|v| v.to_str().ok())
350        .map(|s| s.to_string())
351}
352
353pub(crate) fn extract_file_size(response: &reqwest::Response) -> Option<u64> {
354    let headers = response.headers();
355    headers
356        .get(constants::HEADER_X_LINKED_SIZE)
357        .or_else(|| headers.get(reqwest::header::CONTENT_LENGTH))
358        .and_then(|v| v.to_str().ok())
359        .and_then(|v| v.parse().ok())
360}
361
362pub(crate) fn extract_xet_hash(response: &reqwest::Response) -> Option<String> {
363    response
364        .headers()
365        .get(constants::HEADER_X_XET_HASH)
366        .and_then(|v| v.to_str().ok())
367        .map(|s| s.to_string())
368}
369
370/// Check if a path matches any of the given glob patterns using the `globset` crate.
371pub(super) fn matches_any_glob(patterns: &[String], path: &str) -> bool {
372    patterns.iter().any(|p| {
373        globset::Glob::new(p)
374            .ok()
375            .map(|g| g.compile_matcher().is_match(path))
376            .unwrap_or(false)
377    })
378}
379
380#[cfg(test)]
381mod tests {
382    use super::{BlobSecurityInfo, CommitInfo, FileMetadataInfo, RepoTreeEntry};
383
384    #[test]
385    fn test_repo_tree_entry_deserialize_file() {
386        let json = r#"{"type":"file","oid":"abc123","size":100,"path":"test.txt"}"#;
387        let entry: RepoTreeEntry = serde_json::from_str(json).unwrap();
388        match entry {
389            RepoTreeEntry::File {
390                path,
391                size,
392                xet_hash,
393                security,
394                ..
395            } => {
396                assert_eq!(path, "test.txt");
397                assert_eq!(size, 100);
398                assert!(xet_hash.is_none());
399                assert!(security.is_none());
400            },
401            _ => panic!("Expected File variant"),
402        }
403    }
404
405    #[test]
406    fn test_repo_tree_entry_file_expanded() {
407        let json = r#"{
408            "type":"file","oid":"abc123","size":100,"path":"weights.safetensors",
409            "xetHash":"xet-deadbeef",
410            "securityFileStatus":{"status":"safe","avScan":{"virusFound":false},"pickleImportScan":null},
411            "lastCommit":{"id":"sha","title":"t","date":"2025-01-01T00:00:00Z"}
412        }"#;
413        let entry: RepoTreeEntry = serde_json::from_str(json).unwrap();
414        match entry {
415            RepoTreeEntry::File {
416                xet_hash,
417                security,
418                last_commit,
419                ..
420            } => {
421                assert_eq!(xet_hash.as_deref(), Some("xet-deadbeef"));
422                let security = security.unwrap();
423                assert_eq!(security.status, "safe");
424                assert!(security.av_scan.is_some());
425                assert!(security.pickle_import_scan.is_none());
426                assert!(last_commit.is_some());
427            },
428            _ => panic!("Expected File variant"),
429        }
430    }
431
432    #[test]
433    fn test_repo_tree_entry_directory_with_last_commit() {
434        let json = r#"{"type":"directory","oid":"def456","path":"src","lastCommit":{"id":"sha","title":"t","date":"2025-01-01T00:00:00Z"}}"#;
435        let entry: RepoTreeEntry = serde_json::from_str(json).unwrap();
436        match entry {
437            RepoTreeEntry::Directory { path, last_commit, .. } => {
438                assert_eq!(path, "src");
439                assert!(last_commit.is_some());
440            },
441            _ => panic!("Expected Directory variant"),
442        }
443    }
444
445    #[test]
446    fn test_repo_tree_entry_deserialize_directory() {
447        let json = r#"{"type":"directory","oid":"def456","path":"src"}"#;
448        let entry: RepoTreeEntry = serde_json::from_str(json).unwrap();
449        match entry {
450            RepoTreeEntry::Directory { path, last_commit, .. } => {
451                assert_eq!(path, "src");
452                assert!(last_commit.is_none());
453            },
454            _ => panic!("Expected Directory variant"),
455        }
456    }
457
458    #[test]
459    fn test_blob_security_info_unsafe_status() {
460        let json = r#"{"status":"suspicious","avScan":null,"pickleImportScan":{"matches":[]}}"#;
461        let info: BlobSecurityInfo = serde_json::from_str(json).unwrap();
462        assert_eq!(info.status, "suspicious");
463        assert!(info.av_scan.is_none());
464        assert!(info.pickle_import_scan.is_some());
465    }
466
467    #[test]
468    fn test_commit_info_with_pr() {
469        let json = r#"{
470            "commitUrl":"https://huggingface.co/owner/repo/commit/abc123",
471            "commitOid":"abc123",
472            "prUrl":"https://huggingface.co/owner/repo/discussions/7",
473            "prNum":7
474        }"#;
475        let info: CommitInfo = serde_json::from_str(json).unwrap();
476        assert_eq!(info.commit_oid.as_deref(), Some("abc123"));
477        assert_eq!(info.pr_url.as_deref(), Some("https://huggingface.co/owner/repo/discussions/7"));
478        assert_eq!(info.pr_num, Some(7));
479    }
480
481    #[test]
482    fn test_commit_info_no_pr() {
483        let json = r#"{"commitUrl":"https://huggingface.co/owner/repo/commit/abc","commitOid":"abc"}"#;
484        let info: CommitInfo = serde_json::from_str(json).unwrap();
485        assert_eq!(info.commit_url.as_deref(), Some("https://huggingface.co/owner/repo/commit/abc"));
486        assert!(info.pr_url.is_none());
487        assert!(info.pr_num.is_none());
488    }
489
490    #[test]
491    fn test_file_metadata_info_location_round_trip() {
492        let original = FileMetadataInfo {
493            filename: "config.json".into(),
494            etag: "abc".into(),
495            commit_hash: "deadbeef".into(),
496            xet_hash: None,
497            file_size: 100,
498            location: Some("https://cdn-lfs.huggingface.co/repos/.../config.json".into()),
499        };
500        let json = serde_json::to_string(&original).unwrap();
501        assert!(json.contains("location"));
502        let round_tripped: FileMetadataInfo = serde_json::from_str(&json).unwrap();
503        assert_eq!(round_tripped.location, original.location);
504    }
505
506    #[test]
507    fn test_file_metadata_info_location_optional() {
508        let json = r#"{"filename":"f","etag":"e","commit_hash":"c","xet_hash":null,"file_size":0}"#;
509        let info: FileMetadataInfo = serde_json::from_str(json).unwrap();
510        assert!(info.location.is_none());
511    }
512}
hf_hub/repository/files.rs

hf_hub/repository/
files.rs
