Skip to main content

hf_hub/repository/
upload.rs

1//! Repository commit, upload, and delete builders.
2//!
3//! Builders on [`HFRepository`] for mutating repo contents. Every change goes through a single
4//! commit:
5//!
6//! - [`HFRepository::create_commit`] — low-level: arbitrary mix of [`CommitOperation`] entries in one commit.
7//! - [`HFRepository::upload_file`] — upload one file (bytes or local path) as a single-add commit.
8//! - [`HFRepository::upload_folder`] — recursively upload a local folder, with allow/ignore globs matched against
9//!   `folder_path`-relative paths and a `delete_patterns` glob matched against repo-root paths.
10//! - [`HFRepository::delete_file`] / [`HFRepository::delete_folder`] — single-delete and recursive-delete commits.
11//!
12//! See each builder's docs for the exact path / glob format rules.
13
14use std::collections::HashMap;
15use std::fmt::Write as _;
16#[cfg(not(target_family = "wasm"))]
17use std::io::Read;
18#[cfg(not(target_family = "wasm"))]
19use std::path::{Path, PathBuf};
20
21use base64::Engine;
22use bon::bon;
23use futures::stream::StreamExt;
24use sha2::{Digest, Sha256};
25
26#[cfg(not(target_family = "wasm"))]
27use super::files::matches_any_glob;
28use super::{AddSource, CommitInfo, CommitOperation, HFRepository, RepoTreeEntry, RepoType};
29use crate::client::encode_ref;
30#[cfg(not(target_family = "wasm"))]
31use crate::error::HFError;
32use crate::error::HFResult;
33use crate::progress::{EmitEvent, Progress, UploadEvent};
34use crate::{constants, retry};
35
36/// Internal options struct for [`HFRepository::create_commit`]. Built by the bon-generated
37/// `create_commit()` builder.
38struct CreateCommitParams {
39    operations: Vec<CommitOperation>,
40    commit_message: String,
41    commit_description: Option<String>,
42    revision: Option<String>,
43    create_pr: bool,
44    parent_commit: Option<String>,
45    progress: Option<Progress>,
46}
47
48/// Internal options struct for [`HFRepository::upload_file`].
49struct UploadFileParams {
50    source: AddSource,
51    path_in_repo: String,
52    revision: Option<String>,
53    commit_message: Option<String>,
54    commit_description: Option<String>,
55    create_pr: bool,
56    parent_commit: Option<String>,
57    progress: Option<Progress>,
58}
59
60/// Internal options struct for [`HFRepository::upload_folder`].
61#[cfg(not(target_family = "wasm"))]
62struct UploadFolderParams {
63    folder_path: PathBuf,
64    path_in_repo: Option<String>,
65    revision: Option<String>,
66    commit_message: Option<String>,
67    commit_description: Option<String>,
68    create_pr: bool,
69    allow_patterns: Option<Vec<String>>,
70    ignore_patterns: Option<Vec<String>>,
71    delete_patterns: Option<Vec<String>>,
72    progress: Option<Progress>,
73}
74
75/// Internal options struct for [`HFRepository::delete_file`].
76struct DeleteFileParams {
77    path_in_repo: String,
78    revision: Option<String>,
79    commit_message: Option<String>,
80    create_pr: bool,
81}
82
83/// Internal options struct for [`HFRepository::delete_folder`].
84struct DeleteFolderParams {
85    path_in_repo: String,
86    revision: Option<String>,
87    commit_message: Option<String>,
88    create_pr: bool,
89}
90
91impl<T: RepoType> HFRepository<T> {
92    async fn create_commit_impl(&self, params: CreateCommitParams) -> HFResult<CommitInfo> {
93        let revision = params.revision.as_deref().unwrap_or(constants::DEFAULT_REVISION);
94        let url = format!(
95            "{}/commit/{}",
96            self.hf_client.api_url(self.repo_type.plural(), &self.repo_path()),
97            encode_ref(revision)
98        );
99
100        let add_ops_count = params
101            .operations
102            .iter()
103            .filter(|op| matches!(op, CommitOperation::Add { .. }))
104            .count();
105        let total_bytes: u64 = {
106            let mut total = 0u64;
107            for op in &params.operations {
108                if let CommitOperation::Add { source, .. } = op {
109                    total += match source {
110                        AddSource::Bytes(b) => b.len() as u64,
111                        AddSource::Stream(s) => s.size(),
112                        #[cfg(not(target_family = "wasm"))]
113                        AddSource::File(p) => std::fs::metadata(p).map(|m| m.len()).unwrap_or(0),
114                    };
115                }
116            }
117            total
118        };
119
120        params.progress.emit(UploadEvent::Start {
121            total_files: add_ops_count,
122            total_bytes,
123        });
124
125        // Determine which files should be uploaded via xet (LFS) vs. inline
126        // (regular). Files uploaded via xet are referenced by their SHA256 OID
127        // in the commit NDJSON.
128        let lfs_uploaded: HashMap<String, (String, u64)> =
129            self.preupload_and_upload_lfs_files(&params, revision).await?;
130
131        let mut ndjson_lines: Vec<Vec<u8>> = Vec::new();
132
133        let mut header_value = serde_json::json!({
134            "summary": params.commit_message,
135            "description": params.commit_description.as_deref().unwrap_or(""),
136        });
137        if let Some(ref parent) = params.parent_commit {
138            header_value["parentCommit"] = serde_json::Value::String(parent.clone());
139        }
140        let header_line = serde_json::json!({"key": "header", "value": header_value});
141        ndjson_lines.push(serde_json::to_vec(&header_line)?);
142
143        for op in &params.operations {
144            let line = match op {
145                CommitOperation::Add { path_in_repo, source } => {
146                    if let Some((oid, size)) = lfs_uploaded.get(path_in_repo) {
147                        tracing::info!(
148                            path = path_in_repo.as_str(),
149                            oid = oid.as_str(),
150                            size,
151                            "adding lfsFile entry to commit"
152                        );
153                        serde_json::json!({
154                            "key": "lfsFile",
155                            "value": {
156                                "path": path_in_repo,
157                                "algo": "sha256",
158                                "oid": oid,
159                                "size": size,
160                            }
161                        })
162                    } else {
163                        tracing::info!(path = path_in_repo.as_str(), "adding inline base64 file entry to commit");
164                        Self::inline_base64_entry(path_in_repo, source).await?
165                    }
166                },
167                CommitOperation::Delete { path_in_repo } => {
168                    serde_json::json!({
169                        "key": "deletedFile",
170                        "value": {"path": path_in_repo}
171                    })
172                },
173            };
174            ndjson_lines.push(serde_json::to_vec(&line)?);
175        }
176
177        let body: Vec<u8> = ndjson_lines
178            .into_iter()
179            .flat_map(|mut line| {
180                line.push(b'\n');
181                line
182            })
183            .collect();
184
185        params.progress.emit(UploadEvent::Committing);
186
187        let mut headers = self.hf_client.auth_headers();
188        headers.insert(reqwest::header::CONTENT_TYPE, "application/x-ndjson".parse().unwrap());
189
190        let create_pr = params.create_pr;
191        let response = retry::retry(self.hf_client.retry_config(), || {
192            let mut req = self
193                .hf_client
194                .http_client()
195                .post(&url)
196                .headers(headers.clone())
197                .body(body.clone());
198            if create_pr {
199                req = req.query(&[("create_pr", "1")]);
200            }
201            req.send()
202        })
203        .await?;
204        let repo_path = self.repo_path();
205        let response = self
206            .hf_client
207            .check_response(response, Some(&repo_path), crate::error::NotFoundContext::Repo)
208            .await?;
209
210        params.progress.emit(UploadEvent::Complete);
211        Ok(response.json().await?)
212    }
213
214    async fn inline_base64_entry(path_in_repo: &str, source: &AddSource) -> HFResult<serde_json::Value> {
215        let content: bytes::Bytes = match source {
216            #[cfg(not(target_family = "wasm"))]
217            AddSource::File(path) => bytes::Bytes::from(std::fs::read(path)?),
218            AddSource::Bytes(bytes) => bytes.clone(),
219            AddSource::Stream(s) => {
220                let mut stream = s.open();
221                let mut buf = bytes::BytesMut::with_capacity(s.size() as usize);
222                while let Some(chunk) = stream.next().await {
223                    buf.extend_from_slice(&chunk?);
224                }
225                buf.freeze()
226            },
227        };
228        let b64 = base64::engine::general_purpose::STANDARD.encode(&content);
229        Ok(serde_json::json!({
230            "key": "file",
231            "value": {
232                "content": b64,
233                "path": path_in_repo,
234                "encoding": "base64",
235            }
236        }))
237    }
238
239    async fn upload_file_impl(&self, params: UploadFileParams) -> HFResult<CommitInfo> {
240        let commit_message = params
241            .commit_message
242            .clone()
243            .unwrap_or_else(|| format!("Upload {}", params.path_in_repo));
244
245        self.create_commit_impl(CreateCommitParams {
246            operations: vec![CommitOperation::Add {
247                path_in_repo: params.path_in_repo.clone(),
248                source: params.source.clone(),
249            }],
250            commit_message,
251            commit_description: params.commit_description.clone(),
252            revision: params.revision.clone(),
253            create_pr: params.create_pr,
254            parent_commit: params.parent_commit.clone(),
255            progress: params.progress.clone(),
256        })
257        .await
258    }
259
260    #[cfg(not(target_family = "wasm"))]
261    async fn upload_folder_impl(&self, params: UploadFolderParams) -> HFResult<CommitInfo> {
262        let mut operations = Vec::new();
263
264        let folder = &params.folder_path;
265        let base_repo_path = params.path_in_repo.as_deref().unwrap_or("");
266
267        collect_files_recursive(
268            folder,
269            folder,
270            base_repo_path,
271            &params.allow_patterns,
272            &params.ignore_patterns,
273            &mut operations,
274        )?;
275
276        if let Some(ref delete_patterns) = params.delete_patterns {
277            let revision = params.revision.as_deref().unwrap_or(constants::DEFAULT_REVISION);
278            let stream = self.list_tree().revision(revision.to_string()).recursive(true).send()?;
279            futures::pin_mut!(stream);
280            while let Some(entry) = stream.next().await {
281                let entry = entry?;
282                if let RepoTreeEntry::File { path, .. } = entry
283                    && matches_any_glob(delete_patterns, &path)
284                {
285                    operations.push(CommitOperation::delete(path));
286                }
287            }
288        }
289
290        let commit_message = params.commit_message.clone().unwrap_or_else(|| "Upload folder".to_string());
291
292        self.create_commit_impl(CreateCommitParams {
293            operations,
294            commit_message,
295            commit_description: params.commit_description.clone(),
296            revision: params.revision.clone(),
297            create_pr: params.create_pr,
298            parent_commit: None,
299            progress: params.progress.clone(),
300        })
301        .await
302    }
303
304    async fn delete_file_impl(&self, params: DeleteFileParams) -> HFResult<CommitInfo> {
305        let commit_message = params
306            .commit_message
307            .clone()
308            .unwrap_or_else(|| format!("Delete {}", params.path_in_repo));
309
310        self.create_commit_impl(CreateCommitParams {
311            operations: vec![CommitOperation::delete(params.path_in_repo.clone())],
312            commit_message,
313            commit_description: None,
314            revision: params.revision.clone(),
315            create_pr: params.create_pr,
316            parent_commit: None,
317            progress: None,
318        })
319        .await
320    }
321
322    async fn delete_folder_impl(&self, params: DeleteFolderParams) -> HFResult<CommitInfo> {
323        let revision = params.revision.as_deref().unwrap_or(constants::DEFAULT_REVISION);
324
325        let stream = self.list_tree().revision(revision.to_string()).recursive(true).send()?;
326        futures::pin_mut!(stream);
327
328        let mut operations = Vec::new();
329        let prefix = if params.path_in_repo.ends_with('/') {
330            params.path_in_repo.clone()
331        } else {
332            format!("{}/", params.path_in_repo)
333        };
334
335        while let Some(entry) = stream.next().await {
336            let entry = entry?;
337            if let RepoTreeEntry::File { path, .. } = entry
338                && (path.starts_with(&prefix) || path == params.path_in_repo)
339            {
340                operations.push(CommitOperation::delete(path));
341            }
342        }
343
344        let commit_message = params
345            .commit_message
346            .clone()
347            .unwrap_or_else(|| format!("Delete {}", params.path_in_repo));
348
349        self.create_commit_impl(CreateCommitParams {
350            operations,
351            commit_message,
352            commit_description: None,
353            revision: Some(revision.to_string()),
354            create_pr: params.create_pr,
355            parent_commit: None,
356            progress: None,
357        })
358        .await
359    }
360
361    /// Check upload modes for all files and upload LFS files via xet.
362    ///
363    /// Always calls the preupload endpoint to determine upload mode per file.
364    ///
365    /// Returns a map of path_in_repo -> (sha256_oid, size) for files that were
366    /// uploaded via xet and should be referenced as lfsFile in the commit.
367    async fn preupload_and_upload_lfs_files(
368        &self,
369        params: &CreateCommitParams,
370        revision: &str,
371    ) -> HFResult<HashMap<String, (String, u64)>> {
372        let add_ops: Vec<(&String, &AddSource)> = params
373            .operations
374            .iter()
375            .filter_map(|op| match op {
376                CommitOperation::Add { path_in_repo, source } => Some((path_in_repo, source)),
377                _ => None,
378            })
379            .collect();
380
381        if add_ops.is_empty() {
382            return Ok(HashMap::new());
383        }
384
385        // Step 1: Gather file info (path, size, sample, sha) for preupload check.
386        // Sample + SHA are computed in a single source-read pass — see prepare_source.
387        let mut file_infos: Vec<(String, u64, Vec<u8>, String, &AddSource)> = Vec::new();
388        for (path_in_repo, source) in &add_ops {
389            let (size, sample, sha256_oid) = prepare_source(source).await?;
390            file_infos.push(((*path_in_repo).clone(), size, sample, sha256_oid, source));
391        }
392
393        // Step 2: Call preupload endpoint to classify files as "lfs" or "regular"
394        tracing::info!("calling preupload endpoint to classify {} files", file_infos.len());
395        let upload_modes = self
396            .fetch_upload_modes(
397                &self.repo_path(),
398                self.repo_type.plural(),
399                revision,
400                &file_infos
401                    .iter()
402                    .map(|(path, size, sample, _, _)| (path.as_str(), *size, sample.as_slice()))
403                    .collect::<Vec<_>>(),
404            )
405            .await?;
406        tracing::info!(?upload_modes, "preupload classification complete");
407
408        // Step 3: Identify LFS files (empty files are always regular)
409        let lfs_files: Vec<&(String, u64, Vec<u8>, String, &AddSource)> = file_infos
410            .iter()
411            .filter(|(path, size, _, _, _)| {
412                *size > 0 && upload_modes.get(path.as_str()).map(|m| m == "lfs").unwrap_or(false)
413            })
414            .collect();
415
416        if lfs_files.is_empty() {
417            return Ok(HashMap::new());
418        }
419
420        tracing::info!(
421            lfs_file_count = lfs_files.len(),
422            lfs_files = ?lfs_files.iter().map(|(p, s, _, _, _)| (p.as_str(), *s)).collect::<Vec<_>>(),
423            "files requiring LFS upload"
424        );
425
426        self.upload_lfs_files_via_xet(params, revision, &lfs_files).await
427    }
428
429    /// Call the Hub preupload endpoint to determine the upload mode per file.
430    /// Returns a map of path -> upload mode ("lfs" or "regular").
431    async fn fetch_upload_modes(
432        &self,
433        repo_id: &str,
434        api_segment: &str,
435        revision: &str,
436        files: &[(&str, u64, &[u8])],
437    ) -> HFResult<HashMap<String, String>> {
438        let url = format!("{}/preupload/{}", self.hf_client.api_url(api_segment, repo_id), encode_ref(revision));
439
440        let files_payload: Vec<serde_json::Value> = files
441            .iter()
442            .map(|(path, size, sample)| {
443                serde_json::json!({
444                    "path": path,
445                    "size": size,
446                    "sample": base64::engine::general_purpose::STANDARD.encode(sample),
447                })
448            })
449            .collect();
450
451        let body = serde_json::json!({ "files": files_payload });
452
453        let headers = self.hf_client.auth_headers();
454        let response = retry::retry(self.hf_client.retry_config(), || {
455            self.hf_client
456                .http_client()
457                .post(&url)
458                .headers(headers.clone())
459                .json(&body)
460                .send()
461        })
462        .await?;
463
464        let response = self
465            .hf_client
466            .check_response(response, Some(repo_id), crate::error::NotFoundContext::Repo)
467            .await?;
468
469        let preupload: PreuploadResponse = response.json().await?;
470
471        Ok(preupload.files.into_iter().map(|f| (f.path, f.upload_mode)).collect())
472    }
473
474    /// Compute SHA256, negotiate LFS batch transfer, and upload via xet.
475    async fn upload_lfs_files_via_xet(
476        &self,
477        params: &CreateCommitParams,
478        revision: &str,
479        lfs_files: &[&(String, u64, Vec<u8>, String, &AddSource)],
480    ) -> HFResult<HashMap<String, (String, u64)>> {
481        // Step 4: SHA-256 was already computed alongside the preupload sample in
482        // `prepare_source` — see the per-file `file_infos` tuple. We just
483        // unpack it here instead of re-reading each source.
484        tracing::info!("collecting pre-computed SHA256 for {} LFS files", lfs_files.len());
485        let lfs_with_sha: Vec<(String, u64, String, &AddSource)> = lfs_files
486            .iter()
487            .map(|(path, size, _, sha256_oid, source)| {
488                tracing::info!(path = path.as_str(), size = *size, oid = sha256_oid.as_str(), "SHA256 reused");
489                ((*path).clone(), *size, sha256_oid.clone(), *source)
490            })
491            .collect();
492
493        // Step 5: Call LFS batch endpoint to negotiate transfer method
494        let objects: Vec<(&str, u64)> = lfs_with_sha.iter().map(|(_, size, oid, _)| (oid.as_str(), *size)).collect();
495
496        let repo_path = self.repo_path();
497        tracing::info!("calling LFS batch endpoint for transfer negotiation");
498        let chosen_transfer = self
499            .post_lfs_batch_info(&repo_path, self.repo_type.url_prefix(), revision, &objects)
500            .await?;
501        tracing::info!(?chosen_transfer, "LFS batch transfer negotiation complete");
502
503        // Step 6: If server chose xet, upload via xet
504        if chosen_transfer.as_deref() != Some("xet") {
505            tracing::warn!(
506                ?chosen_transfer,
507                "LFS batch did not choose xet transfer; LFS files will fall through to inline upload"
508            );
509            return Ok(HashMap::new());
510        }
511
512        let xet_files: Vec<(String, AddSource)> = lfs_with_sha
513            .iter()
514            .map(|(path, _, _, source)| (path.clone(), (*source).clone()))
515            .collect();
516
517        self.xet_upload(&xet_files, revision, &params.progress).await?;
518
519        let result: HashMap<String, (String, u64)> = lfs_with_sha
520            .into_iter()
521            .map(|(path, size, oid, _)| (path, (oid, size)))
522            .collect();
523
524        Ok(result)
525    }
526
527    /// Call the LFS batch endpoint to negotiate the transfer method.
528    /// Returns the chosen transfer (e.g., "xet", "basic", "multipart").
529    async fn post_lfs_batch_info(
530        &self,
531        repo_id: &str,
532        url_prefix: &str,
533        revision: &str,
534        objects: &[(&str, u64)],
535    ) -> HFResult<Option<String>> {
536        let url = format!("{}/{}{}.git/info/lfs/objects/batch", self.hf_client.endpoint(), url_prefix, repo_id);
537
538        let objects_payload: Vec<serde_json::Value> = objects
539            .iter()
540            .map(|(oid, size)| {
541                serde_json::json!({
542                    "oid": oid,
543                    "size": size,
544                })
545            })
546            .collect();
547
548        let body = serde_json::json!({
549            "operation": "upload",
550            "transfers": ["basic", "multipart", "xet"],
551            "objects": objects_payload,
552            "hash_algo": "sha256",
553            "ref": { "name": revision },
554        });
555
556        let mut headers = self.hf_client.auth_headers();
557        headers.insert(reqwest::header::ACCEPT, "application/vnd.git-lfs+json".parse().unwrap());
558        headers.insert(reqwest::header::CONTENT_TYPE, "application/vnd.git-lfs+json".parse().unwrap());
559
560        let response = retry::retry(self.hf_client.retry_config(), || {
561            self.hf_client
562                .http_client()
563                .post(&url)
564                .headers(headers.clone())
565                .json(&body)
566                .send()
567        })
568        .await?;
569
570        let response = self
571            .hf_client
572            .check_response(response, Some(repo_id), crate::error::NotFoundContext::Repo)
573            .await?;
574
575        let batch: LfsBatchResponse = response.json().await?;
576        Ok(batch.transfer)
577    }
578}
579
580// --- Preupload and LFS upload integration ---
581
582#[derive(Debug, serde::Deserialize)]
583#[serde(rename_all = "camelCase")]
584struct PreuploadFileInfo {
585    path: String,
586    upload_mode: String,
587}
588
589#[derive(Debug, serde::Deserialize)]
590struct PreuploadResponse {
591    files: Vec<PreuploadFileInfo>,
592}
593
594#[derive(Debug, serde::Deserialize)]
595struct LfsBatchResponse {
596    transfer: Option<String>,
597}
598
599fn hex_encode(bytes: &[u8]) -> String {
600    let mut s = String::with_capacity(bytes.len() * 2);
601    for b in bytes {
602        let _ = write!(s, "{b:02x}");
603    }
604    s
605}
606
607const PREUPLOAD_SAMPLE_SIZE: usize = 512;
608
609// Compute size, preupload sample, and SHA-256 in a single pass so a stream
610// source is only read twice total: the metadata pass here and the xet upload.
611async fn prepare_source(source: &AddSource) -> HFResult<(u64, Vec<u8>, String)> {
612    match source {
613        AddSource::Bytes(bytes) => {
614            let sample = bytes[..std::cmp::min(bytes.len(), PREUPLOAD_SAMPLE_SIZE)].to_vec();
615            let hash = Sha256::digest(bytes);
616            Ok((bytes.len() as u64, sample, hex_encode(&hash)))
617        },
618        AddSource::Stream(s) => {
619            let mut stream = s.open();
620            let mut sample: Vec<u8> = Vec::with_capacity(PREUPLOAD_SAMPLE_SIZE);
621            let mut hasher = Sha256::new();
622            while let Some(chunk) = stream.next().await {
623                let chunk = chunk?;
624                hasher.update(&chunk);
625                if sample.len() < PREUPLOAD_SAMPLE_SIZE {
626                    let take = std::cmp::min(PREUPLOAD_SAMPLE_SIZE - sample.len(), chunk.len());
627                    sample.extend_from_slice(&chunk[..take]);
628                }
629            }
630            Ok((s.size(), sample, hex_encode(&hasher.finalize())))
631        },
632        #[cfg(not(target_family = "wasm"))]
633        AddSource::File(path) => {
634            let path = path.clone();
635            tokio::task::spawn_blocking(move || -> HFResult<(u64, Vec<u8>, String)> {
636                let mut file = std::fs::File::open(&path)?;
637                let size = file.metadata()?.len();
638                let mut hasher = Sha256::new();
639                let mut sample: Vec<u8> = Vec::with_capacity(PREUPLOAD_SAMPLE_SIZE);
640                let mut buf = vec![0u8; 64 * 1024];
641                loop {
642                    let n = file.read(&mut buf)?;
643                    if n == 0 {
644                        break;
645                    }
646                    hasher.update(&buf[..n]);
647                    if sample.len() < PREUPLOAD_SAMPLE_SIZE {
648                        let take = std::cmp::min(PREUPLOAD_SAMPLE_SIZE - sample.len(), n);
649                        sample.extend_from_slice(&buf[..take]);
650                    }
651                }
652                Ok((size, sample, hex_encode(&hasher.finalize())))
653            })
654            .await
655            .map_err(|e| HFError::Other(format!("prepare_source task failed: {e}")))?
656        },
657    }
658}
659
660/// Recursively collect files from a directory into CommitOperation::Add entries.
661/// Respects allow_patterns and ignore_patterns (glob-style).
662#[cfg(not(target_family = "wasm"))]
663fn collect_files_recursive(
664    root: &Path,
665    current: &Path,
666    base_repo_path: &str,
667    allow_patterns: &Option<Vec<String>>,
668    ignore_patterns: &Option<Vec<String>>,
669    operations: &mut Vec<CommitOperation>,
670) -> HFResult<()> {
671    for entry in std::fs::read_dir(current)? {
672        let entry = entry?;
673        let path = entry.path();
674        let metadata = entry.metadata()?;
675
676        if metadata.is_dir() {
677            collect_files_recursive(root, &path, base_repo_path, allow_patterns, ignore_patterns, operations)?;
678        } else if metadata.is_file() {
679            let relative = path.strip_prefix(root).map_err(|e| {
680                HFError::InvalidParameter(format!("path {} is not under {}: {e}", path.display(), root.display()))
681            })?;
682            let relative_str: String = relative
683                .components()
684                .filter_map(|c| match c {
685                    std::path::Component::Normal(s) => Some(s.to_string_lossy().into_owned()),
686                    _ => None,
687                })
688                .collect::<Vec<_>>()
689                .join("/");
690
691            if let Some(allow) = allow_patterns
692                && !matches_any_glob(allow, &relative_str)
693            {
694                continue;
695            }
696            if let Some(ignore) = ignore_patterns
697                && matches_any_glob(ignore, &relative_str)
698            {
699                continue;
700            }
701
702            let repo_path = if base_repo_path.is_empty() {
703                relative_str
704            } else {
705                format!("{}/{}", base_repo_path.trim_end_matches('/'), relative_str)
706            };
707
708            operations.push(CommitOperation::add_file(repo_path, path));
709        }
710    }
711
712    Ok(())
713}
714
715#[bon]
716impl<T: RepoType> HFRepository<T> {
717    /// Create a commit with multiple operations.
718    ///
719    /// This is the lowest-level public mutation API in the files' module. Use it when you need an
720    /// explicit mix of add and delete operations in one commit. For one-shot workflows, prefer
721    /// [`HFRepository::upload_file`], [`HFRepository::upload_folder`],
722    /// [`HFRepository::delete_file`], or [`HFRepository::delete_folder`].
723    ///
724    /// Endpoint: `POST /api/{repo_type}s/{repo_id}/commit/{revision}`.
725    ///
726    /// # Parameters
727    ///
728    /// - `operations` (required): list of file operations to include in the commit.
729    /// - `commit_message` (required): commit message.
730    /// - `commit_description`: extended description for the commit.
731    /// - `revision`: branch to commit to. Defaults to the main branch.
732    /// - `create_pr` (default `false`): create a pull request instead of committing directly.
733    /// - `parent_commit`: expected parent commit SHA. Fails if the branch head moved past it.
734    /// - `progress`: optional progress handler.
735    #[builder(finish_fn = send, derive(Debug, Clone))]
736    pub async fn create_commit(
737        &self,
738        /// List of file operations to include in the commit.
739        operations: Vec<CommitOperation>,
740        /// Commit message.
741        #[builder(into)]
742        commit_message: String,
743        /// Extended description for the commit.
744        #[builder(into)]
745        commit_description: Option<String>,
746        /// Branch to commit to. Defaults to the main branch.
747        #[builder(into)]
748        revision: Option<String>,
749        /// Create a pull request instead of committing directly.
750        #[builder(default)]
751        create_pr: bool,
752        /// Expected parent commit SHA. Fails if the branch head moved past it.
753        #[builder(into)]
754        parent_commit: Option<String>,
755        /// Progress handler.
756        #[builder(into)]
757        progress: Option<Progress>,
758    ) -> HFResult<CommitInfo> {
759        Box::pin(self.create_commit_impl(CreateCommitParams {
760            operations,
761            commit_message,
762            commit_description,
763            revision,
764            create_pr,
765            parent_commit,
766            progress,
767        }))
768        .await
769    }
770
771    /// Upload a single file to a repository.
772    ///
773    /// Convenience wrapper around [`HFRepository::create_commit`]. If `commit_message` is
774    /// omitted, a default `"Upload {path}"` message is used.
775    ///
776    /// # Parameters
777    ///
778    /// - `source` (required): file content source (bytes or local file path).
779    /// - `path_in_repo` (required): destination path within the repository.
780    /// - `revision`: branch to upload to. Defaults to the main branch.
781    /// - `commit_message`, `commit_description`, `create_pr`, `parent_commit`, `progress`: same as
782    ///   [`HFRepository::create_commit`]. `create_pr` defaults to `false`.
783    #[builder(finish_fn = send, derive(Debug, Clone))]
784    pub async fn upload_file(
785        &self,
786        /// File content source.
787        source: AddSource,
788        /// Destination path within the repository.
789        #[builder(into)]
790        path_in_repo: String,
791        /// Branch to upload to. Defaults to the main branch.
792        #[builder(into)]
793        revision: Option<String>,
794        /// Commit message. Same as [`HFRepository::create_commit`].
795        #[builder(into)]
796        commit_message: Option<String>,
797        /// Extended description for the commit. Same as [`HFRepository::create_commit`].
798        #[builder(into)]
799        commit_description: Option<String>,
800        /// Create a pull request instead of committing directly. Same as [`HFRepository::create_commit`].
801        #[builder(default)]
802        create_pr: bool,
803        /// Expected parent commit SHA. Same as [`HFRepository::create_commit`].
804        #[builder(into)]
805        parent_commit: Option<String>,
806        /// Progress handler. Same as [`HFRepository::create_commit`].
807        #[builder(into)]
808        progress: Option<Progress>,
809    ) -> HFResult<CommitInfo> {
810        Box::pin(self.upload_file_impl(UploadFileParams {
811            source,
812            path_in_repo,
813            revision,
814            commit_message,
815            commit_description,
816            create_pr,
817            parent_commit,
818            progress,
819        }))
820        .await
821    }
822
823    /// Upload a local folder to a repository.
824    ///
825    /// The folder is walked recursively and converted into add operations. When `delete_patterns`
826    /// is set, matching remote files are also deleted in the same commit.
827    ///
828    /// All pattern arguments use [`globset`](https://docs.rs/globset) syntax (`*`, `?`, `**`,
829    /// character classes, etc.). Path strings are forward-slash-joined regardless of platform.
830    ///
831    /// # Parameters
832    ///
833    /// - `folder_path` (required): local folder path to upload.
834    /// - `path_in_repo`: destination directory within the repository (default: repo root).
835    /// - `revision`: branch to upload to. Defaults to the main branch.
836    /// - `commit_message`, `commit_description`: commit metadata.
837    /// - `create_pr` (default `false`): create a pull request instead of committing directly.
838    /// - `allow_patterns`: globs selecting which local files to include. Matched against each discovered file's path
839    ///   relative to `folder_path` (e.g., `data/train.bin`, not the absolute path and not prefixed with
840    ///   `path_in_repo`). When set, only files matching at least one pattern are uploaded.
841    /// - `ignore_patterns`: globs of local files to skip. Matched against the same `folder_path`-relative paths as
842    ///   `allow_patterns`.
843    /// - `delete_patterns`: globs of *remote* files to delete in the same commit. Matched against each existing file's
844    ///   full repository path (relative to repo root, **not** relative to `path_in_repo`) — e.g., `old/*.bin` to remove
845    ///   every `.bin` directly under `old/` at the repo root.
846    /// - `progress`: optional progress handler.
847    #[cfg(not(target_family = "wasm"))]
848    #[builder(finish_fn = send, derive(Debug, Clone))]
849    pub async fn upload_folder(
850        &self,
851        /// Local folder path to upload.
852        #[builder(into)]
853        folder_path: PathBuf,
854        /// Destination directory within the repository (default: repo root).
855        #[builder(into)]
856        path_in_repo: Option<String>,
857        /// Branch to upload to. Defaults to the main branch.
858        #[builder(into)]
859        revision: Option<String>,
860        /// Commit message.
861        #[builder(into)]
862        commit_message: Option<String>,
863        /// Extended description for the commit.
864        #[builder(into)]
865        commit_description: Option<String>,
866        /// Create a pull request instead of committing directly.
867        #[builder(default)]
868        create_pr: bool,
869        /// Globs selecting which local files to include. Matched against each discovered file's path
870        /// relative to `folder_path` (e.g., `data/train.bin`, not the absolute path and not prefixed with
871        /// `path_in_repo`). When set, only files matching at least one pattern are uploaded.
872        allow_patterns: Option<Vec<String>>,
873        /// Globs of local files to skip. Matched against the same `folder_path`-relative paths as
874        /// `allow_patterns`.
875        ignore_patterns: Option<Vec<String>>,
876        /// Globs of *remote* files to delete in the same commit. Matched against each existing file's
877        /// full repository path (relative to repo root, **not** relative to `path_in_repo`) — e.g., `old/*.bin` to
878        /// remove every `.bin` directly under `old/` at the repo root.
879        delete_patterns: Option<Vec<String>>,
880        /// Progress handler.
881        #[builder(into)]
882        progress: Option<Progress>,
883    ) -> HFResult<CommitInfo> {
884        Box::pin(self.upload_folder_impl(UploadFolderParams {
885            folder_path,
886            path_in_repo,
887            revision,
888            commit_message,
889            commit_description,
890            create_pr,
891            allow_patterns,
892            ignore_patterns,
893            delete_patterns,
894            progress,
895        }))
896        .await
897    }
898
899    /// Delete a file from a repository.
900    ///
901    /// Convenience wrapper around [`HFRepository::create_commit`]. If `commit_message` is
902    /// omitted, a default `"Delete {path}"` message is used.
903    ///
904    /// # Parameters
905    ///
906    /// - `path_in_repo` (required): path of the file to delete.
907    /// - `revision`: branch to delete from. Defaults to the main branch.
908    /// - `commit_message`: commit message.
909    /// - `create_pr` (default `false`): create a pull request instead of committing directly.
910    #[builder(finish_fn = send, derive(Debug, Clone))]
911    pub async fn delete_file(
912        &self,
913        /// Path of the file to delete.
914        #[builder(into)]
915        path_in_repo: String,
916        /// Branch to delete from. Defaults to the main branch.
917        #[builder(into)]
918        revision: Option<String>,
919        /// Commit message.
920        #[builder(into)]
921        commit_message: Option<String>,
922        /// Create a pull request instead of committing directly.
923        #[builder(default)]
924        create_pr: bool,
925    ) -> HFResult<CommitInfo> {
926        Box::pin(self.delete_file_impl(DeleteFileParams {
927            path_in_repo,
928            revision,
929            commit_message,
930            create_pr,
931        }))
932        .await
933    }
934
935    /// Delete all files under a repository path.
936    ///
937    /// The current tree is listed recursively and every file at or below `path_in_repo` is turned
938    /// into a delete operation. Directories disappear as a consequence of deleting their contents.
939    ///
940    /// # Parameters
941    ///
942    /// - `path_in_repo` (required): folder path within the repository.
943    /// - `revision`: branch to delete from. Defaults to the main branch.
944    /// - `commit_message`: commit message.
945    /// - `create_pr` (default `false`): create a pull request instead of committing directly.
946    #[builder(finish_fn = send, derive(Debug, Clone))]
947    pub async fn delete_folder(
948        &self,
949        /// Folder path within the repository.
950        #[builder(into)]
951        path_in_repo: String,
952        /// Branch to delete from. Defaults to the main branch.
953        #[builder(into)]
954        revision: Option<String>,
955        /// Commit message.
956        #[builder(into)]
957        commit_message: Option<String>,
958        /// Create a pull request instead of committing directly.
959        #[builder(default)]
960        create_pr: bool,
961    ) -> HFResult<CommitInfo> {
962        Box::pin(self.delete_folder_impl(DeleteFolderParams {
963            path_in_repo,
964            revision,
965            commit_message,
966            create_pr,
967        }))
968        .await
969    }
970}
971
972#[cfg(feature = "blocking")]
973#[bon]
974impl<T: RepoType> crate::blocking::HFRepositorySync<T> {
975    /// Blocking counterpart of [`HFRepository::create_commit`]. See the async method for
976    /// parameters and behavior.
977    #[builder(finish_fn = send, derive(Debug, Clone))]
978    pub fn create_commit(
979        &self,
980        operations: Vec<CommitOperation>,
981        #[builder(into)] commit_message: String,
982        #[builder(into)] commit_description: Option<String>,
983        #[builder(into)] revision: Option<String>,
984        #[builder(default)] create_pr: bool,
985        #[builder(into)] parent_commit: Option<String>,
986        #[builder(into)] progress: Option<Progress>,
987    ) -> HFResult<CommitInfo> {
988        self.runtime.block_on(
989            self.inner
990                .create_commit()
991                .operations(operations)
992                .commit_message(commit_message)
993                .maybe_commit_description(commit_description)
994                .maybe_revision(revision)
995                .create_pr(create_pr)
996                .maybe_parent_commit(parent_commit)
997                .maybe_progress(progress)
998                .send(),
999        )
1000    }
1001
1002    /// Blocking counterpart of [`HFRepository::upload_file`]. See the async method for parameters
1003    /// and behavior.
1004    #[builder(finish_fn = send, derive(Debug, Clone))]
1005    pub fn upload_file(
1006        &self,
1007        source: AddSource,
1008        #[builder(into)] path_in_repo: String,
1009        #[builder(into)] revision: Option<String>,
1010        #[builder(into)] commit_message: Option<String>,
1011        #[builder(into)] commit_description: Option<String>,
1012        #[builder(default)] create_pr: bool,
1013        #[builder(into)] parent_commit: Option<String>,
1014        #[builder(into)] progress: Option<Progress>,
1015    ) -> HFResult<CommitInfo> {
1016        self.runtime.block_on(
1017            self.inner
1018                .upload_file()
1019                .source(source)
1020                .path_in_repo(path_in_repo)
1021                .maybe_revision(revision)
1022                .maybe_commit_message(commit_message)
1023                .maybe_commit_description(commit_description)
1024                .create_pr(create_pr)
1025                .maybe_parent_commit(parent_commit)
1026                .maybe_progress(progress)
1027                .send(),
1028        )
1029    }
1030
1031    /// Blocking counterpart of [`HFRepository::upload_folder`]. See the async method for
1032    /// parameters and behavior.
1033    #[cfg(not(target_family = "wasm"))]
1034    #[builder(finish_fn = send, derive(Debug, Clone))]
1035    pub fn upload_folder(
1036        &self,
1037        #[builder(into)] folder_path: PathBuf,
1038        #[builder(into)] path_in_repo: Option<String>,
1039        #[builder(into)] revision: Option<String>,
1040        #[builder(into)] commit_message: Option<String>,
1041        #[builder(into)] commit_description: Option<String>,
1042        #[builder(default)] create_pr: bool,
1043        allow_patterns: Option<Vec<String>>,
1044        ignore_patterns: Option<Vec<String>>,
1045        delete_patterns: Option<Vec<String>>,
1046        #[builder(into)] progress: Option<Progress>,
1047    ) -> HFResult<CommitInfo> {
1048        self.runtime.block_on(
1049            self.inner
1050                .upload_folder()
1051                .folder_path(folder_path)
1052                .maybe_path_in_repo(path_in_repo)
1053                .maybe_revision(revision)
1054                .maybe_commit_message(commit_message)
1055                .maybe_commit_description(commit_description)
1056                .create_pr(create_pr)
1057                .maybe_allow_patterns(allow_patterns)
1058                .maybe_ignore_patterns(ignore_patterns)
1059                .maybe_delete_patterns(delete_patterns)
1060                .maybe_progress(progress)
1061                .send(),
1062        )
1063    }
1064
1065    /// Blocking counterpart of [`HFRepository::delete_file`]. See the async method for parameters
1066    /// and behavior.
1067    #[builder(finish_fn = send, derive(Debug, Clone))]
1068    pub fn delete_file(
1069        &self,
1070        #[builder(into)] path_in_repo: String,
1071        #[builder(into)] revision: Option<String>,
1072        #[builder(into)] commit_message: Option<String>,
1073        #[builder(default)] create_pr: bool,
1074    ) -> HFResult<CommitInfo> {
1075        self.runtime.block_on(
1076            self.inner
1077                .delete_file()
1078                .path_in_repo(path_in_repo)
1079                .maybe_revision(revision)
1080                .maybe_commit_message(commit_message)
1081                .create_pr(create_pr)
1082                .send(),
1083        )
1084    }
1085
1086    /// Blocking counterpart of [`HFRepository::delete_folder`]. See the async method for
1087    /// parameters and behavior.
1088    #[builder(finish_fn = send, derive(Debug, Clone))]
1089    pub fn delete_folder(
1090        &self,
1091        #[builder(into)] path_in_repo: String,
1092        #[builder(into)] revision: Option<String>,
1093        #[builder(into)] commit_message: Option<String>,
1094        #[builder(default)] create_pr: bool,
1095    ) -> HFResult<CommitInfo> {
1096        self.runtime.block_on(
1097            self.inner
1098                .delete_folder()
1099                .path_in_repo(path_in_repo)
1100                .maybe_revision(revision)
1101                .maybe_commit_message(commit_message)
1102                .create_pr(create_pr)
1103                .send(),
1104        )
1105    }
1106}