hf_hub/progress.rs
1//! Progress reporting for upload and download operations.
2//!
3//! Implement [`ProgressHandler`] and pass the handler to the `.progress(...)` setter on any
4//! method builder that supports progress reporting (upload, download, snapshot download,
5//! `create_commit`, bucket sync, etc.). Each `.progress(...)` argument is converted to [`Progress`]
6//! via [`Into`]: an owned handler, an `Arc<H>`, [`Progress::new`], or a [`Progress`] value. When no handler is set,
7//! the library emits nothing — there is no runtime cost.
8//!
9//! # Event model
10//!
11//! Every operation that supports progress emits a stream of [`ProgressEvent`]s
12//! framed by a `Start` event and a `Complete` event. If the operation returns
13//! an error, `Complete` is **not** emitted — consumers should rely on the
14//! returned `Result` for operation success, not on observing `Complete`.
15//!
16//! Upload and download events are distinct enums ([`UploadEvent`] and
17//! [`DownloadEvent`]) wrapped in a [`ProgressEvent`] discriminator. An
18//! individual operation only emits one variant family (e.g., `create_commit`
19//! only emits `Upload(*)`, `snapshot_download` only emits `Download(*)`).
20//!
21//! ## Upload event sequence
22//!
23//! ```text
24//! Start ──┐
25//! │ (silent preflight: preupload API, LFS classification)
26//! Progress ── Progress ── … ── Progress
27//! │ (active upload — poll loop fires ~every 100ms)
28//! Committing
29//! │ (silent: commit API round-trip)
30//! Complete
31//! ```
32//!
33//! ## Download event sequence
34//!
35//! ```text
36//! Start ──┐
37//! │ (HEAD fan-out may precede this for snapshot downloads)
38//! Progress ── Progress ── … ── AggregateProgress ── Progress ── …
39//! │ (Progress = per-file deltas; AggregateProgress = xet batch
40//! │ totals. Either or both, interleaved.)
41//! Complete
42//! ```
43//!
44//! # Implementing a handler
45//!
46//! ```
47//! use std::sync::Arc;
48//!
49//! use hf_hub::progress::{DownloadEvent, Progress, ProgressEvent, ProgressHandler, UploadEvent};
50//!
51//! struct PrintHandler;
52//!
53//! impl ProgressHandler for PrintHandler {
54//! fn on_progress(&self, event: &ProgressEvent) {
55//! match event {
56//! ProgressEvent::Upload(UploadEvent::Start {
57//! total_files,
58//! total_bytes,
59//! }) => {
60//! println!("Uploading {total_files} file(s), {total_bytes} bytes");
61//! },
62//! ProgressEvent::Upload(UploadEvent::Progress {
63//! bytes_completed,
64//! total_bytes,
65//! ..
66//! }) => {
67//! println!(" {bytes_completed}/{total_bytes}");
68//! },
69//! ProgressEvent::Upload(UploadEvent::Committing) => {
70//! println!("Committing...");
71//! },
72//! ProgressEvent::Upload(UploadEvent::Complete) => {
73//! println!("Done.");
74//! },
75//! _ => {},
76//! }
77//! }
78//! }
79//! ```
80//!
81//! # Thread safety and performance contract
82//!
83//! [`ProgressHandler`] requires `Send + Sync` because the library may invoke
84//! `on_progress` from arbitrary tokio tasks, including background poll loops
85//! running on ~100ms tick intervals during active transfers. Implementations
86//! should:
87//!
88//! - **Never block.** Blocking `on_progress` blocks the emitting task, which for upload/download poll loops means
89//! delaying subsequent progress observations. For network streams the library calls `on_progress` on the stream-read
90//! path itself — slow handlers directly slow the transfer.
91//! - **Not panic.** Panics propagate through the tokio runtime and can abort the operation.
92//! - **Be idempotent / tolerant of redundant state.** The library guarantees event *ordering* but not *deduplication*;
93//! e.g., a file may receive multiple `FileStatus::Complete` events across `Progress` and a final cleanup emit in edge
94//! cases. Consumers that track completion should use a set keyed by filename to ignore repeats.
95
96use std::sync::Arc;
97
98/// Receives progress updates from long-running upload and download operations.
99///
100/// Register a handler by wrapping it in [`Progress`] and passing it to the
101/// `.progress(...)` setter of any method builder that supports progress reporting.
102/// See the [module-level docs](self) for the event model, ordering guarantees, and
103/// the must-not-block / `Send + Sync` contract.
104///
105/// # Example
106///
107/// ```
108/// use std::sync::atomic::{AtomicU64, Ordering};
109///
110/// use hf_hub::progress::{ProgressEvent, ProgressHandler, UploadEvent};
111///
112/// struct ByteCounter {
113/// bytes: AtomicU64,
114/// }
115///
116/// impl ProgressHandler for ByteCounter {
117/// fn on_progress(&self, event: &ProgressEvent) {
118/// if let ProgressEvent::Upload(UploadEvent::Progress {
119/// bytes_completed, ..
120/// }) = event
121/// {
122/// self.bytes.store(*bytes_completed, Ordering::Relaxed);
123/// }
124/// }
125/// }
126/// ```
127pub trait ProgressHandler: Send + Sync {
128 /// Invoked by the library for each progress event. The `event` reference is
129 /// only valid for the duration of the call.
130 fn on_progress(&self, event: &ProgressEvent);
131}
132
133/// Shared-ownership wrapper around a [`ProgressHandler`] trait object.
134///
135/// Internally an `Arc<dyn ProgressHandler>`, so cloning is cheap. `progress` setters
136/// take `impl Into<Progress>`, so an owned handler, an `Arc<H>`, or an
137/// `Arc<dyn ProgressHandler>` can be passed directly.
138///
139/// ```
140/// use std::sync::Arc;
141///
142/// use hf_hub::progress::{Progress, ProgressEvent, ProgressHandler};
143///
144/// struct Noop;
145/// impl ProgressHandler for Noop {
146/// fn on_progress(&self, _event: &ProgressEvent) {}
147/// }
148///
149/// let handler: Progress = Noop.into();
150/// let shared: Progress = Arc::new(Noop).into();
151/// let direct = Progress::new(Noop);
152/// ```
153pub struct Progress(Arc<dyn ProgressHandler>);
154
155impl Progress {
156 /// Wrap a handler value in a new `Progress`.
157 pub fn new<H: ProgressHandler + 'static>(handler: H) -> Self {
158 Self(Arc::new(handler))
159 }
160}
161
162impl Clone for Progress {
163 fn clone(&self) -> Self {
164 Self(Arc::clone(&self.0))
165 }
166}
167
168impl std::fmt::Debug for Progress {
169 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
170 f.debug_struct("Progress").finish_non_exhaustive()
171 }
172}
173
174impl std::ops::Deref for Progress {
175 type Target = dyn ProgressHandler;
176 fn deref(&self) -> &Self::Target {
177 &*self.0
178 }
179}
180
181impl<H: ProgressHandler + 'static> From<H> for Progress {
182 fn from(handler: H) -> Self {
183 Self(Arc::new(handler))
184 }
185}
186
187impl<H: ProgressHandler + 'static> From<Arc<H>> for Progress {
188 fn from(handler: Arc<H>) -> Self {
189 Self(handler)
190 }
191}
192
193impl From<Arc<dyn ProgressHandler>> for Progress {
194 fn from(handler: Arc<dyn ProgressHandler>) -> Self {
195 Self(handler)
196 }
197}
198
199/// Top-level progress event dispatched to [`ProgressHandler::on_progress`].
200///
201/// A single operation emits only one variant family — uploads never produce
202/// `Download(*)` and vice versa.
203#[derive(Debug, Clone)]
204pub enum ProgressEvent {
205 /// Emitted by upload operations (`upload_file`, `upload_folder`, `create_commit`,
206 /// `HFBucket::upload_files`, bucket sync in the upload direction).
207 Upload(UploadEvent),
208 /// Emitted by download operations (`download_file`, `snapshot_download`,
209 /// `HFBucket::download_files`, bucket sync in the download direction).
210 Download(DownloadEvent),
211}
212
213/// Lifecycle events for a single upload operation. See the [module-level
214/// docs](self) for the `Start` → `Progress` → `Committing` → `Complete` ordering and
215/// the silent-gap caveats.
216#[derive(Debug, Clone)]
217pub enum UploadEvent {
218 /// Upload has begun; totals are known.
219 Start {
220 /// Number of files the operation will upload (excludes deletes and other
221 /// non-add operations in a commit).
222 total_files: usize,
223 /// Sum of source-content sizes in bytes, before xet deduplication.
224 total_bytes: u64,
225 },
226
227 /// Byte-level progress during the active upload phase, emitted at ~10Hz by the
228 /// xet upload poll loop.
229 ///
230 /// Two byte-count dimensions are reported because xet performs content-defined
231 /// deduplication. The `bytes_completed` / `total_bytes` pair tracks logical
232 /// content bytes (use for a "% processed" bar); the `transfer_bytes_*` triplet
233 /// tracks post-dedup network bytes actually sent (use for a "network activity"
234 /// bar). For deduplicated data, `transfer_bytes` ≪ `total_bytes`.
235 ///
236 /// `files` is a snapshot of every xet-tracked file's state at this event. May
237 /// be empty for operations that don't go through xet (small inline files skip
238 /// `Progress` entirely).
239 Progress {
240 /// Logical content bytes processed so far across all files.
241 bytes_completed: u64,
242 /// Total logical content bytes for the operation (matches `Start.total_bytes`).
243 total_bytes: u64,
244 /// Rate of logical content processing in bytes/sec. `None` during warm-up.
245 bytes_per_sec: Option<f64>,
246 /// Post-dedup network bytes actually sent so far.
247 transfer_bytes_completed: u64,
248 /// Total post-dedup network bytes the operation is expected to send.
249 transfer_bytes: u64,
250 /// Rate of network transfer in bytes/sec. `None` during warm-up.
251 transfer_bytes_per_sec: Option<f64>,
252 /// Per-file snapshot of every xet-tracked file in the upload.
253 files: Vec<FileProgress>,
254 },
255
256 /// Emitted once, immediately before the commit API call. Signals that all byte
257 /// transfer is done; the call itself is silent until `Complete`.
258 Committing,
259
260 /// Terminal event on success. Not emitted on failure — check the returned `Result`.
261 Complete,
262}
263
264/// Lifecycle events for a single download operation. See the [module-level
265/// docs](self) for ordering, the two-channel `Progress` vs `AggregateProgress`
266/// model, and cache-hit fast paths.
267#[derive(Debug, Clone)]
268pub enum DownloadEvent {
269 /// Download operation has begun; totals are known. Fires after the HEAD round-trip
270 /// (or HEAD fan-out for `snapshot_download`).
271 Start {
272 /// Number of files to download.
273 total_files: usize,
274 /// Sum of remote file sizes in bytes, as reported by HEAD responses.
275 total_bytes: u64,
276 },
277
278 /// Per-file progress **delta** — `files` contains only files whose status or
279 /// byte count changed since the previous `Progress` event. Consumers wanting a
280 /// running view of every file must accumulate state by filename.
281 Progress {
282 /// Files whose state changed since the previous `Progress` event.
283 files: Vec<FileProgress>,
284 },
285
286 /// Aggregate byte-level progress for the in-flight xet batch (~10Hz). Reports
287 /// cumulative bytes for the entire batch with no per-file breakdown — xet
288 /// reports aggregate stats only.
289 AggregateProgress {
290 /// Bytes downloaded so far across the in-flight xet batch.
291 bytes_completed: u64,
292 /// Total bytes for the in-flight xet batch.
293 total_bytes: u64,
294 /// Download rate in bytes/sec. `None` until enough samples accumulate.
295 bytes_per_sec: Option<f64>,
296 },
297
298 /// Terminal event on success. Not emitted on failure — check the returned `Result`.
299 Complete,
300}
301
302/// Progress for a single file, carried in `Progress` events. See the parent
303/// variant's docs for whether `files` is a snapshot ([`UploadEvent::Progress`]) or
304/// a delta ([`DownloadEvent::Progress`]).
305#[derive(Debug, Clone)]
306pub struct FileProgress {
307 /// Path as known by the repository or bucket (the `path_in_repo` used when
308 /// uploading, or the remote path as returned from tree listing).
309 pub filename: String,
310 /// Bytes transferred so far for this file.
311 pub bytes_completed: u64,
312 /// Total bytes expected for this file. Zero when the size is unknown (e.g.,
313 /// fast-path cached files emitted purely to signal completion).
314 pub total_bytes: u64,
315 /// Current lifecycle stage.
316 pub status: FileStatus,
317}
318
319/// Lifecycle stage of an individual file within a transfer: `Started` →
320/// `InProgress` → `Complete`. Not every stage is observed for every file (fast
321/// transfers may skip from `Started` to `Complete`, cache hits emit only `Complete`).
322#[derive(Debug, Clone, PartialEq, Eq)]
323pub enum FileStatus {
324 /// File has been queued for transfer but no bytes have moved yet.
325 Started,
326 /// Bytes are actively being transferred.
327 InProgress,
328 /// All bytes for this file have been transferred. Terminal state.
329 Complete,
330}
331
332impl From<UploadEvent> for ProgressEvent {
333 fn from(event: UploadEvent) -> Self {
334 ProgressEvent::Upload(event)
335 }
336}
337
338impl From<DownloadEvent> for ProgressEvent {
339 fn from(event: DownloadEvent) -> Self {
340 ProgressEvent::Download(event)
341 }
342}
343
344pub(crate) trait EmitEvent {
345 fn emit(&self, event: impl Into<ProgressEvent>);
346}
347
348impl<T: ProgressHandler + ?Sized> EmitEvent for T {
349 fn emit(&self, event: impl Into<ProgressEvent>) {
350 self.on_progress(&event.into());
351 }
352}
353
354impl EmitEvent for Option<Progress> {
355 fn emit(&self, event: impl Into<ProgressEvent>) {
356 if let Some(h) = self {
357 h.on_progress(&event.into());
358 }
359 }
360}
361
362#[cfg(test)]
363mod tests {
364 use std::sync::{Arc, Mutex};
365
366 use super::*;
367
368 struct RecordingHandler {
369 events: Mutex<Vec<ProgressEvent>>,
370 }
371
372 impl RecordingHandler {
373 fn new() -> Self {
374 Self {
375 events: Mutex::new(Vec::new()),
376 }
377 }
378
379 fn events(&self) -> Vec<ProgressEvent> {
380 self.events.lock().unwrap().clone()
381 }
382 }
383
384 impl ProgressHandler for RecordingHandler {
385 fn on_progress(&self, event: &ProgressEvent) {
386 self.events.lock().unwrap().push(event.clone());
387 }
388 }
389
390 #[test]
391 fn handler_is_send_sync() {
392 fn assert_send_sync<T: Send + Sync>() {}
393 assert_send_sync::<Arc<RecordingHandler>>();
394 }
395
396 #[test]
397 fn emit_with_none_is_noop() {
398 let progress: Option<Progress> = None;
399 progress.emit(DownloadEvent::Complete);
400 }
401
402 #[test]
403 fn emit_records_events() {
404 let handler = Arc::new(RecordingHandler::new());
405 let progress: Option<Progress> = Some(handler.clone().into());
406
407 progress.emit(UploadEvent::Start {
408 total_files: 2,
409 total_bytes: 1024,
410 });
411 progress.emit(UploadEvent::Progress {
412 bytes_completed: 512,
413 total_bytes: 1024,
414 bytes_per_sec: Some(100.0),
415 transfer_bytes_completed: 0,
416 transfer_bytes: 0,
417 transfer_bytes_per_sec: None,
418 files: vec![],
419 });
420 progress.emit(UploadEvent::Complete);
421
422 let events = handler.events();
423 assert_eq!(events.len(), 3);
424 assert!(matches!(events[0], ProgressEvent::Upload(UploadEvent::Start { .. })));
425 assert!(matches!(events[1], ProgressEvent::Upload(UploadEvent::Progress { .. })));
426 assert!(matches!(events[2], ProgressEvent::Upload(UploadEvent::Complete)));
427 }
428
429 #[test]
430 fn download_file_lifecycle() {
431 let handler = Arc::new(RecordingHandler::new());
432 let progress: Option<Progress> = Some(handler.clone().into());
433
434 progress.emit(DownloadEvent::Start {
435 total_files: 1,
436 total_bytes: 1000,
437 });
438 progress.emit(DownloadEvent::Progress {
439 files: vec![FileProgress {
440 filename: "file.bin".to_string(),
441 bytes_completed: 0,
442 total_bytes: 1000,
443 status: FileStatus::Started,
444 }],
445 });
446 progress.emit(DownloadEvent::Progress {
447 files: vec![FileProgress {
448 filename: "file.bin".to_string(),
449 bytes_completed: 500,
450 total_bytes: 1000,
451 status: FileStatus::InProgress,
452 }],
453 });
454 progress.emit(DownloadEvent::Progress {
455 files: vec![FileProgress {
456 filename: "file.bin".to_string(),
457 bytes_completed: 1000,
458 total_bytes: 1000,
459 status: FileStatus::Complete,
460 }],
461 });
462 progress.emit(DownloadEvent::Complete);
463
464 let events = handler.events();
465 assert_eq!(events.len(), 5);
466 }
467
468 #[test]
469 fn upload_event_ordering() {
470 let handler = Arc::new(RecordingHandler::new());
471 let progress: Option<Progress> = Some(handler.clone().into());
472
473 progress.emit(UploadEvent::Start {
474 total_files: 1,
475 total_bytes: 100,
476 });
477 progress.emit(UploadEvent::Progress {
478 bytes_completed: 50,
479 total_bytes: 100,
480 bytes_per_sec: None,
481 transfer_bytes_completed: 0,
482 transfer_bytes: 0,
483 transfer_bytes_per_sec: None,
484 files: vec![],
485 });
486 progress.emit(UploadEvent::Committing);
487 progress.emit(UploadEvent::Complete);
488
489 let events = handler.events();
490 assert_eq!(events.len(), 4);
491 assert!(matches!(events[0], ProgressEvent::Upload(UploadEvent::Start { .. })));
492 assert!(matches!(events[1], ProgressEvent::Upload(UploadEvent::Progress { .. })));
493 assert!(matches!(events[2], ProgressEvent::Upload(UploadEvent::Committing)));
494 assert!(matches!(events[3], ProgressEvent::Upload(UploadEvent::Complete)));
495 }
496
497 #[test]
498 fn upload_progress_with_per_file_data() {
499 let handler = Arc::new(RecordingHandler::new());
500 let progress: Option<Progress> = Some(handler.clone().into());
501
502 progress.emit(UploadEvent::Progress {
503 bytes_completed: 500,
504 total_bytes: 1000,
505 bytes_per_sec: Some(100.0),
506 transfer_bytes_completed: 250,
507 transfer_bytes: 800,
508 transfer_bytes_per_sec: Some(50.0),
509 files: vec![
510 FileProgress {
511 filename: "model/weights.bin".to_string(),
512 bytes_completed: 300,
513 total_bytes: 600,
514 status: FileStatus::InProgress,
515 },
516 FileProgress {
517 filename: "config.json".to_string(),
518 bytes_completed: 200,
519 total_bytes: 400,
520 status: FileStatus::InProgress,
521 },
522 ],
523 });
524
525 let events = handler.events();
526 assert_eq!(events.len(), 1);
527 if let ProgressEvent::Upload(UploadEvent::Progress {
528 files,
529 transfer_bytes_completed,
530 transfer_bytes,
531 transfer_bytes_per_sec,
532 ..
533 }) = &events[0]
534 {
535 assert_eq!(files.len(), 2);
536 assert_eq!(files[0].filename, "model/weights.bin");
537 assert_eq!(files[1].filename, "config.json");
538 assert_eq!(*transfer_bytes_completed, 250);
539 assert_eq!(*transfer_bytes, 800);
540 assert_eq!(*transfer_bytes_per_sec, Some(50.0));
541 } else {
542 panic!("expected Upload(Progress)");
543 }
544 }
545}