1use std::collections::{BTreeMap, VecDeque};
2use std::path::{Path, PathBuf};
3use std::sync::Arc;
4use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
5use std::time::{Duration, SystemTime, UNIX_EPOCH};
6
7use async_trait::async_trait;
8use base64::Engine;
9use futures_util::StreamExt;
10use reqwest::header::{
11 ACCEPT_ENCODING, AUTHORIZATION, COOKIE, HeaderMap, HeaderName, HeaderValue, LOCATION,
12};
13use reqwest::redirect::Policy;
14use reqwest::{Client, Response, StatusCode};
15use shaul_core::{
16 BackendCapabilities, Checksum, ChecksumAlgorithm, Config, DownloadBackend, DownloadOutcome,
17 DownloadRequest, Error, EventKind, EventSink, FileOutcome, FileResume, Header, Resource,
18 ResourceFile, ResourceKind, Result, TaskId, TaskResume, Validator, VerifyMode, completed_bytes,
19 merge_ranges, missing_ranges, part_path, sanitize_file_name, split_range,
20};
21use tokio::io::{AsyncSeekExt, AsyncWriteExt, SeekFrom};
22use tokio::sync::Mutex;
23use tokio_util::sync::CancellationToken;
24use url::Url;
25
26use crate::filename::parse_content_disposition;
27
28const MAX_REDIRECTS: usize = 10;
29
30const CAPTURE_HEADERS: &[&str] = &[
31 "content-disposition",
32 "etag",
33 "last-modified",
34 "content-length",
35 "accept-ranges",
36 "content-range",
37 "digest",
38 "content-md5",
39 "x-linked-etag",
40 "x-linked-size",
41 "x-repo-commit",
42 "x-xet-hash",
43 "x-xet-token-expiration",
44 "x-xet-refresh-route",
45 "x-error-code",
46];
47
48const RESERVED_HEADERS: &[&str] = &[
49 "range",
50 "if-range",
51 "host",
52 "content-length",
53 "accept-encoding",
54 "connection",
55 "transfer-encoding",
56];
57
58#[derive(Clone)]
59pub struct HttpBackend {
60 client: Client,
61}
62
63impl HttpBackend {
64 pub fn new(config: &Config) -> Result<Self> {
65 let client = Client::builder()
66 .redirect(Policy::none())
67 .connect_timeout(Duration::from_secs(
68 config.network.connect_timeout_secs.max(1),
69 ))
70 .pool_max_idle_per_host(config.download.per_host.max(1))
71 .user_agent(config.network.user_agent.clone())
72 .build()
73 .map_err(|error| Error::Backend(format!("failed to build HTTP client: {error}")))?;
74 Ok(Self { client })
75 }
76
77 pub async fn download_file(
81 &self,
82 request: &DownloadRequest,
83 file: &ResourceFile,
84 task_resume: &mut TaskResume,
85 ) -> Result<FileOutcome> {
86 let key = file.url.as_str().to_string();
87
88 if let Some(stored) = task_resume.files.get(&key).cloned()
89 && stored.is_complete()
90 {
91 let final_path = manifest_path(&request.destination, &stored.final_path);
92 let part = manifest_path(&request.destination, &stored.part_path);
93 if tokio::fs::try_exists(&final_path).await? {
94 let size = tokio::fs::metadata(&final_path).await?.len();
95 let relative = relative_path(&request.destination, &final_path);
96 request.events.emit(EventKind::FileStarted {
97 task_id: request.task_id,
98 file: relative.clone(),
99 total_bytes: Some(size),
100 });
101 request.events.emit(EventKind::Progress {
102 task_id: request.task_id,
103 file: relative.clone(),
104 downloaded_bytes: size,
105 total_bytes: Some(size),
106 });
107 return Ok(FileOutcome::new(
108 final_path,
109 relative,
110 size,
111 stored.checksum.or(request.options.checksum.clone()),
112 false,
113 ));
114 }
115 if tokio::fs::try_exists(&part).await? {
116 ensure_parent(&final_path).await?;
117 remove_if_exists(&final_path).await?;
118 tokio::fs::rename(&part, &final_path).await?;
119 let size = tokio::fs::metadata(&final_path).await?.len();
120 let relative = relative_path(&request.destination, &final_path);
121 request.events.emit(EventKind::FileStarted {
122 task_id: request.task_id,
123 file: relative.clone(),
124 total_bytes: Some(size),
125 });
126 return Ok(FileOutcome::new(
127 final_path,
128 relative,
129 size,
130 stored.checksum.or(request.options.checksum.clone()),
131 false,
132 ));
133 }
134 }
135
136 let probe = self.probe(request, file).await?;
137 let total = probe.total_size;
138 let validator = probe.validator();
139 let checksum = request
140 .options
141 .checksum
142 .clone()
143 .or_else(|| probe.checksum.clone())
144 .or_else(|| file.checksum.clone());
145 if request.options.verify == VerifyMode::Require && checksum.is_none() {
146 return Err(Error::ChecksumMissing);
147 }
148
149 let default_name = file
150 .path
151 .rsplit('/')
152 .next()
153 .map(sanitize_file_name)
154 .unwrap_or_else(|| "download".to_string());
155 let (mut final_path, relative) =
156 resolve_final_path(request, file, probe.filename.as_deref(), &default_name)?;
157
158 let stored = task_resume.files.get(&key).cloned();
159 let final_for_manifest = path_for_manifest(&request.destination, &final_path);
160 let stored_matches_path = stored
161 .as_ref()
162 .map(|stored| stored.final_path == final_for_manifest)
163 .unwrap_or(false);
164 let resume_valid = request.options.resume
165 && stored
166 .as_ref()
167 .map(|stored| {
168 stored_matches_path
169 && !stored.validator.is_empty()
170 && stored.validator.matches(&validator)
171 && match (stored.total_size, total) {
172 (Some(old), Some(new)) => old == new,
173 _ => true,
174 }
175 })
176 .unwrap_or(false);
177
178 if stored.is_some() && !resume_valid {
179 request.events.emit(EventKind::Warning {
180 task_id: Some(request.task_id),
181 message: "resume state is no longer valid; restarting this file".to_string(),
182 });
183 }
184
185 final_path = apply_conflict_policy(final_path, &request.options).await?;
186 let part_name = part_path(&final_path);
187 let part_for_manifest = path_for_manifest(&request.destination, &part_name);
188
189 let mut file_resume = FileResume {
190 url: key.clone(),
191 final_path: path_for_manifest(&request.destination, &final_path),
192 part_path: part_for_manifest.clone(),
193 validator: validator.clone(),
194 total_size: total,
195 segment_size: request.options.segment_size,
196 completed: if resume_valid {
197 stored
198 .as_ref()
199 .map(|stored| stored.completed.clone())
200 .unwrap_or_default()
201 } else {
202 Vec::new()
203 },
204 checksum: checksum.clone(),
205 metadata: {
206 let mut metadata = probe.headers.clone();
207 metadata.insert("final-url".to_string(), redacted_url(&probe.final_url));
208 metadata
209 },
210 updated_at_ms: now_ms(),
211 };
212
213 let existing_part_len = tokio::fs::metadata(&part_name)
215 .await
216 .ok()
217 .map(|metadata| metadata.len());
218 if !resume_valid {
219 file_resume.completed.clear();
220 } else if let Some(len) = existing_part_len {
221 let limit = total.map(|total| total.min(len)).unwrap_or(len);
222 file_resume.completed = shaul_core::range::clamp_to_len(&file_resume.completed, limit);
223 } else {
224 file_resume.completed.clear();
225 }
226 file_resume.normalize();
227
228 ensure_parent(&final_path).await?;
229 if let Some(total) = total {
230 let file = tokio::fs::OpenOptions::new()
231 .create(true)
232 .write(true)
233 .truncate(false)
234 .open(&part_name)
235 .await?;
236 if !resume_valid {
237 file.set_len(0).await?;
238 }
239 file.set_len(total).await?;
240 file.sync_all().await?;
241 } else if !resume_valid {
242 tokio::fs::OpenOptions::new()
243 .create(true)
244 .write(true)
245 .truncate(true)
246 .open(&part_name)
247 .await?;
248 } else if !tokio::fs::try_exists(&part_name).await? {
249 tokio::fs::File::create(&part_name).await?;
250 file_resume.completed.clear();
251 }
252
253 task_resume.files.insert(key.clone(), file_resume.clone());
254 request.checkpoint.save(task_resume).await?;
255
256 let initial = completed_bytes(&file_resume.completed);
257 let progress = Arc::new(AtomicU64::new(initial));
258 request.events.emit(EventKind::FileStarted {
259 task_id: request.task_id,
260 file: relative.clone(),
261 total_bytes: total,
262 });
263
264 let progress_stop = CancellationToken::new();
265 let reporter = spawn_progress_reporter(
266 request.events.clone(),
267 request.task_id,
268 relative.clone(),
269 total,
270 progress.clone(),
271 progress_stop.clone(),
272 );
273
274 let transfer_result = if let Some(0) = total {
275 task_resume.files.insert(key.clone(), file_resume.clone());
277 request.checkpoint.save(task_resume).await?;
278 Ok(file_resume.clone())
279 } else if probe.range_supported
280 && total.map(|total| total > 0).unwrap_or(false)
281 && request.options.connections > 1
282 {
283 self.transfer_segmented(
284 request,
285 file,
286 &part_name,
287 &key,
288 file_resume.clone(),
289 task_resume.clone(),
290 progress.clone(),
291 total,
292 )
293 .await
294 } else {
295 self.transfer_single(
296 request,
297 file,
298 &part_name,
299 &key,
300 file_resume.clone(),
301 task_resume.clone(),
302 progress.clone(),
303 probe.range_supported,
304 )
305 .await
306 };
307
308 progress_stop.cancel();
309 let _ = reporter.await;
310
311 let mut final_resume = match transfer_result {
312 Ok(resume) => resume,
313 Err(error) => return Err(error),
314 };
315
316 if let Some(total) = final_resume.total_size
317 && total > 0
318 {
319 final_resume.completed = vec![shaul_core::ByteRange::new(0, total)?];
320 }
321 final_resume.normalize();
322 request.events.emit(EventKind::Progress {
323 task_id: request.task_id,
324 file: relative.clone(),
325 downloaded_bytes: final_resume.completed_bytes(),
326 total_bytes: final_resume.total_size,
327 });
328 task_resume.files.insert(key, final_resume.clone());
329 request.checkpoint.save(task_resume).await?;
330
331 ensure_parent(&final_path).await?;
332 remove_if_exists(&final_path).await?;
333 if tokio::fs::try_exists(&part_name).await? {
334 tokio::fs::rename(&part_name, &final_path).await?;
335 } else {
336 tokio::fs::write(&final_path, []).await?;
337 }
338 let size = tokio::fs::metadata(&final_path).await?.len();
339 Ok(FileOutcome::new(
340 final_path, relative, size, checksum, false,
341 ))
342 }
343
344 async fn probe(&self, request: &DownloadRequest, file: &ResourceFile) -> Result<Probe> {
345 let mut attempt = 0u32;
346 loop {
347 let headers = build_headers(&request.options, &file.headers, Some("bytes=0-0"), None)?;
348 let sent = self
349 .send_follow(&file.url, &headers, &request.options, &request.cancel)
350 .await;
351 match sent {
352 Err(Error::Cancelled) => return Err(Error::Cancelled),
353 Err(error) => {
354 if attempt >= request.options.retries {
355 return Err(error);
356 }
357 sleep_or_cancel(backoff(attempt), &request.cancel).await?;
358 attempt += 1;
359 }
360 Ok((response, captured, final_url)) => {
361 let status = response.status();
362 let retry_after = retry_after(response.headers());
363 if status.is_server_error()
364 || status == StatusCode::REQUEST_TIMEOUT
365 || status == StatusCode::TOO_MANY_REQUESTS
366 {
367 drop(response);
368 if attempt >= request.options.retries {
369 return Err(http_error(status, &captured));
370 }
371 sleep_or_cancel(
372 retry_after.unwrap_or_else(|| backoff(attempt)),
373 &request.cancel,
374 )
375 .await?;
376 attempt += 1;
377 continue;
378 }
379 if matches!(
380 status,
381 StatusCode::UNAUTHORIZED
382 | StatusCode::FORBIDDEN
383 | StatusCode::NOT_FOUND
384 | StatusCode::GONE
385 ) {
386 return Err(http_error(status, &captured));
387 }
388
389 let content_range = captured
390 .get("content-range")
391 .and_then(|value| parse_content_range(value));
392 let linked_size = captured
393 .get("x-linked-size")
394 .and_then(|value| parse_u64(value));
395 let content_length = captured
396 .get("content-length")
397 .and_then(|value| parse_u64(value));
398 let (range_supported, total_size) = match status {
399 StatusCode::PARTIAL_CONTENT => (
400 true,
401 content_range
402 .and_then(|(_, _, total)| total)
403 .or(linked_size),
404 ),
405 StatusCode::OK => match content_range {
406 Some((_, _, total)) => (true, total.or(linked_size)),
412 None => (false, linked_size.or(content_length)),
413 },
414 StatusCode::RANGE_NOT_SATISFIABLE => {
415 (true, content_range.and_then(|(_, _, total)| total))
416 }
417 _ => {
418 return Err(Error::Backend(format!(
419 "unexpected HTTP status during probe: {status}"
420 )));
421 }
422 };
423
424 let checksum = captured
425 .get("x-linked-etag")
426 .and_then(|value| Checksum::from_etag(value))
427 .or_else(|| parse_digest(&captured));
428 let etag = captured
429 .get("etag")
430 .cloned()
431 .or_else(|| captured.get("x-linked-etag").cloned());
432 let last_modified = captured.get("last-modified").cloned();
433 let filename = captured
434 .get("content-disposition")
435 .and_then(|value| parse_content_disposition(value));
436 let mut headers = captured;
437 if let Some(total) = total_size {
438 headers.insert("total-size".to_string(), total.to_string());
439 }
440 return Ok(Probe {
441 final_url,
442 total_size,
443 range_supported,
444 filename,
445 etag,
446 last_modified,
447 checksum,
448 headers,
449 });
450 }
451 }
452 }
453 }
454
455 async fn send_follow(
456 &self,
457 url: &Url,
458 headers: &HeaderMap,
459 options: &shaul_core::ResolvedOptions,
460 cancel: &CancellationToken,
461 ) -> Result<(Response, BTreeMap<String, String>, Url)> {
462 let mut current = url.clone();
463 let mut headers = headers.clone();
464 let mut captured = BTreeMap::new();
465
466 for _ in 0..MAX_REDIRECTS {
467 let response = tokio::select! {
468 _ = cancel.cancelled() => return Err(Error::Cancelled),
469 response = self.client.get(current.clone()).headers(headers.clone()).send() => {
470 response.map_err(|error| Error::Backend(format!("request failed: {error}")))?
471 }
472 };
473 capture_headers(&response, &mut captured);
474 let status = response.status();
475 if matches!(
476 status,
477 StatusCode::MOVED_PERMANENTLY
478 | StatusCode::FOUND
479 | StatusCode::SEE_OTHER
480 | StatusCode::TEMPORARY_REDIRECT
481 | StatusCode::PERMANENT_REDIRECT
482 ) {
483 let location = response
484 .headers()
485 .get(LOCATION)
486 .and_then(|value| value.to_str().ok())
487 .ok_or_else(|| {
488 Error::Backend("redirect response missing Location".to_string())
489 })?;
490 let next = current
491 .join(location)
492 .map_err(|error| Error::Backend(format!("invalid redirect URL: {error}")))?;
493 if !same_origin(¤t, &next) && !options.request.allow_cross_origin_auth {
494 for name in options.request.sensitive_header_names() {
495 headers.remove(name.as_str());
496 }
497 headers.remove(AUTHORIZATION);
498 headers.remove(COOKIE);
499 }
500 current = next;
501 continue;
502 }
503 return Ok((response, captured, current));
504 }
505 Err(Error::Backend("too many redirects".to_string()))
506 }
507
508 #[allow(clippy::too_many_arguments)]
509 async fn transfer_segmented(
510 &self,
511 request: &DownloadRequest,
512 file: &ResourceFile,
513 part_name: &Path,
514 key: &str,
515 file_resume: FileResume,
516 task_resume: TaskResume,
517 progress: Arc<AtomicU64>,
518 total: Option<u64>,
519 ) -> Result<FileResume> {
520 let total =
521 total.ok_or_else(|| Error::Backend("segmented transfer without size".to_string()))?;
522 let missing = missing_ranges(total, &file_resume.completed);
523 let mut chunks = Vec::new();
524 for range in missing {
525 chunks.extend(split_range(range, request.options.segment_size));
526 }
527 if chunks.is_empty() {
528 return Ok(file_resume);
529 }
530
531 let mut state = task_resume.clone();
532 state.files.insert(key.to_string(), file_resume.clone());
533 let resume_state = Arc::new(Mutex::new(state));
534 let checkpoint_lock = Arc::new(Mutex::new(()));
535 let queue = Arc::new(std::sync::Mutex::new(VecDeque::from(chunks)));
536 let range_broken = Arc::new(AtomicBool::new(false));
537 let child_cancel = request.cancel.child_token();
538 let if_range = strong_if_range(&file_resume.validator);
539 let workers = request
540 .options
541 .connections
542 .min(queue.lock().map(|queue| queue.len()).unwrap_or(1))
543 .max(1);
544
545 let mut set = tokio::task::JoinSet::new();
546 for _ in 0..workers {
547 let backend = self.clone();
548 let request = request.clone();
549 let file = file.clone();
550 let part_name = part_name.to_path_buf();
551 let key = key.to_string();
552 let queue = queue.clone();
553 let resume_state = resume_state.clone();
554 let checkpoint_lock = checkpoint_lock.clone();
555 let range_broken = range_broken.clone();
556 let progress = progress.clone();
557 let cancel = child_cancel.clone();
558 let if_range = if_range.clone();
559 set.spawn(async move {
560 backend
561 .segment_worker(
562 &request,
563 &file,
564 &part_name,
565 &key,
566 queue,
567 resume_state,
568 checkpoint_lock,
569 range_broken,
570 progress,
571 cancel,
572 total,
573 if_range,
574 )
575 .await
576 });
577 }
578
579 let mut first_error = None;
580 while let Some(joined) = set.join_next().await {
581 match joined {
582 Ok(Ok(())) => {}
583 Ok(Err(FetchError::Cancelled)) => {}
584 Ok(Err(FetchError::RangeBroken)) => {}
585 Ok(Err(FetchError::Fatal(error))) => {
586 if first_error.is_none() {
587 first_error = Some(error);
588 }
589 child_cancel.cancel();
590 }
591 Ok(Err(FetchError::Retryable(message))) => {
592 if first_error.is_none() {
593 first_error = Some(Error::Backend(message));
594 }
595 child_cancel.cancel();
596 }
597 Err(error) => {
598 if first_error.is_none() {
599 first_error = Some(Error::Other(format!("segment worker failed: {error}")));
600 }
601 child_cancel.cancel();
602 }
603 }
604 }
605
606 if let Some(error) = first_error {
607 return Err(error);
608 }
609
610 if range_broken.load(Ordering::SeqCst) {
611 request.events.emit(EventKind::Warning {
612 task_id: Some(request.task_id),
613 message: "server ignored a ranged request; falling back to a single stream"
614 .to_string(),
615 });
616 let file_handle = tokio::fs::OpenOptions::new()
617 .create(true)
618 .write(true)
619 .truncate(false)
620 .open(part_name)
621 .await?;
622 file_handle.set_len(0).await?;
623 file_handle.sync_all().await?;
624 progress.store(0, Ordering::Relaxed);
625 let mut reset = file_resume;
626 reset.completed.clear();
627 reset.normalize();
628 return self
629 .transfer_single(
630 request,
631 file,
632 part_name,
633 key,
634 reset,
635 task_resume,
636 progress,
637 false,
638 )
639 .await;
640 }
641
642 let final_state = resume_state.lock().await.clone();
643 let mut final_resume = final_state
644 .files
645 .get(key)
646 .cloned()
647 .ok_or_else(|| Error::Backend("missing resume state after transfer".to_string()))?;
648 final_resume.completed = vec![shaul_core::ByteRange::new(0, total)?];
649 final_resume.normalize();
650 let mut snapshot = final_state;
651 snapshot.files.insert(key.to_string(), final_resume.clone());
652 request.checkpoint.save(&snapshot).await?;
653 Ok(final_resume)
654 }
655
656 #[allow(clippy::too_many_arguments)]
657 async fn segment_worker(
658 &self,
659 request: &DownloadRequest,
660 file: &ResourceFile,
661 part_name: &Path,
662 key: &str,
663 queue: Arc<std::sync::Mutex<VecDeque<shaul_core::ByteRange>>>,
664 resume_state: Arc<Mutex<TaskResume>>,
665 checkpoint_lock: Arc<Mutex<()>>,
666 range_broken: Arc<AtomicBool>,
667 progress: Arc<AtomicU64>,
668 cancel: CancellationToken,
669 total: u64,
670 if_range: Option<String>,
671 ) -> std::result::Result<(), FetchError> {
672 let mut part = tokio::fs::OpenOptions::new()
673 .write(true)
674 .open(part_name)
675 .await
676 .map_err(|error| FetchError::Fatal(Error::Io(error)))?;
677 loop {
678 if cancel.is_cancelled() || range_broken.load(Ordering::SeqCst) {
679 break;
680 }
681 let range = {
682 let mut queue = queue
683 .lock()
684 .map_err(|_| FetchError::Fatal(Error::Other("work queue poisoned".into())))?;
685 queue.pop_front()
686 };
687 let Some(range) = range else {
688 break;
689 };
690 match self
691 .fetch_range(
692 request,
693 file,
694 range,
695 &mut part,
696 &progress,
697 &cancel,
698 total,
699 if_range.as_deref(),
700 )
701 .await
702 {
703 Ok(()) => {
704 let _guard = checkpoint_lock.lock().await;
705 let snapshot = {
706 let mut state = resume_state.lock().await;
707 if let Some(entry) = state.files.get_mut(key) {
708 entry.completed.push(range);
709 entry.normalize();
710 }
711 state.clone()
712 };
713 request
714 .checkpoint
715 .save(&snapshot)
716 .await
717 .map_err(FetchError::Fatal)?;
718 }
719 Err(FetchError::Cancelled) => break,
720 Err(FetchError::RangeBroken) => {
721 range_broken.store(true, Ordering::SeqCst);
722 cancel.cancel();
723 break;
724 }
725 Err(error) => {
726 cancel.cancel();
727 return Err(error);
728 }
729 }
730 }
731 part.flush()
732 .await
733 .map_err(|error| FetchError::Fatal(Error::Io(error)))?;
734 Ok(())
735 }
736
737 #[allow(clippy::too_many_arguments)]
738 async fn fetch_range(
739 &self,
740 request: &DownloadRequest,
741 file: &ResourceFile,
742 range: shaul_core::ByteRange,
743 part: &mut tokio::fs::File,
744 progress: &AtomicU64,
745 cancel: &CancellationToken,
746 total: u64,
747 if_range: Option<&str>,
748 ) -> std::result::Result<(), FetchError> {
749 let mut attempt = 0u32;
750 loop {
751 if cancel.is_cancelled() {
752 return Err(FetchError::Cancelled);
753 }
754 let range_header = format!("bytes={}-{}", range.start, range.end - 1);
755 let headers = build_headers(
756 &request.options,
757 &file.headers,
758 Some(&range_header),
759 if_range,
760 )
761 .map_err(FetchError::Fatal)?;
762 let sent = self
763 .send_follow(&file.url, &headers, &request.options, cancel)
764 .await;
765 let (response, captured, _) = match sent {
766 Err(Error::Cancelled) => return Err(FetchError::Cancelled),
767 Err(error) => {
768 if attempt >= request.options.retries {
769 return Err(FetchError::Fatal(error));
770 }
771 sleep_or_cancel(backoff(attempt), cancel)
772 .await
773 .map_err(FetchError::Fatal)?;
774 attempt += 1;
775 continue;
776 }
777 Ok(response) => response,
778 };
779 let status = response.status();
780 let retry_after = retry_after(response.headers());
781
782 if status == StatusCode::RANGE_NOT_SATISFIABLE {
783 return Err(FetchError::Fatal(Error::Backend(format!(
784 "range {} is not satisfiable",
785 range_header
786 ))));
787 }
788 if status.is_server_error()
789 || status == StatusCode::REQUEST_TIMEOUT
790 || status == StatusCode::TOO_MANY_REQUESTS
791 {
792 drop(response);
793 if attempt >= request.options.retries {
794 return Err(FetchError::Fatal(http_error(status, &captured)));
795 }
796 sleep_or_cancel(retry_after.unwrap_or_else(|| backoff(attempt)), cancel)
797 .await
798 .map_err(FetchError::Fatal)?;
799 attempt += 1;
800 continue;
801 }
802
803 let content_range = captured
804 .get("content-range")
805 .and_then(|value| parse_content_range(value));
806 if status == StatusCode::OK && content_range.is_none() {
807 return Err(FetchError::RangeBroken);
808 }
809 let is_partial = status == StatusCode::PARTIAL_CONTENT
810 || (status == StatusCode::OK && content_range.is_some());
811 if !is_partial {
812 return Err(FetchError::Fatal(http_error(status, &captured)));
813 }
814 let parsed = content_range.ok_or_else(|| {
815 FetchError::Fatal(Error::Backend(
816 "ranged response missing a valid Content-Range".to_string(),
817 ))
818 })?;
819 if parsed.0 != range.start || parsed.1 + 1 != range.end {
820 return Err(FetchError::Fatal(Error::Backend(format!(
821 "server returned range {}-{}, expected {}-{}",
822 parsed.0,
823 parsed.1,
824 range.start,
825 range.end - 1
826 ))));
827 }
828 if let Some(server_total) = parsed.2
829 && server_total != total
830 {
831 return Err(FetchError::Fatal(Error::Backend(format!(
832 "file size changed from {total} to {server_total}"
833 ))));
834 }
835
836 part.seek(SeekFrom::Start(range.start))
837 .await
838 .map_err(|error| FetchError::Fatal(Error::Io(error)))?;
839 let mut written = 0u64;
840 let mut stream = response.bytes_stream();
841 let retry_message;
842 loop {
843 let next = tokio::select! {
844 _ = cancel.cancelled() => return Err(FetchError::Cancelled),
845 next = tokio::time::timeout(request.options.timeout, stream.next()) => next,
846 };
847 match next {
848 Ok(Some(Ok(chunk))) => {
849 if written + chunk.len() as u64 > range.len() {
850 retry_message =
851 Some("server sent more data than requested".to_string());
852 break;
853 }
854 part.write_all(&chunk)
855 .await
856 .map_err(|error| FetchError::Fatal(Error::Io(error)))?;
857 written += chunk.len() as u64;
858 progress.fetch_add(chunk.len() as u64, Ordering::Relaxed);
859 }
860 Ok(Some(Err(error))) => {
861 retry_message = Some(format!("stream error: {error}"));
862 break;
863 }
864 Ok(None) => {
865 if written == range.len() {
866 return Ok(());
867 }
868 retry_message = Some(format!(
869 "short ranged response: {written} of {} bytes",
870 range.len()
871 ));
872 break;
873 }
874 Err(_) => {
875 retry_message = Some("read timed out".to_string());
876 break;
877 }
878 }
879 }
880
881 let current = progress.load(Ordering::Relaxed);
884 progress.fetch_sub(written.min(current), Ordering::Relaxed);
885 if attempt >= request.options.retries {
886 return Err(FetchError::Retryable(
887 retry_message.unwrap_or_else(|| "range transfer failed".to_string()),
888 ));
889 }
890 sleep_or_cancel(backoff(attempt), cancel)
891 .await
892 .map_err(FetchError::Fatal)?;
893 attempt += 1;
894 }
895 }
896
897 #[allow(clippy::too_many_arguments)]
898 async fn transfer_single(
899 &self,
900 request: &DownloadRequest,
901 file: &ResourceFile,
902 part_name: &Path,
903 key: &str,
904 mut file_resume: FileResume,
905 mut task_resume: TaskResume,
906 progress: Arc<AtomicU64>,
907 range_supported: bool,
908 ) -> Result<FileResume> {
909 let mut offset = if request.options.resume && range_supported {
910 contiguous_end(&file_resume.completed)
911 } else {
912 0
913 };
914 if file_resume
915 .total_size
916 .map(|total| offset >= total)
917 .unwrap_or(false)
918 {
919 return Ok(file_resume);
920 }
921
922 let mut part = tokio::fs::OpenOptions::new()
923 .create(true)
924 .write(true)
925 .truncate(false)
926 .open(part_name)
927 .await?;
928 if offset == 0 {
929 part.set_len(0).await?;
930 }
931 part.seek(SeekFrom::Start(offset)).await?;
932
933 let mut attempt = 0u32;
934 loop {
935 if request.cancel.is_cancelled() {
936 checkpoint_single(request, &mut task_resume, key, &mut file_resume, offset).await?;
937 return Err(Error::Cancelled);
938 }
939 let range = if offset > 0 && range_supported {
940 Some(format!("bytes={offset}-"))
941 } else {
942 None
943 };
944 let if_range = if offset > 0 {
945 strong_if_range(&file_resume.validator)
946 } else {
947 None
948 };
949 let headers = build_headers(
950 &request.options,
951 &file.headers,
952 range.as_deref(),
953 if_range.as_deref(),
954 )?;
955 let sent = self
956 .send_follow(&file.url, &headers, &request.options, &request.cancel)
957 .await;
958 let (response, captured, _) = match sent {
959 Err(Error::Cancelled) => {
960 checkpoint_single(request, &mut task_resume, key, &mut file_resume, offset)
961 .await?;
962 return Err(Error::Cancelled);
963 }
964 Err(error) => {
965 checkpoint_single(request, &mut task_resume, key, &mut file_resume, offset)
966 .await?;
967 if attempt >= request.options.retries {
968 return Err(error);
969 }
970 sleep_or_cancel(backoff(attempt), &request.cancel).await?;
971 attempt += 1;
972 continue;
973 }
974 Ok(response) => response,
975 };
976 let status = response.status();
977 let retry_after = retry_after(response.headers());
978
979 let content_range = captured
980 .get("content-range")
981 .and_then(|value| parse_content_range(value));
982 if status.is_server_error()
983 || status == StatusCode::REQUEST_TIMEOUT
984 || status == StatusCode::TOO_MANY_REQUESTS
985 {
986 drop(response);
987 checkpoint_single(request, &mut task_resume, key, &mut file_resume, offset).await?;
988 if attempt >= request.options.retries {
989 return Err(http_error(status, &captured));
990 }
991 sleep_or_cancel(
992 retry_after.unwrap_or_else(|| backoff(attempt)),
993 &request.cancel,
994 )
995 .await?;
996 attempt += 1;
997 continue;
998 }
999
1000 let is_partial = status == StatusCode::PARTIAL_CONTENT
1001 || (status == StatusCode::OK && content_range.is_some());
1002 if is_partial {
1003 if let Some((start, _, _)) = content_range
1004 && offset > 0
1005 && start != offset
1006 {
1007 return Err(Error::Backend(format!(
1008 "server resumed at {start}, expected {offset}"
1009 )));
1010 }
1011 } else if status == StatusCode::OK {
1012 if offset > 0 {
1013 part.set_len(0).await?;
1014 part.seek(SeekFrom::Start(0)).await?;
1015 offset = 0;
1016 progress.store(0, Ordering::Relaxed);
1017 }
1018 } else {
1019 return Err(http_error(status, &captured));
1020 }
1021
1022 part.seek(SeekFrom::Start(offset)).await?;
1023 let mut stream = response.bytes_stream();
1024 let mut since_checkpoint = 0u64;
1025 let retry_message;
1026 loop {
1027 let next = tokio::select! {
1028 _ = request.cancel.cancelled() => {
1029 checkpoint_single(request, &mut task_resume, key, &mut file_resume, offset).await?;
1030 part.sync_all().await?;
1031 return Err(Error::Cancelled);
1032 }
1033 next = tokio::time::timeout(request.options.timeout, stream.next()) => next,
1034 };
1035 match next {
1036 Ok(Some(Ok(chunk))) => {
1037 if let Some(total) = file_resume.total_size
1038 && offset + chunk.len() as u64 > total
1039 {
1040 return Err(Error::Backend(
1041 "server sent more data than the known file size".to_string(),
1042 ));
1043 }
1044 part.write_all(&chunk).await?;
1045 offset += chunk.len() as u64;
1046 progress.fetch_add(chunk.len() as u64, Ordering::Relaxed);
1047 since_checkpoint += chunk.len() as u64;
1048 if since_checkpoint >= request.options.segment_size {
1049 checkpoint_single(
1050 request,
1051 &mut task_resume,
1052 key,
1053 &mut file_resume,
1054 offset,
1055 )
1056 .await?;
1057 since_checkpoint = 0;
1058 }
1059 }
1060 Ok(Some(Err(error))) => {
1061 retry_message = Some(format!("stream error: {error}"));
1062 break;
1063 }
1064 Ok(None) => {
1065 if let Some(total) = file_resume.total_size
1066 && offset < total
1067 {
1068 retry_message =
1069 Some(format!("short response: {offset} of {total} bytes"));
1070 break;
1071 }
1072 part.flush().await?;
1073 part.sync_all().await?;
1074 file_resume.total_size = file_resume.total_size.or(Some(offset));
1075 file_resume.completed = if offset > 0 {
1076 vec![shaul_core::ByteRange::new(0, offset)?]
1077 } else {
1078 Vec::new()
1079 };
1080 file_resume.normalize();
1081 task_resume
1082 .files
1083 .insert(key.to_string(), file_resume.clone());
1084 request.checkpoint.save(&task_resume).await?;
1085 return Ok(file_resume);
1086 }
1087 Err(_) => {
1088 retry_message = Some("read timed out".to_string());
1089 break;
1090 }
1091 }
1092 }
1093
1094 part.flush().await?;
1095 checkpoint_single(request, &mut task_resume, key, &mut file_resume, offset).await?;
1096 if attempt >= request.options.retries {
1097 return Err(Error::Backend(
1098 retry_message.unwrap_or_else(|| "download failed".to_string()),
1099 ));
1100 }
1101 sleep_or_cancel(backoff(attempt), &request.cancel).await?;
1102 attempt += 1;
1103 }
1104 }
1105}
1106
1107#[async_trait]
1108impl DownloadBackend for HttpBackend {
1109 fn name(&self) -> &'static str {
1110 "http"
1111 }
1112
1113 fn capabilities(&self) -> BackendCapabilities {
1114 BackendCapabilities::new(true, true, true)
1115 }
1116
1117 fn supports(&self, resource: &Resource) -> bool {
1118 matches!(&resource.kind, ResourceKind::Http)
1119 }
1120
1121 async fn download(&self, request: DownloadRequest) -> Result<DownloadOutcome> {
1122 let mut task_resume = request.resume.clone().unwrap_or_default();
1123 let mut outcomes = Vec::new();
1124 for file in &request.resource.files {
1125 if request.cancel.is_cancelled() {
1126 return Err(Error::Cancelled);
1127 }
1128 let outcome = self.download_file(&request, file, &mut task_resume).await?;
1129 outcomes.push(outcome);
1130 request.checkpoint.save(&task_resume).await?;
1131 }
1132 Ok(DownloadOutcome::Completed(outcomes))
1133 }
1134}
1135
1136struct Probe {
1137 final_url: Url,
1138 total_size: Option<u64>,
1139 range_supported: bool,
1140 filename: Option<String>,
1141 etag: Option<String>,
1142 last_modified: Option<String>,
1143 checksum: Option<Checksum>,
1144 headers: BTreeMap<String, String>,
1145}
1146
1147impl Probe {
1148 fn validator(&self) -> Validator {
1149 Validator {
1150 etag: self.etag.clone(),
1151 last_modified: self.last_modified.clone(),
1152 size: self.total_size,
1153 }
1154 }
1155}
1156
1157#[derive(Debug)]
1158enum FetchError {
1159 RangeBroken,
1160 Cancelled,
1161 Retryable(String),
1162 Fatal(Error),
1163}
1164
1165fn build_headers(
1166 options: &shaul_core::ResolvedOptions,
1167 file_headers: &[Header],
1168 range: Option<&str>,
1169 if_range: Option<&str>,
1170) -> Result<HeaderMap> {
1171 let mut headers = HeaderMap::new();
1172 for header in &options.request.headers {
1173 if !is_reserved(&header.name) {
1174 insert_header(
1175 &mut headers,
1176 &header.name,
1177 &shaul_core::expand_env(&header.value)?,
1178 )?;
1179 }
1180 }
1181 for header in file_headers {
1182 if !is_reserved(&header.name) {
1183 insert_header(
1184 &mut headers,
1185 &header.name,
1186 &shaul_core::expand_env(&header.value)?,
1187 )?;
1188 }
1189 }
1190
1191 if !headers.contains_key(AUTHORIZATION)
1192 && let Some(auth) = &options.request.auth
1193 {
1194 match auth {
1195 shaul_core::Auth::Bearer { token } => {
1196 insert_header(
1197 &mut headers,
1198 "authorization",
1199 &format!("Bearer {}", shaul_core::expand_env(token)?),
1200 )?;
1201 }
1202 shaul_core::Auth::Basic { username, password } => {
1203 let encoded = base64::engine::general_purpose::STANDARD.encode(format!(
1204 "{}:{}",
1205 shaul_core::expand_env(username)?,
1206 shaul_core::expand_env(password)?
1207 ));
1208 insert_header(&mut headers, "authorization", &format!("Basic {encoded}"))?;
1209 }
1210 shaul_core::Auth::Header(header) => {
1211 insert_header(
1212 &mut headers,
1213 &header.name,
1214 &shaul_core::expand_env(&header.value)?,
1215 )?;
1216 }
1217 }
1218 }
1219
1220 headers.insert(ACCEPT_ENCODING, HeaderValue::from_static("identity"));
1221 insert_header(&mut headers, "user-agent", &options.user_agent)?;
1222 if let Some(range) = range {
1223 insert_header(&mut headers, "range", range)?;
1224 }
1225 if let Some(if_range) = if_range {
1226 insert_header(&mut headers, "if-range", if_range)?;
1227 }
1228 Ok(headers)
1229}
1230
1231fn insert_header(headers: &mut HeaderMap, name: &str, value: &str) -> Result<()> {
1232 let name = HeaderName::from_bytes(name.as_bytes())
1233 .map_err(|_| Error::Config(format!("invalid header name: {name}")))?;
1234 let value = HeaderValue::from_str(value)
1235 .map_err(|_| Error::Config(format!("invalid value for header: {name}")))?;
1236 headers.insert(name, value);
1237 Ok(())
1238}
1239
1240fn is_reserved(name: &str) -> bool {
1241 RESERVED_HEADERS
1242 .iter()
1243 .any(|reserved| name.eq_ignore_ascii_case(reserved))
1244}
1245
1246fn capture_headers(response: &Response, captured: &mut BTreeMap<String, String>) {
1247 for name in CAPTURE_HEADERS {
1248 if let Some(value) = response.headers().get(*name)
1249 && let Ok(value) = value.to_str()
1250 {
1251 captured.insert((*name).to_string(), value.to_string());
1252 }
1253 }
1254}
1255
1256fn redacted_url(url: &Url) -> String {
1257 let mut url = url.clone();
1258 url.set_query(None);
1259 url.set_fragment(None);
1260 url.to_string()
1261}
1262
1263fn same_origin(left: &Url, right: &Url) -> bool {
1264 left.scheme() == right.scheme()
1265 && left.host_str() == right.host_str()
1266 && left.port_or_known_default() == right.port_or_known_default()
1267}
1268
1269fn parse_content_range(value: &str) -> Option<(u64, u64, Option<u64>)> {
1270 let rest = value.trim().strip_prefix("bytes")?.trim();
1271 let (range, total) = rest.split_once('/')?;
1272 let total = if total.trim() == "*" {
1273 None
1274 } else {
1275 Some(total.trim().parse::<u64>().ok()?)
1276 };
1277 if range.trim() == "*" {
1278 return Some((0, 0, total));
1279 }
1280 let (start, end) = range.split_once('-')?;
1281 Some((
1282 start.trim().parse::<u64>().ok()?,
1283 end.trim().parse::<u64>().ok()?,
1284 total,
1285 ))
1286}
1287
1288fn parse_u64(value: &str) -> Option<u64> {
1289 value.trim().parse::<u64>().ok()
1290}
1291
1292fn retry_after(headers: &HeaderMap) -> Option<Duration> {
1293 headers
1294 .get("retry-after")
1295 .and_then(|value| value.to_str().ok())
1296 .and_then(|value| value.trim().parse::<u64>().ok())
1297 .map(|seconds| Duration::from_secs(seconds.min(60)))
1298}
1299
1300fn backoff(attempt: u32) -> Duration {
1301 Duration::from_millis(250u64.saturating_mul(1u64 << attempt.min(5)))
1302}
1303
1304async fn sleep_or_cancel(duration: Duration, cancel: &CancellationToken) -> Result<()> {
1305 tokio::select! {
1306 _ = cancel.cancelled() => Err(Error::Cancelled),
1307 _ = tokio::time::sleep(duration) => Ok(()),
1308 }
1309}
1310
1311fn http_error(status: StatusCode, captured: &BTreeMap<String, String>) -> Error {
1312 let code = captured
1313 .get("x-error-code")
1314 .map(|code| format!(" ({code})"))
1315 .unwrap_or_default();
1316 Error::Backend(format!("HTTP {status}{code}"))
1317}
1318
1319fn parse_digest(captured: &BTreeMap<String, String>) -> Option<Checksum> {
1320 if let Some(value) = captured.get("digest") {
1321 for part in value.split(',') {
1322 let Some((algorithm, encoded)) = part.trim().split_once('=') else {
1323 continue;
1324 };
1325 let algorithm = match algorithm.trim().to_ascii_lowercase().as_str() {
1326 "sha-256" | "sha256" => ChecksumAlgorithm::Sha256,
1327 "sha-512" | "sha512" => ChecksumAlgorithm::Sha512,
1328 "sha" | "sha-1" | "sha1" => ChecksumAlgorithm::Sha1,
1329 "md5" => ChecksumAlgorithm::Md5,
1330 _ => continue,
1331 };
1332 if let Ok(bytes) = base64::engine::general_purpose::STANDARD.decode(encoded.trim())
1333 && let Ok(checksum) = Checksum::new(algorithm, hex::encode(bytes))
1334 {
1335 return Some(checksum);
1336 }
1337 }
1338 }
1339 if let Some(value) = captured.get("content-md5")
1340 && let Ok(bytes) = base64::engine::general_purpose::STANDARD.decode(value.trim())
1341 && let Ok(checksum) = Checksum::new(ChecksumAlgorithm::Md5, hex::encode(bytes))
1342 {
1343 return Some(checksum);
1344 }
1345 None
1346}
1347
1348fn strong_if_range(validator: &Validator) -> Option<String> {
1349 if let Some(etag) = &validator.etag
1350 && !etag.starts_with("W/")
1351 {
1352 return Some(etag.clone());
1353 }
1354 validator.last_modified.clone()
1355}
1356
1357fn contiguous_end(completed: &[shaul_core::ByteRange]) -> u64 {
1358 let merged = merge_ranges(completed.to_vec());
1359 if merged.len() == 1 && merged[0].start == 0 {
1360 merged[0].end
1361 } else {
1362 0
1363 }
1364}
1365
1366fn manifest_path(destination: &Path, value: &str) -> PathBuf {
1367 let path = Path::new(value);
1368 if path.is_absolute() {
1369 path.to_path_buf()
1370 } else {
1371 destination.join(path)
1372 }
1373}
1374
1375fn path_for_manifest(destination: &Path, path: &Path) -> String {
1376 match path.strip_prefix(destination) {
1377 Ok(relative) => relative.to_string_lossy().replace('\\', "/"),
1378 Err(_) => path.to_string_lossy().to_string(),
1379 }
1380}
1381
1382fn relative_path(destination: &Path, path: &Path) -> String {
1383 match path.strip_prefix(destination) {
1384 Ok(relative) => relative.to_string_lossy().replace('\\', "/"),
1385 Err(_) => path
1386 .file_name()
1387 .map(|name| name.to_string_lossy().to_string())
1388 .unwrap_or_else(|| "download".to_string()),
1389 }
1390}
1391
1392fn resolve_final_path(
1393 request: &DownloadRequest,
1394 file: &ResourceFile,
1395 probe_filename: Option<&str>,
1396 default_name: &str,
1397) -> Result<(PathBuf, String)> {
1398 let single = request.resource.files.len() == 1;
1399 let response_name = if file.use_response_filename {
1400 probe_filename.map(sanitize_file_name)
1401 } else {
1402 None
1403 };
1404 let name = response_name
1405 .clone()
1406 .unwrap_or_else(|| default_name.to_string());
1407
1408 if let Some(output) = &request.options.output {
1409 if single {
1410 if output.is_dir() {
1411 let relative = if file.use_response_filename {
1412 name.clone()
1413 } else {
1414 file.path.clone()
1415 };
1416 let path = output.join(relative.replace('/', std::path::MAIN_SEPARATOR_STR));
1417 return Ok((path.clone(), relative_path(&request.destination, &path)));
1418 }
1419 return Ok((output.clone(), relative_path(&request.destination, output)));
1420 }
1421 let path = output.join(file.path.replace('/', std::path::MAIN_SEPARATOR_STR));
1422 return Ok((path.clone(), relative_path(&request.destination, &path)));
1423 }
1424
1425 let relative = if single && file.use_response_filename {
1426 name
1427 } else {
1428 file.path.clone()
1429 };
1430 let path = request
1431 .destination
1432 .join(relative.replace('/', std::path::MAIN_SEPARATOR_STR));
1433 Ok((path.clone(), relative_path(&request.destination, &path)))
1434}
1435
1436async fn apply_conflict_policy(
1437 path: PathBuf,
1438 options: &shaul_core::ResolvedOptions,
1439) -> Result<PathBuf> {
1440 if !tokio::fs::try_exists(&path).await? {
1441 return Ok(path);
1442 }
1443 if options.overwrite {
1444 remove_if_exists(&path).await?;
1445 return Ok(path);
1446 }
1447 if options.rename_on_conflict {
1448 return unique_path(&path).await;
1449 }
1450 Err(Error::Backend(format!(
1451 "destination already exists: {} (use --force or --rename)",
1452 path.display()
1453 )))
1454}
1455
1456async fn unique_path(path: &Path) -> Result<PathBuf> {
1457 let stem = path
1458 .file_stem()
1459 .map(|stem| stem.to_string_lossy().to_string())
1460 .unwrap_or_else(|| "download".to_string());
1461 let extension = path
1462 .extension()
1463 .map(|extension| format!(".{}", extension.to_string_lossy()))
1464 .unwrap_or_default();
1465 for index in 1..10_000 {
1466 let candidate = path.with_file_name(format!("{stem}-{index}{extension}"));
1467 if !tokio::fs::try_exists(&candidate).await? {
1468 return Ok(candidate);
1469 }
1470 }
1471 Err(Error::Backend(
1472 "could not find a free destination name".to_string(),
1473 ))
1474}
1475
1476fn now_ms() -> u64 {
1477 SystemTime::now()
1478 .duration_since(UNIX_EPOCH)
1479 .map(|duration| duration.as_millis() as u64)
1480 .unwrap_or(0)
1481}
1482
1483async fn ensure_parent(path: &Path) -> Result<()> {
1484 if let Some(parent) = path.parent()
1485 && !parent.as_os_str().is_empty()
1486 {
1487 tokio::fs::create_dir_all(parent).await?;
1488 }
1489 Ok(())
1490}
1491
1492async fn remove_if_exists(path: &Path) -> Result<()> {
1493 match tokio::fs::remove_file(path).await {
1494 Ok(()) => Ok(()),
1495 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
1496 Err(error) => Err(Error::Io(error)),
1497 }
1498}
1499
1500async fn checkpoint_single(
1501 request: &DownloadRequest,
1502 task_resume: &mut TaskResume,
1503 key: &str,
1504 file_resume: &mut FileResume,
1505 offset: u64,
1506) -> Result<()> {
1507 file_resume.completed = if offset > 0 {
1508 vec![shaul_core::ByteRange::new(0, offset)?]
1509 } else {
1510 Vec::new()
1511 };
1512 file_resume.normalize();
1513 task_resume
1514 .files
1515 .insert(key.to_string(), file_resume.clone());
1516 request.checkpoint.save(task_resume).await
1517}
1518
1519fn spawn_progress_reporter(
1520 events: EventSink,
1521 task_id: TaskId,
1522 file: String,
1523 total: Option<u64>,
1524 downloaded: Arc<AtomicU64>,
1525 stop: CancellationToken,
1526) -> tokio::task::JoinHandle<()> {
1527 tokio::spawn(async move {
1528 let mut ticker = tokio::time::interval(Duration::from_millis(100));
1529 ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
1530 loop {
1531 tokio::select! {
1532 _ = ticker.tick() => {
1533 events.emit(EventKind::Progress {
1534 task_id,
1535 file: file.clone(),
1536 downloaded_bytes: downloaded.load(Ordering::Relaxed),
1537 total_bytes: total,
1538 });
1539 }
1540 _ = stop.cancelled() => {
1541 events.emit(EventKind::Progress {
1542 task_id,
1543 file,
1544 downloaded_bytes: downloaded.load(Ordering::Relaxed),
1545 total_bytes: total,
1546 });
1547 break;
1548 }
1549 }
1550 }
1551 })
1552}