1use std::collections::{BTreeMap, VecDeque};
2use std::path::{Path, PathBuf};
3use std::sync::Arc;
4use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
5use std::time::{Duration, SystemTime, UNIX_EPOCH};
6
7use async_trait::async_trait;
8use base64::Engine;
9use futures_util::StreamExt;
10use reqwest::header::{
11 ACCEPT_ENCODING, AUTHORIZATION, COOKIE, HeaderMap, HeaderName, HeaderValue, LOCATION,
12};
13use reqwest::redirect::Policy;
14use reqwest::{Client, Response, StatusCode};
15use shaul_core::{
16 BackendCapabilities, Checksum, ChecksumAlgorithm, Config, DownloadBackend, DownloadOutcome,
17 DownloadRequest, Error, EventKind, EventSink, FileOutcome, FileResume, Header, Resource,
18 ResourceFile, ResourceKind, Result, TaskId, TaskResume, Validator, VerifyMode, completed_bytes,
19 merge_ranges, missing_ranges, part_path, sanitize_file_name, split_range,
20};
21use tokio::io::{AsyncSeekExt, AsyncWriteExt, SeekFrom};
22use tokio::sync::Mutex;
23use tokio_util::sync::CancellationToken;
24use url::Url;
25
26use crate::filename::parse_content_disposition;
27
28const MAX_REDIRECTS: usize = 10;
29
30const CAPTURE_HEADERS: &[&str] = &[
31 "content-disposition",
32 "etag",
33 "last-modified",
34 "content-length",
35 "accept-ranges",
36 "content-range",
37 "digest",
38 "content-md5",
39 "x-linked-etag",
40 "x-linked-size",
41 "x-repo-commit",
42 "x-xet-hash",
43 "x-xet-token-expiration",
44 "x-xet-refresh-route",
45 "x-error-code",
46];
47
48const RESERVED_HEADERS: &[&str] = &[
49 "range",
50 "if-range",
51 "host",
52 "content-length",
53 "accept-encoding",
54 "connection",
55 "transfer-encoding",
56];
57
58#[derive(Clone)]
60pub struct HttpBackend {
61 client: Client,
62}
63
64impl HttpBackend {
65 pub fn new(config: &Config) -> Result<Self> {
68 let client = Client::builder()
69 .redirect(Policy::none())
70 .connect_timeout(Duration::from_secs(
71 config.network.connect_timeout_secs.max(1),
72 ))
73 .pool_max_idle_per_host(config.download.per_host.max(1))
74 .user_agent(config.network.user_agent.clone())
75 .build()
76 .map_err(|error| Error::Backend(format!("failed to build HTTP client: {error}")))?;
77 Ok(Self { client })
78 }
79
80 pub async fn download_file(
84 &self,
85 request: &DownloadRequest,
86 file: &ResourceFile,
87 task_resume: &mut TaskResume,
88 ) -> Result<FileOutcome> {
89 let key = file.url.as_str().to_string();
90
91 if let Some(stored) = task_resume.files.get(&key).cloned()
92 && stored.is_complete()
93 {
94 let final_path = manifest_path(&request.destination, &stored.final_path);
95 let part = manifest_path(&request.destination, &stored.part_path);
96 if tokio::fs::try_exists(&final_path).await? {
97 let size = tokio::fs::metadata(&final_path).await?.len();
98 let relative = relative_path(&request.destination, &final_path);
99 request.events.emit(EventKind::FileStarted {
100 task_id: request.task_id,
101 file: relative.clone(),
102 total_bytes: Some(size),
103 });
104 request.events.emit(EventKind::Progress {
105 task_id: request.task_id,
106 file: relative.clone(),
107 downloaded_bytes: size,
108 total_bytes: Some(size),
109 });
110 return Ok(FileOutcome::new(
111 final_path,
112 relative,
113 size,
114 stored.checksum.or(request.options.checksum.clone()),
115 false,
116 ));
117 }
118 if tokio::fs::try_exists(&part).await? {
119 ensure_parent(&final_path).await?;
120 remove_if_exists(&final_path).await?;
121 tokio::fs::rename(&part, &final_path).await?;
122 let size = tokio::fs::metadata(&final_path).await?.len();
123 let relative = relative_path(&request.destination, &final_path);
124 request.events.emit(EventKind::FileStarted {
125 task_id: request.task_id,
126 file: relative.clone(),
127 total_bytes: Some(size),
128 });
129 return Ok(FileOutcome::new(
130 final_path,
131 relative,
132 size,
133 stored.checksum.or(request.options.checksum.clone()),
134 false,
135 ));
136 }
137 }
138
139 let probe = self.probe(request, file).await?;
140 let total = probe.total_size;
141 let validator = probe.validator();
142 let checksum = request
143 .options
144 .checksum
145 .clone()
146 .or_else(|| probe.checksum.clone())
147 .or_else(|| file.checksum.clone());
148 if request.options.verify == VerifyMode::Require && checksum.is_none() {
149 return Err(Error::ChecksumMissing);
150 }
151
152 let default_name = file
153 .path
154 .rsplit('/')
155 .next()
156 .map(sanitize_file_name)
157 .unwrap_or_else(|| "download".to_string());
158 let (mut final_path, relative) =
159 resolve_final_path(request, file, probe.filename.as_deref(), &default_name)?;
160
161 let stored = task_resume.files.get(&key).cloned();
162 let final_for_manifest = path_for_manifest(&request.destination, &final_path);
163 let stored_matches_path = stored
164 .as_ref()
165 .map(|stored| stored.final_path == final_for_manifest)
166 .unwrap_or(false);
167 let resume_valid = request.options.resume
168 && stored
169 .as_ref()
170 .map(|stored| {
171 stored_matches_path
172 && !stored.validator.is_empty()
173 && stored.validator.matches(&validator)
174 && match (stored.total_size, total) {
175 (Some(old), Some(new)) => old == new,
176 _ => true,
177 }
178 })
179 .unwrap_or(false);
180
181 if stored.is_some() && !resume_valid {
182 request.events.emit(EventKind::Warning {
183 task_id: Some(request.task_id),
184 message: "resume state is no longer valid; restarting this file".to_string(),
185 });
186 }
187
188 final_path = apply_conflict_policy(final_path, &request.options).await?;
189 let part_name = part_path(&final_path);
190 let part_for_manifest = path_for_manifest(&request.destination, &part_name);
191
192 let mut file_resume = FileResume {
193 url: key.clone(),
194 final_path: path_for_manifest(&request.destination, &final_path),
195 part_path: part_for_manifest.clone(),
196 validator: validator.clone(),
197 total_size: total,
198 segment_size: request.options.segment_size,
199 completed: if resume_valid {
200 stored
201 .as_ref()
202 .map(|stored| stored.completed.clone())
203 .unwrap_or_default()
204 } else {
205 Vec::new()
206 },
207 checksum: checksum.clone(),
208 metadata: {
209 let mut metadata = probe.headers.clone();
210 metadata.insert("final-url".to_string(), redacted_url(&probe.final_url));
211 metadata
212 },
213 updated_at_ms: now_ms(),
214 };
215
216 let existing_part_len = tokio::fs::metadata(&part_name)
218 .await
219 .ok()
220 .map(|metadata| metadata.len());
221 if !resume_valid {
222 file_resume.completed.clear();
223 } else if let Some(len) = existing_part_len {
224 let limit = total.map(|total| total.min(len)).unwrap_or(len);
225 file_resume.completed = shaul_core::range::clamp_to_len(&file_resume.completed, limit);
226 } else {
227 file_resume.completed.clear();
228 }
229 file_resume.normalize();
230
231 ensure_parent(&final_path).await?;
232 if let Some(total) = total {
233 let file = tokio::fs::OpenOptions::new()
234 .create(true)
235 .write(true)
236 .truncate(false)
237 .open(&part_name)
238 .await?;
239 if !resume_valid {
240 file.set_len(0).await?;
241 }
242 file.set_len(total).await?;
243 file.sync_all().await?;
244 } else if !resume_valid {
245 tokio::fs::OpenOptions::new()
246 .create(true)
247 .write(true)
248 .truncate(true)
249 .open(&part_name)
250 .await?;
251 } else if !tokio::fs::try_exists(&part_name).await? {
252 tokio::fs::File::create(&part_name).await?;
253 file_resume.completed.clear();
254 }
255
256 task_resume.files.insert(key.clone(), file_resume.clone());
257 request.checkpoint.save(task_resume).await?;
258
259 let initial = completed_bytes(&file_resume.completed);
260 let progress = Arc::new(AtomicU64::new(initial));
261 request.events.emit(EventKind::FileStarted {
262 task_id: request.task_id,
263 file: relative.clone(),
264 total_bytes: total,
265 });
266
267 let progress_stop = CancellationToken::new();
268 let reporter = spawn_progress_reporter(
269 request.events.clone(),
270 request.task_id,
271 relative.clone(),
272 total,
273 progress.clone(),
274 progress_stop.clone(),
275 );
276
277 let transfer_result = if let Some(0) = total {
278 task_resume.files.insert(key.clone(), file_resume.clone());
280 request.checkpoint.save(task_resume).await?;
281 Ok(file_resume.clone())
282 } else if probe.range_supported
283 && total.map(|total| total > 0).unwrap_or(false)
284 && request.options.connections > 1
285 {
286 self.transfer_segmented(
287 request,
288 file,
289 &part_name,
290 &key,
291 file_resume.clone(),
292 task_resume.clone(),
293 progress.clone(),
294 total,
295 )
296 .await
297 } else {
298 self.transfer_single(
299 request,
300 file,
301 &part_name,
302 &key,
303 file_resume.clone(),
304 task_resume.clone(),
305 progress.clone(),
306 probe.range_supported,
307 )
308 .await
309 };
310
311 progress_stop.cancel();
312 let _ = reporter.await;
313
314 let mut final_resume = match transfer_result {
315 Ok(resume) => resume,
316 Err(error) => return Err(error),
317 };
318
319 if let Some(total) = final_resume.total_size
320 && total > 0
321 {
322 final_resume.completed = vec![shaul_core::ByteRange::new(0, total)?];
323 }
324 final_resume.normalize();
325 request.events.emit(EventKind::Progress {
326 task_id: request.task_id,
327 file: relative.clone(),
328 downloaded_bytes: final_resume.completed_bytes(),
329 total_bytes: final_resume.total_size,
330 });
331 task_resume.files.insert(key, final_resume.clone());
332 request.checkpoint.save(task_resume).await?;
333
334 ensure_parent(&final_path).await?;
335 remove_if_exists(&final_path).await?;
336 if tokio::fs::try_exists(&part_name).await? {
337 tokio::fs::rename(&part_name, &final_path).await?;
338 } else {
339 tokio::fs::write(&final_path, []).await?;
340 }
341 let size = tokio::fs::metadata(&final_path).await?.len();
342 Ok(FileOutcome::new(
343 final_path, relative, size, checksum, false,
344 ))
345 }
346
347 async fn probe(&self, request: &DownloadRequest, file: &ResourceFile) -> Result<Probe> {
348 let mut attempt = 0u32;
349 loop {
350 let headers = build_headers(&request.options, &file.headers, Some("bytes=0-0"), None)?;
351 let sent = self
352 .send_follow(&file.url, &headers, &request.options, &request.cancel)
353 .await;
354 match sent {
355 Err(Error::Cancelled) => return Err(Error::Cancelled),
356 Err(error) => {
357 if attempt >= request.options.retries {
358 return Err(error);
359 }
360 sleep_or_cancel(backoff(attempt), &request.cancel).await?;
361 attempt += 1;
362 }
363 Ok((response, captured, final_url)) => {
364 let status = response.status();
365 let retry_after = retry_after(response.headers());
366 if status.is_server_error()
367 || status == StatusCode::REQUEST_TIMEOUT
368 || status == StatusCode::TOO_MANY_REQUESTS
369 {
370 drop(response);
371 if attempt >= request.options.retries {
372 return Err(http_error(status, &captured));
373 }
374 sleep_or_cancel(
375 retry_after.unwrap_or_else(|| backoff(attempt)),
376 &request.cancel,
377 )
378 .await?;
379 attempt += 1;
380 continue;
381 }
382 if matches!(
383 status,
384 StatusCode::UNAUTHORIZED
385 | StatusCode::FORBIDDEN
386 | StatusCode::NOT_FOUND
387 | StatusCode::GONE
388 ) {
389 return Err(http_error(status, &captured));
390 }
391
392 let content_range = captured
393 .get("content-range")
394 .and_then(|value| parse_content_range(value));
395 let linked_size = captured
396 .get("x-linked-size")
397 .and_then(|value| parse_u64(value));
398 let content_length = captured
399 .get("content-length")
400 .and_then(|value| parse_u64(value));
401 let (range_supported, total_size) = match status {
402 StatusCode::PARTIAL_CONTENT => (
403 true,
404 content_range
405 .and_then(|(_, _, total)| total)
406 .or(linked_size),
407 ),
408 StatusCode::OK => match content_range {
409 Some((_, _, total)) => (true, total.or(linked_size)),
415 None => (false, linked_size.or(content_length)),
416 },
417 StatusCode::RANGE_NOT_SATISFIABLE => {
418 (true, content_range.and_then(|(_, _, total)| total))
419 }
420 _ => {
421 return Err(Error::Backend(format!(
422 "unexpected HTTP status during probe: {status}"
423 )));
424 }
425 };
426
427 let checksum = captured
428 .get("x-linked-etag")
429 .and_then(|value| Checksum::from_etag(value))
430 .or_else(|| parse_digest(&captured));
431 let etag = captured
432 .get("etag")
433 .cloned()
434 .or_else(|| captured.get("x-linked-etag").cloned());
435 let last_modified = captured.get("last-modified").cloned();
436 let filename = captured
437 .get("content-disposition")
438 .and_then(|value| parse_content_disposition(value));
439 let mut headers = captured;
440 if let Some(total) = total_size {
441 headers.insert("total-size".to_string(), total.to_string());
442 }
443 return Ok(Probe {
444 final_url,
445 total_size,
446 range_supported,
447 filename,
448 etag,
449 last_modified,
450 checksum,
451 headers,
452 });
453 }
454 }
455 }
456 }
457
458 async fn send_follow(
459 &self,
460 url: &Url,
461 headers: &HeaderMap,
462 options: &shaul_core::ResolvedOptions,
463 cancel: &CancellationToken,
464 ) -> Result<(Response, BTreeMap<String, String>, Url)> {
465 let mut current = url.clone();
466 let mut headers = headers.clone();
467 let mut captured = BTreeMap::new();
468
469 for _ in 0..MAX_REDIRECTS {
470 let response = tokio::select! {
471 _ = cancel.cancelled() => return Err(Error::Cancelled),
472 response = self.client.get(current.clone()).headers(headers.clone()).send() => {
473 response.map_err(|error| Error::Backend(format!("request failed: {error}")))?
474 }
475 };
476 capture_headers(&response, &mut captured);
477 let status = response.status();
478 if matches!(
479 status,
480 StatusCode::MOVED_PERMANENTLY
481 | StatusCode::FOUND
482 | StatusCode::SEE_OTHER
483 | StatusCode::TEMPORARY_REDIRECT
484 | StatusCode::PERMANENT_REDIRECT
485 ) {
486 let location = response
487 .headers()
488 .get(LOCATION)
489 .and_then(|value| value.to_str().ok())
490 .ok_or_else(|| {
491 Error::Backend("redirect response missing Location".to_string())
492 })?;
493 let next = current
494 .join(location)
495 .map_err(|error| Error::Backend(format!("invalid redirect URL: {error}")))?;
496 if !same_origin(¤t, &next) && !options.request.allow_cross_origin_auth {
497 for name in options.request.sensitive_header_names() {
498 headers.remove(name.as_str());
499 }
500 headers.remove(AUTHORIZATION);
501 headers.remove(COOKIE);
502 }
503 current = next;
504 continue;
505 }
506 return Ok((response, captured, current));
507 }
508 Err(Error::Backend("too many redirects".to_string()))
509 }
510
511 #[allow(clippy::too_many_arguments)]
512 async fn transfer_segmented(
513 &self,
514 request: &DownloadRequest,
515 file: &ResourceFile,
516 part_name: &Path,
517 key: &str,
518 file_resume: FileResume,
519 task_resume: TaskResume,
520 progress: Arc<AtomicU64>,
521 total: Option<u64>,
522 ) -> Result<FileResume> {
523 let total =
524 total.ok_or_else(|| Error::Backend("segmented transfer without size".to_string()))?;
525 let missing = missing_ranges(total, &file_resume.completed);
526 let mut chunks = Vec::new();
527 for range in missing {
528 chunks.extend(split_range(range, request.options.segment_size));
529 }
530 if chunks.is_empty() {
531 return Ok(file_resume);
532 }
533
534 let mut state = task_resume.clone();
535 state.files.insert(key.to_string(), file_resume.clone());
536 let resume_state = Arc::new(Mutex::new(state));
537 let checkpoint_lock = Arc::new(Mutex::new(()));
538 let queue = Arc::new(std::sync::Mutex::new(VecDeque::from(chunks)));
539 let range_broken = Arc::new(AtomicBool::new(false));
540 let child_cancel = request.cancel.child_token();
541 let if_range = strong_if_range(&file_resume.validator);
542 let workers = request
543 .options
544 .connections
545 .min(queue.lock().map(|queue| queue.len()).unwrap_or(1))
546 .max(1);
547
548 let mut set = tokio::task::JoinSet::new();
549 for _ in 0..workers {
550 let backend = self.clone();
551 let request = request.clone();
552 let file = file.clone();
553 let part_name = part_name.to_path_buf();
554 let key = key.to_string();
555 let queue = queue.clone();
556 let resume_state = resume_state.clone();
557 let checkpoint_lock = checkpoint_lock.clone();
558 let range_broken = range_broken.clone();
559 let progress = progress.clone();
560 let cancel = child_cancel.clone();
561 let if_range = if_range.clone();
562 set.spawn(async move {
563 backend
564 .segment_worker(
565 &request,
566 &file,
567 &part_name,
568 &key,
569 queue,
570 resume_state,
571 checkpoint_lock,
572 range_broken,
573 progress,
574 cancel,
575 total,
576 if_range,
577 )
578 .await
579 });
580 }
581
582 let mut first_error = None;
583 while let Some(joined) = set.join_next().await {
584 match joined {
585 Ok(Ok(())) => {}
586 Ok(Err(FetchError::Cancelled)) => {}
587 Ok(Err(FetchError::RangeBroken)) => {}
588 Ok(Err(FetchError::Fatal(error))) => {
589 if first_error.is_none() {
590 first_error = Some(error);
591 }
592 child_cancel.cancel();
593 }
594 Ok(Err(FetchError::Retryable(message))) => {
595 if first_error.is_none() {
596 first_error = Some(Error::Backend(message));
597 }
598 child_cancel.cancel();
599 }
600 Err(error) => {
601 if first_error.is_none() {
602 first_error = Some(Error::Other(format!("segment worker failed: {error}")));
603 }
604 child_cancel.cancel();
605 }
606 }
607 }
608
609 if let Some(error) = first_error {
610 return Err(error);
611 }
612
613 if range_broken.load(Ordering::SeqCst) {
614 request.events.emit(EventKind::Warning {
615 task_id: Some(request.task_id),
616 message: "server ignored a ranged request; falling back to a single stream"
617 .to_string(),
618 });
619 let file_handle = tokio::fs::OpenOptions::new()
620 .create(true)
621 .write(true)
622 .truncate(false)
623 .open(part_name)
624 .await?;
625 file_handle.set_len(0).await?;
626 file_handle.sync_all().await?;
627 progress.store(0, Ordering::Relaxed);
628 let mut reset = file_resume;
629 reset.completed.clear();
630 reset.normalize();
631 return self
632 .transfer_single(
633 request,
634 file,
635 part_name,
636 key,
637 reset,
638 task_resume,
639 progress,
640 false,
641 )
642 .await;
643 }
644
645 let final_state = resume_state.lock().await.clone();
646 let mut final_resume = final_state
647 .files
648 .get(key)
649 .cloned()
650 .ok_or_else(|| Error::Backend("missing resume state after transfer".to_string()))?;
651 final_resume.completed = vec![shaul_core::ByteRange::new(0, total)?];
652 final_resume.normalize();
653 let mut snapshot = final_state;
654 snapshot.files.insert(key.to_string(), final_resume.clone());
655 request.checkpoint.save(&snapshot).await?;
656 Ok(final_resume)
657 }
658
659 #[allow(clippy::too_many_arguments)]
660 async fn segment_worker(
661 &self,
662 request: &DownloadRequest,
663 file: &ResourceFile,
664 part_name: &Path,
665 key: &str,
666 queue: Arc<std::sync::Mutex<VecDeque<shaul_core::ByteRange>>>,
667 resume_state: Arc<Mutex<TaskResume>>,
668 checkpoint_lock: Arc<Mutex<()>>,
669 range_broken: Arc<AtomicBool>,
670 progress: Arc<AtomicU64>,
671 cancel: CancellationToken,
672 total: u64,
673 if_range: Option<String>,
674 ) -> std::result::Result<(), FetchError> {
675 let mut part = tokio::fs::OpenOptions::new()
676 .write(true)
677 .open(part_name)
678 .await
679 .map_err(|error| FetchError::Fatal(Error::Io(error)))?;
680 loop {
681 if cancel.is_cancelled() || range_broken.load(Ordering::SeqCst) {
682 break;
683 }
684 let range = {
685 let mut queue = queue
686 .lock()
687 .map_err(|_| FetchError::Fatal(Error::Other("work queue poisoned".into())))?;
688 queue.pop_front()
689 };
690 let Some(range) = range else {
691 break;
692 };
693 match self
694 .fetch_range(
695 request,
696 file,
697 range,
698 &mut part,
699 &progress,
700 &cancel,
701 total,
702 if_range.as_deref(),
703 )
704 .await
705 {
706 Ok(()) => {
707 let _guard = checkpoint_lock.lock().await;
708 let snapshot = {
709 let mut state = resume_state.lock().await;
710 if let Some(entry) = state.files.get_mut(key) {
711 entry.completed.push(range);
712 entry.normalize();
713 }
714 state.clone()
715 };
716 request
717 .checkpoint
718 .save(&snapshot)
719 .await
720 .map_err(FetchError::Fatal)?;
721 }
722 Err(FetchError::Cancelled) => break,
723 Err(FetchError::RangeBroken) => {
724 range_broken.store(true, Ordering::SeqCst);
725 cancel.cancel();
726 break;
727 }
728 Err(error) => {
729 cancel.cancel();
730 return Err(error);
731 }
732 }
733 }
734 part.flush()
735 .await
736 .map_err(|error| FetchError::Fatal(Error::Io(error)))?;
737 Ok(())
738 }
739
740 #[allow(clippy::too_many_arguments)]
741 async fn fetch_range(
742 &self,
743 request: &DownloadRequest,
744 file: &ResourceFile,
745 range: shaul_core::ByteRange,
746 part: &mut tokio::fs::File,
747 progress: &AtomicU64,
748 cancel: &CancellationToken,
749 total: u64,
750 if_range: Option<&str>,
751 ) -> std::result::Result<(), FetchError> {
752 let mut attempt = 0u32;
753 loop {
754 if cancel.is_cancelled() {
755 return Err(FetchError::Cancelled);
756 }
757 let range_header = format!("bytes={}-{}", range.start, range.end - 1);
758 let headers = build_headers(
759 &request.options,
760 &file.headers,
761 Some(&range_header),
762 if_range,
763 )
764 .map_err(FetchError::Fatal)?;
765 let sent = self
766 .send_follow(&file.url, &headers, &request.options, cancel)
767 .await;
768 let (response, captured, _) = match sent {
769 Err(Error::Cancelled) => return Err(FetchError::Cancelled),
770 Err(error) => {
771 if attempt >= request.options.retries {
772 return Err(FetchError::Fatal(error));
773 }
774 sleep_or_cancel(backoff(attempt), cancel)
775 .await
776 .map_err(FetchError::Fatal)?;
777 attempt += 1;
778 continue;
779 }
780 Ok(response) => response,
781 };
782 let status = response.status();
783 let retry_after = retry_after(response.headers());
784
785 if status == StatusCode::RANGE_NOT_SATISFIABLE {
786 return Err(FetchError::Fatal(Error::Backend(format!(
787 "range {} is not satisfiable",
788 range_header
789 ))));
790 }
791 if status.is_server_error()
792 || status == StatusCode::REQUEST_TIMEOUT
793 || status == StatusCode::TOO_MANY_REQUESTS
794 {
795 drop(response);
796 if attempt >= request.options.retries {
797 return Err(FetchError::Fatal(http_error(status, &captured)));
798 }
799 sleep_or_cancel(retry_after.unwrap_or_else(|| backoff(attempt)), cancel)
800 .await
801 .map_err(FetchError::Fatal)?;
802 attempt += 1;
803 continue;
804 }
805
806 let content_range = captured
807 .get("content-range")
808 .and_then(|value| parse_content_range(value));
809 if status == StatusCode::OK && content_range.is_none() {
810 return Err(FetchError::RangeBroken);
811 }
812 let is_partial = status == StatusCode::PARTIAL_CONTENT
813 || (status == StatusCode::OK && content_range.is_some());
814 if !is_partial {
815 return Err(FetchError::Fatal(http_error(status, &captured)));
816 }
817 let parsed = content_range.ok_or_else(|| {
818 FetchError::Fatal(Error::Backend(
819 "ranged response missing a valid Content-Range".to_string(),
820 ))
821 })?;
822 if parsed.0 != range.start || parsed.1 + 1 != range.end {
823 return Err(FetchError::Fatal(Error::Backend(format!(
824 "server returned range {}-{}, expected {}-{}",
825 parsed.0,
826 parsed.1,
827 range.start,
828 range.end - 1
829 ))));
830 }
831 if let Some(server_total) = parsed.2
832 && server_total != total
833 {
834 return Err(FetchError::Fatal(Error::Backend(format!(
835 "file size changed from {total} to {server_total}"
836 ))));
837 }
838
839 part.seek(SeekFrom::Start(range.start))
840 .await
841 .map_err(|error| FetchError::Fatal(Error::Io(error)))?;
842 let mut written = 0u64;
843 let mut stream = response.bytes_stream();
844 let retry_message;
845 loop {
846 let next = tokio::select! {
847 _ = cancel.cancelled() => return Err(FetchError::Cancelled),
848 next = tokio::time::timeout(request.options.timeout, stream.next()) => next,
849 };
850 match next {
851 Ok(Some(Ok(chunk))) => {
852 if written + chunk.len() as u64 > range.len() {
853 retry_message =
854 Some("server sent more data than requested".to_string());
855 break;
856 }
857 part.write_all(&chunk)
858 .await
859 .map_err(|error| FetchError::Fatal(Error::Io(error)))?;
860 written += chunk.len() as u64;
861 progress.fetch_add(chunk.len() as u64, Ordering::Relaxed);
862 }
863 Ok(Some(Err(error))) => {
864 retry_message = Some(format!("stream error: {error}"));
865 break;
866 }
867 Ok(None) => {
868 if written == range.len() {
869 return Ok(());
870 }
871 retry_message = Some(format!(
872 "short ranged response: {written} of {} bytes",
873 range.len()
874 ));
875 break;
876 }
877 Err(_) => {
878 retry_message = Some("read timed out".to_string());
879 break;
880 }
881 }
882 }
883
884 let current = progress.load(Ordering::Relaxed);
887 progress.fetch_sub(written.min(current), Ordering::Relaxed);
888 if attempt >= request.options.retries {
889 return Err(FetchError::Retryable(
890 retry_message.unwrap_or_else(|| "range transfer failed".to_string()),
891 ));
892 }
893 sleep_or_cancel(backoff(attempt), cancel)
894 .await
895 .map_err(FetchError::Fatal)?;
896 attempt += 1;
897 }
898 }
899
900 #[allow(clippy::too_many_arguments)]
901 async fn transfer_single(
902 &self,
903 request: &DownloadRequest,
904 file: &ResourceFile,
905 part_name: &Path,
906 key: &str,
907 mut file_resume: FileResume,
908 mut task_resume: TaskResume,
909 progress: Arc<AtomicU64>,
910 range_supported: bool,
911 ) -> Result<FileResume> {
912 let mut offset = if request.options.resume && range_supported {
913 contiguous_end(&file_resume.completed)
914 } else {
915 0
916 };
917 if file_resume
918 .total_size
919 .map(|total| offset >= total)
920 .unwrap_or(false)
921 {
922 return Ok(file_resume);
923 }
924
925 let mut part = tokio::fs::OpenOptions::new()
926 .create(true)
927 .write(true)
928 .truncate(false)
929 .open(part_name)
930 .await?;
931 if offset == 0 {
932 part.set_len(0).await?;
933 }
934 part.seek(SeekFrom::Start(offset)).await?;
935
936 let mut attempt = 0u32;
937 loop {
938 if request.cancel.is_cancelled() {
939 checkpoint_single(request, &mut task_resume, key, &mut file_resume, offset).await?;
940 return Err(Error::Cancelled);
941 }
942 let range = if offset > 0 && range_supported {
943 Some(format!("bytes={offset}-"))
944 } else {
945 None
946 };
947 let if_range = if offset > 0 {
948 strong_if_range(&file_resume.validator)
949 } else {
950 None
951 };
952 let headers = build_headers(
953 &request.options,
954 &file.headers,
955 range.as_deref(),
956 if_range.as_deref(),
957 )?;
958 let sent = self
959 .send_follow(&file.url, &headers, &request.options, &request.cancel)
960 .await;
961 let (response, captured, _) = match sent {
962 Err(Error::Cancelled) => {
963 checkpoint_single(request, &mut task_resume, key, &mut file_resume, offset)
964 .await?;
965 return Err(Error::Cancelled);
966 }
967 Err(error) => {
968 checkpoint_single(request, &mut task_resume, key, &mut file_resume, offset)
969 .await?;
970 if attempt >= request.options.retries {
971 return Err(error);
972 }
973 sleep_or_cancel(backoff(attempt), &request.cancel).await?;
974 attempt += 1;
975 continue;
976 }
977 Ok(response) => response,
978 };
979 let status = response.status();
980 let retry_after = retry_after(response.headers());
981
982 let content_range = captured
983 .get("content-range")
984 .and_then(|value| parse_content_range(value));
985 if status.is_server_error()
986 || status == StatusCode::REQUEST_TIMEOUT
987 || status == StatusCode::TOO_MANY_REQUESTS
988 {
989 drop(response);
990 checkpoint_single(request, &mut task_resume, key, &mut file_resume, offset).await?;
991 if attempt >= request.options.retries {
992 return Err(http_error(status, &captured));
993 }
994 sleep_or_cancel(
995 retry_after.unwrap_or_else(|| backoff(attempt)),
996 &request.cancel,
997 )
998 .await?;
999 attempt += 1;
1000 continue;
1001 }
1002
1003 let is_partial = status == StatusCode::PARTIAL_CONTENT
1004 || (status == StatusCode::OK && content_range.is_some());
1005 if is_partial {
1006 if let Some((start, _, _)) = content_range
1007 && offset > 0
1008 && start != offset
1009 {
1010 return Err(Error::Backend(format!(
1011 "server resumed at {start}, expected {offset}"
1012 )));
1013 }
1014 } else if status == StatusCode::OK {
1015 if offset > 0 {
1016 part.set_len(0).await?;
1017 part.seek(SeekFrom::Start(0)).await?;
1018 offset = 0;
1019 progress.store(0, Ordering::Relaxed);
1020 }
1021 } else {
1022 return Err(http_error(status, &captured));
1023 }
1024
1025 part.seek(SeekFrom::Start(offset)).await?;
1026 let mut stream = response.bytes_stream();
1027 let mut since_checkpoint = 0u64;
1028 let retry_message;
1029 loop {
1030 let next = tokio::select! {
1031 _ = request.cancel.cancelled() => {
1032 checkpoint_single(request, &mut task_resume, key, &mut file_resume, offset).await?;
1033 part.sync_all().await?;
1034 return Err(Error::Cancelled);
1035 }
1036 next = tokio::time::timeout(request.options.timeout, stream.next()) => next,
1037 };
1038 match next {
1039 Ok(Some(Ok(chunk))) => {
1040 if let Some(total) = file_resume.total_size
1041 && offset + chunk.len() as u64 > total
1042 {
1043 return Err(Error::Backend(
1044 "server sent more data than the known file size".to_string(),
1045 ));
1046 }
1047 part.write_all(&chunk).await?;
1048 offset += chunk.len() as u64;
1049 progress.fetch_add(chunk.len() as u64, Ordering::Relaxed);
1050 since_checkpoint += chunk.len() as u64;
1051 if since_checkpoint >= request.options.segment_size {
1052 checkpoint_single(
1053 request,
1054 &mut task_resume,
1055 key,
1056 &mut file_resume,
1057 offset,
1058 )
1059 .await?;
1060 since_checkpoint = 0;
1061 }
1062 }
1063 Ok(Some(Err(error))) => {
1064 retry_message = Some(format!("stream error: {error}"));
1065 break;
1066 }
1067 Ok(None) => {
1068 if let Some(total) = file_resume.total_size
1069 && offset < total
1070 {
1071 retry_message =
1072 Some(format!("short response: {offset} of {total} bytes"));
1073 break;
1074 }
1075 part.flush().await?;
1076 part.sync_all().await?;
1077 file_resume.total_size = file_resume.total_size.or(Some(offset));
1078 file_resume.completed = if offset > 0 {
1079 vec![shaul_core::ByteRange::new(0, offset)?]
1080 } else {
1081 Vec::new()
1082 };
1083 file_resume.normalize();
1084 task_resume
1085 .files
1086 .insert(key.to_string(), file_resume.clone());
1087 request.checkpoint.save(&task_resume).await?;
1088 return Ok(file_resume);
1089 }
1090 Err(_) => {
1091 retry_message = Some("read timed out".to_string());
1092 break;
1093 }
1094 }
1095 }
1096
1097 part.flush().await?;
1098 checkpoint_single(request, &mut task_resume, key, &mut file_resume, offset).await?;
1099 if attempt >= request.options.retries {
1100 return Err(Error::Backend(
1101 retry_message.unwrap_or_else(|| "download failed".to_string()),
1102 ));
1103 }
1104 sleep_or_cancel(backoff(attempt), &request.cancel).await?;
1105 attempt += 1;
1106 }
1107 }
1108}
1109
1110#[async_trait]
1111impl DownloadBackend for HttpBackend {
1112 fn name(&self) -> &'static str {
1113 "http"
1114 }
1115
1116 fn capabilities(&self) -> BackendCapabilities {
1117 BackendCapabilities::new(true, true, true)
1118 }
1119
1120 fn supports(&self, resource: &Resource) -> bool {
1121 matches!(&resource.kind, ResourceKind::Http)
1122 }
1123
1124 async fn download(&self, request: DownloadRequest) -> Result<DownloadOutcome> {
1125 let mut task_resume = request.resume.clone().unwrap_or_default();
1126 let mut outcomes = Vec::new();
1127 for file in &request.resource.files {
1128 if request.cancel.is_cancelled() {
1129 return Err(Error::Cancelled);
1130 }
1131 let outcome = self.download_file(&request, file, &mut task_resume).await?;
1132 outcomes.push(outcome);
1133 request.checkpoint.save(&task_resume).await?;
1134 }
1135 Ok(DownloadOutcome::Completed(outcomes))
1136 }
1137}
1138
1139struct Probe {
1140 final_url: Url,
1141 total_size: Option<u64>,
1142 range_supported: bool,
1143 filename: Option<String>,
1144 etag: Option<String>,
1145 last_modified: Option<String>,
1146 checksum: Option<Checksum>,
1147 headers: BTreeMap<String, String>,
1148}
1149
1150impl Probe {
1151 fn validator(&self) -> Validator {
1152 Validator {
1153 etag: self.etag.clone(),
1154 last_modified: self.last_modified.clone(),
1155 size: self.total_size,
1156 }
1157 }
1158}
1159
1160#[derive(Debug)]
1161enum FetchError {
1162 RangeBroken,
1163 Cancelled,
1164 Retryable(String),
1165 Fatal(Error),
1166}
1167
1168fn build_headers(
1169 options: &shaul_core::ResolvedOptions,
1170 file_headers: &[Header],
1171 range: Option<&str>,
1172 if_range: Option<&str>,
1173) -> Result<HeaderMap> {
1174 let mut headers = HeaderMap::new();
1175 for header in &options.request.headers {
1176 if !is_reserved(&header.name) {
1177 insert_header(
1178 &mut headers,
1179 &header.name,
1180 &shaul_core::expand_env(&header.value)?,
1181 )?;
1182 }
1183 }
1184 for header in file_headers {
1185 if !is_reserved(&header.name) {
1186 insert_header(
1187 &mut headers,
1188 &header.name,
1189 &shaul_core::expand_env(&header.value)?,
1190 )?;
1191 }
1192 }
1193
1194 if !headers.contains_key(AUTHORIZATION)
1195 && let Some(auth) = &options.request.auth
1196 {
1197 match auth {
1198 shaul_core::Auth::Bearer { token } => {
1199 insert_header(
1200 &mut headers,
1201 "authorization",
1202 &format!("Bearer {}", shaul_core::expand_env(token)?),
1203 )?;
1204 }
1205 shaul_core::Auth::Basic { username, password } => {
1206 let encoded = base64::engine::general_purpose::STANDARD.encode(format!(
1207 "{}:{}",
1208 shaul_core::expand_env(username)?,
1209 shaul_core::expand_env(password)?
1210 ));
1211 insert_header(&mut headers, "authorization", &format!("Basic {encoded}"))?;
1212 }
1213 shaul_core::Auth::Header(header) => {
1214 insert_header(
1215 &mut headers,
1216 &header.name,
1217 &shaul_core::expand_env(&header.value)?,
1218 )?;
1219 }
1220 }
1221 }
1222
1223 headers.insert(ACCEPT_ENCODING, HeaderValue::from_static("identity"));
1224 insert_header(&mut headers, "user-agent", &options.user_agent)?;
1225 if let Some(range) = range {
1226 insert_header(&mut headers, "range", range)?;
1227 }
1228 if let Some(if_range) = if_range {
1229 insert_header(&mut headers, "if-range", if_range)?;
1230 }
1231 Ok(headers)
1232}
1233
1234fn insert_header(headers: &mut HeaderMap, name: &str, value: &str) -> Result<()> {
1235 let name = HeaderName::from_bytes(name.as_bytes())
1236 .map_err(|_| Error::Config(format!("invalid header name: {name}")))?;
1237 let value = HeaderValue::from_str(value)
1238 .map_err(|_| Error::Config(format!("invalid value for header: {name}")))?;
1239 headers.insert(name, value);
1240 Ok(())
1241}
1242
1243fn is_reserved(name: &str) -> bool {
1244 RESERVED_HEADERS
1245 .iter()
1246 .any(|reserved| name.eq_ignore_ascii_case(reserved))
1247}
1248
1249fn capture_headers(response: &Response, captured: &mut BTreeMap<String, String>) {
1250 for name in CAPTURE_HEADERS {
1251 if let Some(value) = response.headers().get(*name)
1252 && let Ok(value) = value.to_str()
1253 {
1254 captured.insert((*name).to_string(), value.to_string());
1255 }
1256 }
1257}
1258
1259fn redacted_url(url: &Url) -> String {
1260 let mut url = url.clone();
1261 url.set_query(None);
1262 url.set_fragment(None);
1263 url.to_string()
1264}
1265
1266fn same_origin(left: &Url, right: &Url) -> bool {
1267 left.scheme() == right.scheme()
1268 && left.host_str() == right.host_str()
1269 && left.port_or_known_default() == right.port_or_known_default()
1270}
1271
1272fn parse_content_range(value: &str) -> Option<(u64, u64, Option<u64>)> {
1273 let rest = value.trim().strip_prefix("bytes")?.trim();
1274 let (range, total) = rest.split_once('/')?;
1275 let total = if total.trim() == "*" {
1276 None
1277 } else {
1278 Some(total.trim().parse::<u64>().ok()?)
1279 };
1280 if range.trim() == "*" {
1281 return Some((0, 0, total));
1282 }
1283 let (start, end) = range.split_once('-')?;
1284 Some((
1285 start.trim().parse::<u64>().ok()?,
1286 end.trim().parse::<u64>().ok()?,
1287 total,
1288 ))
1289}
1290
1291fn parse_u64(value: &str) -> Option<u64> {
1292 value.trim().parse::<u64>().ok()
1293}
1294
1295fn retry_after(headers: &HeaderMap) -> Option<Duration> {
1296 headers
1297 .get("retry-after")
1298 .and_then(|value| value.to_str().ok())
1299 .and_then(|value| value.trim().parse::<u64>().ok())
1300 .map(|seconds| Duration::from_secs(seconds.min(60)))
1301}
1302
1303fn backoff(attempt: u32) -> Duration {
1304 Duration::from_millis(250u64.saturating_mul(1u64 << attempt.min(5)))
1305}
1306
1307async fn sleep_or_cancel(duration: Duration, cancel: &CancellationToken) -> Result<()> {
1308 tokio::select! {
1309 _ = cancel.cancelled() => Err(Error::Cancelled),
1310 _ = tokio::time::sleep(duration) => Ok(()),
1311 }
1312}
1313
1314fn http_error(status: StatusCode, captured: &BTreeMap<String, String>) -> Error {
1315 let code = captured
1316 .get("x-error-code")
1317 .map(|code| format!(" ({code})"))
1318 .unwrap_or_default();
1319 Error::Backend(format!("HTTP {status}{code}"))
1320}
1321
1322fn parse_digest(captured: &BTreeMap<String, String>) -> Option<Checksum> {
1323 if let Some(value) = captured.get("digest") {
1324 for part in value.split(',') {
1325 let Some((algorithm, encoded)) = part.trim().split_once('=') else {
1326 continue;
1327 };
1328 let algorithm = match algorithm.trim().to_ascii_lowercase().as_str() {
1329 "sha-256" | "sha256" => ChecksumAlgorithm::Sha256,
1330 "sha-512" | "sha512" => ChecksumAlgorithm::Sha512,
1331 "sha" | "sha-1" | "sha1" => ChecksumAlgorithm::Sha1,
1332 "md5" => ChecksumAlgorithm::Md5,
1333 _ => continue,
1334 };
1335 if let Ok(bytes) = base64::engine::general_purpose::STANDARD.decode(encoded.trim())
1336 && let Ok(checksum) = Checksum::new(algorithm, hex::encode(bytes))
1337 {
1338 return Some(checksum);
1339 }
1340 }
1341 }
1342 if let Some(value) = captured.get("content-md5")
1343 && let Ok(bytes) = base64::engine::general_purpose::STANDARD.decode(value.trim())
1344 && let Ok(checksum) = Checksum::new(ChecksumAlgorithm::Md5, hex::encode(bytes))
1345 {
1346 return Some(checksum);
1347 }
1348 None
1349}
1350
1351fn strong_if_range(validator: &Validator) -> Option<String> {
1352 if let Some(etag) = &validator.etag
1353 && !etag.starts_with("W/")
1354 {
1355 return Some(etag.clone());
1356 }
1357 validator.last_modified.clone()
1358}
1359
1360fn contiguous_end(completed: &[shaul_core::ByteRange]) -> u64 {
1361 let merged = merge_ranges(completed.to_vec());
1362 if merged.len() == 1 && merged[0].start == 0 {
1363 merged[0].end
1364 } else {
1365 0
1366 }
1367}
1368
1369fn manifest_path(destination: &Path, value: &str) -> PathBuf {
1370 let path = Path::new(value);
1371 if path.is_absolute() {
1372 path.to_path_buf()
1373 } else {
1374 destination.join(path)
1375 }
1376}
1377
1378fn path_for_manifest(destination: &Path, path: &Path) -> String {
1379 match path.strip_prefix(destination) {
1380 Ok(relative) => relative.to_string_lossy().replace('\\', "/"),
1381 Err(_) => path.to_string_lossy().to_string(),
1382 }
1383}
1384
1385fn relative_path(destination: &Path, path: &Path) -> String {
1386 match path.strip_prefix(destination) {
1387 Ok(relative) => relative.to_string_lossy().replace('\\', "/"),
1388 Err(_) => path
1389 .file_name()
1390 .map(|name| name.to_string_lossy().to_string())
1391 .unwrap_or_else(|| "download".to_string()),
1392 }
1393}
1394
1395fn resolve_final_path(
1396 request: &DownloadRequest,
1397 file: &ResourceFile,
1398 probe_filename: Option<&str>,
1399 default_name: &str,
1400) -> Result<(PathBuf, String)> {
1401 let single = request.resource.files.len() == 1;
1402 let response_name = if file.use_response_filename {
1403 probe_filename.map(sanitize_file_name)
1404 } else {
1405 None
1406 };
1407 let name = response_name
1408 .clone()
1409 .unwrap_or_else(|| default_name.to_string());
1410
1411 if let Some(output) = &request.options.output {
1412 if single {
1413 if output.is_dir() {
1414 let relative = if file.use_response_filename {
1415 name.clone()
1416 } else {
1417 file.path.clone()
1418 };
1419 let path = output.join(relative.replace('/', std::path::MAIN_SEPARATOR_STR));
1420 return Ok((path.clone(), relative_path(&request.destination, &path)));
1421 }
1422 return Ok((output.clone(), relative_path(&request.destination, output)));
1423 }
1424 let path = output.join(file.path.replace('/', std::path::MAIN_SEPARATOR_STR));
1425 return Ok((path.clone(), relative_path(&request.destination, &path)));
1426 }
1427
1428 let relative = if single && file.use_response_filename {
1429 name
1430 } else {
1431 file.path.clone()
1432 };
1433 let path = request
1434 .destination
1435 .join(relative.replace('/', std::path::MAIN_SEPARATOR_STR));
1436 Ok((path.clone(), relative_path(&request.destination, &path)))
1437}
1438
1439async fn apply_conflict_policy(
1440 path: PathBuf,
1441 options: &shaul_core::ResolvedOptions,
1442) -> Result<PathBuf> {
1443 if !tokio::fs::try_exists(&path).await? {
1444 return Ok(path);
1445 }
1446 if options.overwrite {
1447 remove_if_exists(&path).await?;
1448 return Ok(path);
1449 }
1450 if options.rename_on_conflict {
1451 return unique_path(&path).await;
1452 }
1453 Err(Error::Backend(format!(
1454 "destination already exists: {} (use --force or --rename)",
1455 path.display()
1456 )))
1457}
1458
1459async fn unique_path(path: &Path) -> Result<PathBuf> {
1460 let stem = path
1461 .file_stem()
1462 .map(|stem| stem.to_string_lossy().to_string())
1463 .unwrap_or_else(|| "download".to_string());
1464 let extension = path
1465 .extension()
1466 .map(|extension| format!(".{}", extension.to_string_lossy()))
1467 .unwrap_or_default();
1468 for index in 1..10_000 {
1469 let candidate = path.with_file_name(format!("{stem}-{index}{extension}"));
1470 if !tokio::fs::try_exists(&candidate).await? {
1471 return Ok(candidate);
1472 }
1473 }
1474 Err(Error::Backend(
1475 "could not find a free destination name".to_string(),
1476 ))
1477}
1478
1479fn now_ms() -> u64 {
1480 SystemTime::now()
1481 .duration_since(UNIX_EPOCH)
1482 .map(|duration| duration.as_millis() as u64)
1483 .unwrap_or(0)
1484}
1485
1486async fn ensure_parent(path: &Path) -> Result<()> {
1487 if let Some(parent) = path.parent()
1488 && !parent.as_os_str().is_empty()
1489 {
1490 tokio::fs::create_dir_all(parent).await?;
1491 }
1492 Ok(())
1493}
1494
1495async fn remove_if_exists(path: &Path) -> Result<()> {
1496 match tokio::fs::remove_file(path).await {
1497 Ok(()) => Ok(()),
1498 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
1499 Err(error) => Err(Error::Io(error)),
1500 }
1501}
1502
1503async fn checkpoint_single(
1504 request: &DownloadRequest,
1505 task_resume: &mut TaskResume,
1506 key: &str,
1507 file_resume: &mut FileResume,
1508 offset: u64,
1509) -> Result<()> {
1510 file_resume.completed = if offset > 0 {
1511 vec![shaul_core::ByteRange::new(0, offset)?]
1512 } else {
1513 Vec::new()
1514 };
1515 file_resume.normalize();
1516 task_resume
1517 .files
1518 .insert(key.to_string(), file_resume.clone());
1519 request.checkpoint.save(task_resume).await
1520}
1521
1522fn spawn_progress_reporter(
1523 events: EventSink,
1524 task_id: TaskId,
1525 file: String,
1526 total: Option<u64>,
1527 downloaded: Arc<AtomicU64>,
1528 stop: CancellationToken,
1529) -> tokio::task::JoinHandle<()> {
1530 tokio::spawn(async move {
1531 let mut ticker = tokio::time::interval(Duration::from_millis(100));
1532 ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
1533 loop {
1534 tokio::select! {
1535 _ = ticker.tick() => {
1536 events.emit(EventKind::Progress {
1537 task_id,
1538 file: file.clone(),
1539 downloaded_bytes: downloaded.load(Ordering::Relaxed),
1540 total_bytes: total,
1541 });
1542 }
1543 _ = stop.cancelled() => {
1544 events.emit(EventKind::Progress {
1545 task_id,
1546 file,
1547 downloaded_bytes: downloaded.load(Ordering::Relaxed),
1548 total_bytes: total,
1549 });
1550 break;
1551 }
1552 }
1553 }
1554 })
1555}